def validation_on_dilated_lesions(normal_validation_dir,
dilated_validation_dir,
gt_dir,
evaluation_labels,
recompute=True):
utils.mkdir(dilated_validation_dir)
list_validation_subdir = utils.list_subfolders(normal_validation_dir)
loop_info = utils.LoopInfo(len(list_validation_subdir), 5, 'validating',
True)
for val_idx, validation_subdir in enumerate(list_validation_subdir):
loop_info.update(val_idx)
# dilate lesion
dilated_validation_subdir = os.path.join(
dilated_validation_dir, os.path.basename(validation_subdir))
dilate_lesions(validation_subdir,
dilated_validation_subdir,
recompute=recompute)
# compute new dice scores
path_dice = os.path.join(dilated_validation_subdir, 'dice.npy')
dice_evaluation(gt_dir,
dilated_validation_subdir,
evaluation_labels,
path_dice=path_dice,
recompute=recompute)
def build_longitudinal_consensus(labels_dir_1,
labels_dir_2,
result_dir,
recompute=True):
# create result dir
utils.mkdir(result_dir)
# list all segmentations
path_labels_1 = utils.list_files(labels_dir_1)
path_labels_2 = utils.list_files(labels_dir_2)
for path_lab_1, path_lab_2 in zip(path_labels_1, path_labels_2):
# check if result is already saved
path_result = os.path.join(result_dir, os.path.basename(path_lab_1))
if (not os.path.isfile(path_result)) | recompute:
# load volumes
lab_1, aff, h = utils.load_volume(path_lab_1, im_only=False)
lab_2 = utils.load_volume(path_lab_2)
# compute and save consensus
dist_masp_1 = edit_volumes.compute_distance_map(lab_1,
crop_margin=20)
dist_masp_2 = edit_volumes.compute_distance_map(lab_2,
crop_margin=20)
consensus = (np.mean(np.stack([dist_masp_1, dist_masp_2], axis=-1),
axis=-1) > 0) * 1
utils.save_volume(consensus, aff, h, path_result)
def dilate_lesions(labels_dir, result_dir, recompute=True):
utils.mkdir(result_dir)
path_labels = utils.list_images_in_folder(labels_dir)
for path_label in path_labels:
path_result_label = os.path.join(result_dir,
os.path.basename(path_label))
if (not os.path.isfile(path_result_label)) | recompute:
label, aff, h = utils.load_volume(path_label, im_only=False)
# define lesion, WM, and LV masks
WM = (label == 2) | (label == 41)
lesion = label == 77
LV_and_lesion = (label == 4) | lesion
# morphological operations to bridge the gaps between lesions and LV
morph_struct = utils.build_binary_structure(2, len(WM.shape))
LV_and_lesion = binary_dilation(LV_and_lesion, morph_struct)
LV_and_lesion = binary_erosion(LV_and_lesion, morph_struct)
lesion = (LV_and_lesion & WM) | lesion
label[lesion] = 77
# save new label maps
utils.save_volume(label, aff, h, path_result_label)
def run_validation_on_aseg_gt(list_supervised_model_dir,
list_aseg_gt_dir,
path_label_list,
recompute=False):
list_main_samseg_validation_dir = [
os.path.join(p, 'validation_samseg') for p in list_supervised_model_dir
]
# loop over architectures
for (main_samseg_validation_dir,
gt_dir) in zip(list_main_samseg_validation_dir, list_aseg_gt_dir):
# list model subdirs
main_aseg_validation_dir = os.path.join(
os.path.dirname(main_samseg_validation_dir), 'validation')
utils.mkdir(main_aseg_validation_dir)
list_samseg_validation_subdir = utils.list_subfolders(
main_samseg_validation_dir)
# lover over models
for samseg_validation_subdir in list_samseg_validation_subdir:
# create equivalent aseg subdir
aseg_validation_subdir = os.path.join(
main_aseg_validation_dir,
os.path.basename(samseg_validation_subdir))
utils.mkdir(aseg_validation_subdir)
path_aseg_dice = os.path.join(aseg_validation_subdir, 'dice.npy')
# compute dice with aseg gt
if (not os.path.isfile(path_aseg_dice)) | recompute:
dice_evaluation(gt_dir, samseg_validation_subdir,
path_label_list, path_aseg_dice)
def estimate_t2_cropping(image_dir, result_dir=None, dilation=5):
"""This function takes all the hippocampus images (with 2 channels) within the specified directory, and estimates
the cropping dimensions around the hippocampus in the t2 channel.
It returns the mean and sts deviation for the minimal and maximal croppings, proportional to image size.
:param image_dir: path of the folder containing hippocampus images
:param result_dir: if not None, path of the folder where to write the computed statistics.
:param dilation: dilation coefficient used to extract full brain mask. Default is 5.
:returns t2_cropping_stats: numpy vector of size 4 [mean min crop, std min crop, mean max crop, std max crop]
"""
# create result dir
if result_dir is not None:
utils.mkdir(result_dir)
# loop through images
list_image_paths = utils.list_images_in_folder(image_dir)
max_cropping_proportions = np.zeros(len(list_image_paths))
min_cropping_proportions = np.zeros(len(list_image_paths))
loop_info = utils.LoopInfo(len(list_image_paths), 10, 'processing')
for im_idx, image_path in enumerate(list_image_paths):
loop_info.update(im_idx)
# load t2 channel
im = utils.load_volume(image_path)
t2 = im[..., 1]
shape = t2.shape
hdim = int(np.argmax(shape))
# mask image
_, mask = edit_volumes.mask_volume(t2,
threshold=0,
dilate=dilation,
return_mask=True)
# find cropping indices
indices = np.nonzero(mask)[hdim]
min_cropping_proportions[im_idx] = np.maximum(
np.min(indices) + int(dilation / 2), 0) / shape[hdim]
max_cropping_proportions[im_idx] = np.minimum(
np.max(indices) - int(dilation / 2), shape[hdim]) / shape[hdim]
# compute and save stats
t2_cropping_stats = np.array([
np.mean(min_cropping_proportions),
np.std(min_cropping_proportions),
np.mean(max_cropping_proportions),
np.std(max_cropping_proportions)
])
# save stats if necessary
if result_dir is not None:
np.save(os.path.join(result_dir, 't2_cropping_stats.npy'),
t2_cropping_stats)
return t2_cropping_stats
def train_model(model,
generator,
learning_rate,
lr_decay,
n_epochs,
n_steps,
model_dir,
metric_type,
path_checkpoint=None):
# prepare log folder
log_dir = os.path.join(model_dir, 'logs')
utils.mkdir(log_dir)
# model saving callback
save_file_name = os.path.join(model_dir, '%s_{epoch:03d}.h5' % metric_type)
callbacks = [KC.ModelCheckpoint(save_file_name, verbose=1)]
# TensorBoard callback
if metric_type == 'dice':
callbacks.append(KC.TensorBoard(log_dir=log_dir, histogram_freq=0, write_graph=True, write_images=False))
compile_model = True
init_epoch = 0
if path_checkpoint is not None:
if metric_type in path_checkpoint:
custom_l2i = {key: value for (key, value) in getmembers(l2i_layers, isclass) if key != 'Layer'}
custom_nrn = {key: value for (key, value) in getmembers(nrn_layers, isclass) if key != 'Layer'}
custom_objects = {**custom_l2i, **custom_nrn, 'tf': tf, 'keras': keras, 'loss': metrics.IdentityLoss().loss}
model = models.load_model(path_checkpoint, custom_objects=custom_objects)
compile_model = False
init_epoch = int(os.path.basename(path_checkpoint).split(metric_type)[1][1:-3])
else:
model.load_weights(path_checkpoint, by_name=True)
# compile
if compile_model:
model.compile(optimizer=Adam(lr=learning_rate, decay=lr_decay),
loss=metrics.IdentityLoss().loss,
loss_weights=[1.0])
# fit
model.fit_generator(generator,
epochs=n_epochs,
steps_per_epoch=n_steps,
callbacks=callbacks,
initial_epoch=init_epoch)
def paste_lesions_on_buckner(lesion_dir,
buckner_dir,
result_dir,
dilate=2,
recompute=False):
path_lesions = utils.list_images_in_folder(lesion_dir)
path_buckners = utils.list_images_in_folder(buckner_dir)
utils.mkdir(result_dir)
# loop over buckner label maps
loop_info = utils.LoopInfo(len(path_buckners), 1, 'processing', True)
for idx_buckner, path_buckner in enumerate(path_buckners):
loop_info.update(idx_buckner)
buckner_name = os.path.basename(path_buckner).replace(
'_seg', '').replace('.nii.gz', '')
buckner = utils.load_volume(path_buckner)
WM = (buckner == 2) | (buckner == 7) | (buckner == 16) | (
buckner == 41) | (buckner == 46)
# loop over challenge data
for path_lesion in path_lesions:
lesion_name = os.path.basename(path_lesion).replace(
'.samseg_and_lesions.nii.gz', '')
path_result = os.path.join(
result_dir, buckner_name + '_' + lesion_name + '.nii.gz')
if (not os.path.isfile(path_result)) | recompute:
lesion = utils.load_volume(path_lesion)
assert lesion.shape == buckner.shape, 'lesions should have same shape as buckner labels'
# define lesion, WM, and LV masks
lesion = (lesion == 77) & WM
LV_and_lesion = (buckner == 4) | lesion
# morphological operations to bridge the gaps between lesions and LV
morph_struct = utils.build_binary_structure(
dilate, len(lesion.shape))
lesion = binary_dilation(LV_and_lesion, morph_struct)
lesion = binary_erosion(lesion, morph_struct)
lesion = lesion & WM
buckner_lesions = np.where(lesion, 77, buckner)
# save map
utils.save_volume(buckner_lesions, None, None, path_result)
def downsample_upsample_flair(flair_image_dir, recompute=True):
# downsample images
downsample_image_dir = flair_image_dir + '_downsampled_1_1_2.2'
utils.mkdir(downsample_image_dir)
edit_volumes.mri_convert_images_in_dir(flair_image_dir,
downsample_image_dir,
interpolation='nearest',
voxsize=[1, 1, 2.2],
recompute=recompute)
# upsample images
upsample_image_dir = flair_image_dir + '_resampled_back_to_1_1_1'
utils.mkdir(upsample_image_dir)
edit_volumes.mri_convert_images_in_dir(downsample_image_dir,
upsample_image_dir,
reference_dir=flair_image_dir,
recompute=recompute)
def prepare_hippo_testing_images(main_image_dir,
main_result_dir,
target_res,
padding_margin=85,
delete_intermediate_files=True,
path_freesurfer='/usr/local/freesurfer/',
verbose=True,
recompute=True):
"""This function creates multi-modal images of the right and left hippocampi at the target resolution.
In that purpose it loops over subjects (assumed to be sorted between healthy and AD subfolders) and calls
preprocess_adni_hippo on each of them.
:param main_image_dir: path of main directory with images to prepare for testing. Should be organised as follows:
main_image_dir/state_dir(AD or healthy)/subject_dir/images(t1.mgz, t2.mgz, and aseg.mgz)
:param main_result_dir: path of main directory where prepared images and labels will be writen.
Will be organised as follows: main_result_dir/state_dir(AD or healthy)/subject_dir/images(hippo_left.nii.gz,
hippo_right.nii.gz, hippo_left_aseg.nii.gz, hippo_right_aseg.nii.gz)
:param target_res: resolution at which to resample the label maps, and the images.
Can be a number (isotropic resolution), a sequence, or a 1d numpy array.
:param padding_margin: (optional) margin to add around hippocampi when cropping
:param delete_intermediate_files: (optional) whether to delete temporary files. Default is True.
:param path_freesurfer: (optional) path of FreeSurfer home, to use mri_convert
:param verbose: (optional) whether to print out mri_convert output when resampling images.
:param recompute: (optional) whether to recompute result files even if they already exists"""
# create results dir
utils.mkdir(main_result_dir)
# loop over states (i.e. AD and healthy)
list_states = utils.list_subfolders(main_image_dir, whole_path=False)
for state in list_states:
# create state directory in result folder
state_dir = os.path.join(main_image_dir, state)
result_state_dir = os.path.join(main_result_dir, state)
utils.mkdir(result_state_dir)
# loop over subjects
list_subjects = utils.list_subfolders(state_dir, whole_path=False)
for subject in list_subjects:
# create subject directoty in state subfolder
subject_dir = os.path.join(state_dir, subject)
result_subject_dir = os.path.join(result_state_dir, subject)
utils.mkdir(result_subject_dir)
# get file paths
t1_path = os.path.join(subject_dir, 't1.mgz')
t2_path = os.path.join(subject_dir, 't2.mgz')
aseg_path = os.path.join(subject_dir, 'aseg.mgz')
preprocess_adni_hippo(t1_path,
t2_path,
aseg_path,
result_subject_dir,
target_res,
padding_margin,
remove=delete_intermediate_files,
path_freesurfer=path_freesurfer,
verbose=verbose,
recompute=recompute)
def validation_on_dilated_lesions(normal_validation_dir,
dilated_validation_dir,
gt_dir,
evaluation_labels,
recompute=True):
utils.mkdir(dilated_validation_dir)
list_validation_subdir = utils.list_subfolders(normal_validation_dir)
for val_idx, validation_subdir in enumerate(list_validation_subdir):
utils.print_loop_info(val_idx, len(list_validation_subdir), 5)
# dilate lesion
dilated_validation_subdir = os.path.join(
dilated_validation_dir, os.path.basename(validation_subdir))
dilate_lesions(validation_subdir,
dilated_validation_subdir,
recompute=recompute)
# compute new dice scores
path_dice = os.path.join(dilated_validation_subdir, 'dice.npy')
if (not os.path.isfile(path_dice)) | recompute:
dice_evaluation(gt_dir, dilated_validation_subdir,
evaluation_labels, path_dice)
prior_stds = '../../data/labels_classes_priors/prior_stds.npy'
########################################################################################################
# instantiate BrainGenerator object
brain_generator = BrainGenerator(labels_dir=path_label_map,
generation_labels=generation_labels,
output_labels=output_labels,
generation_classes=generation_classes,
prior_distributions=prior_distribution,
prior_means=prior_means,
prior_stds=prior_stds,
output_shape=output_shape)
# create result dir
utils.mkdir(result_dir)
for n in range(n_examples):
# generate new image and corresponding labels
start = time.time()
im, lab = brain_generator.generate_brain()
end = time.time()
print('generation {0:d} took {1:.01f}s'.format(n, end - start))
# save output image and label map
utils.save_volume(np.squeeze(im), brain_generator.aff,
brain_generator.header,
os.path.join(result_dir, 't1_%s.nii.gz' % n))
utils.save_volume(np.squeeze(lab), brain_generator.aff,
brain_generator.header,
def prepare_output_files(path_images, out_seg, out_posteriors, out_volumes):
# convert path to absolute paths
path_images = os.path.abspath(path_images)
if out_seg is not None:
out_seg = os.path.abspath(out_seg)
if out_posteriors is not None:
out_posteriors = os.path.abspath(out_posteriors)
if out_volumes is not None:
out_volumes = os.path.abspath(out_volumes)
# prepare input/output volumes
if ('.nii.gz' not in path_images) & ('.nii' not in path_images) & ('.mgz' not in path_images) & \
('.npz' not in path_images):
images_to_segment = utils.list_images_in_folder(path_images)
assert len(images_to_segment) > 0, "Could not find any training data"
if out_seg:
utils.mkdir(out_seg)
out_seg = [os.path.join(out_seg, os.path.basename(image)).replace('.nii', '_seg.nii') for image in
images_to_segment]
out_seg = [seg_path.replace('.mgz', '_seg.mgz') for seg_path in out_seg]
out_seg = [seg_path.replace('.npz', '_seg.npz') for seg_path in out_seg]
else:
out_seg = [out_seg] * len(images_to_segment)
if out_posteriors:
utils.mkdir(out_posteriors)
out_posteriors = [os.path.join(out_posteriors, os.path.basename(image)).replace('.nii',
'_posteriors.nii') for image in images_to_segment]
out_posteriors = [posteriors_path.replace('.mgz', '_posteriors.mgz')
for posteriors_path in out_posteriors]
out_posteriors = [posteriors_path.replace('.npz', '_posteriors.npz')
for posteriors_path in out_posteriors]
else:
out_posteriors = [out_posteriors] * len(images_to_segment)
else:
assert os.path.exists(path_images), "Could not find image to segment"
images_to_segment = [path_images]
if out_seg is not None:
if ('.nii.gz' not in out_seg) & ('.nii' not in out_seg) & ('.mgz' not in out_seg) & ('.npz' not in out_seg):
utils.mkdir(out_seg)
filename = os.path.basename(path_images).replace('.nii', '_seg.nii')
filename = filename.replace('mgz', '_seg.mgz')
filename = filename.replace('.npz', '_seg.npz')
out_seg = os.path.join(out_seg, filename)
else:
utils.mkdir(os.path.dirname(out_seg))
out_seg = [out_seg]
if out_posteriors is not None:
if ('.nii.gz' not in out_posteriors) & ('.nii' not in out_posteriors) & ('.mgz' not in out_posteriors) & \
('.npz' not in out_posteriors):
utils.mkdir(out_posteriors)
filename = os.path.basename(path_images).replace('.nii', '_posteriors.nii')
filename = filename.replace('mgz', '_posteriors.mgz')
filename = filename.replace('.npz', '_posteriors.npz')
out_posteriors = os.path.join(out_posteriors, filename)
else:
utils.mkdir(os.path.dirname(out_posteriors))
out_posteriors = [out_posteriors]
if out_volumes:
if out_volumes[-4:] != '.csv':
print('out_volumes provided without csv extension. Adding csv extension to output_volumes.')
out_volumes += '.csv'
utils.mkdir(os.path.dirname(out_volumes))
return images_to_segment, out_seg, out_posteriors, out_volumes
def preprocess_asegs(aseg_dir,
lesion_gt_dir,
list_incorrect,
list_correct,
lesion_label_in_gt=77,
dilate=2,
recompute=False):
# align asegs to gt dir (cropping to same dimension)
cropped_dir = aseg_dir + '_cropped'
edit_volumes.mri_convert_images_in_dir(aseg_dir,
cropped_dir,
interpolation='nearest',
reference_dir=lesion_gt_dir,
recompute=recompute)
# correct for aseg labels
corrected_dir = cropped_dir + '_corrected'
edit_volumes.correct_labels_in_dir(cropped_dir,
list_incorrect,
list_correct,
corrected_dir,
smooth=False,
recompute=recompute)
# list gt and aseg, and create result dir
list_lesion_labels = utils.list_images_in_folder(lesion_gt_dir)
list_aseg_labels = utils.list_images_in_folder(corrected_dir)
inpainted_dir = corrected_dir + '_lesion_inpainted'
utils.mkdir(inpainted_dir)
# loop over subjects
for path_lesion_label, path_aseg_label in zip(list_lesion_labels,
list_aseg_labels):
path_result = os.path.join(inpainted_dir,
os.path.basename(path_aseg_label))
if (not os.path.isfile(path_result)) | recompute:
# paste lesion label
lesions = utils.load_volume(path_lesion_label)
aseg_label, aff, h = utils.load_volume(path_aseg_label,
im_only=False)
lesion_mask = lesions == lesion_label_in_gt
aseg_label[lesion_mask] = 77
utils.save_volume(aseg_label, aff, h, path_result)
# dilate lesion and ventricle
dilated_dir = inpainted_dir + '_dilated'
utils.mkdir(dilated_dir)
list_inpainted_aseg = utils.list_images_in_folder(inpainted_dir)
for path_aseg in list_inpainted_aseg:
path_result = os.path.join(dilated_dir, os.path.basename(path_aseg))
if (not os.path.isfile(path_result)) | recompute:
# define lesion, WM, and LV masks
aseg, aff, h = utils.load_volume(path_aseg, im_only=False)
WM = aseg == 2
LV = aseg == 4
lesion = aseg == 77
# morphological operations to bridge the gaps between lesions and LV
morph_struct = utils.build_binary_structure(
dilate, len(aseg.shape))
dilated_LV_or_lesion = binary_dilation(LV | lesion, morph_struct)
filled_LV_or_lesion = binary_erosion(dilated_LV_or_lesion,
morph_struct)
LV = LV | (filled_LV_or_lesion & WM)
aseg[LV] = 4
# save map
utils.save_volume(aseg, aff, h, path_result)
def prepare_output_files(path_images, out_seg, out_posteriors, out_resampled,
out_volumes, recompute):
'''
Prepare output files.
'''
# check inputs
assert path_images is not None, 'please specify an input file/folder (--i)'
assert out_seg is not None, 'please specify an output file/folder (--o)'
# convert path to absolute paths
path_images = os.path.abspath(path_images)
basename = os.path.basename(path_images)
out_seg = os.path.abspath(out_seg) if (out_seg is not None) else out_seg
out_posteriors = os.path.abspath(out_posteriors) if (
out_posteriors is not None) else out_posteriors
out_resampled = os.path.abspath(out_resampled) if (
out_resampled is not None) else out_resampled
out_volumes = os.path.abspath(out_volumes) if (
out_volumes is not None) else out_volumes
# path_images is a folder
if ('.nii.gz' not in basename) & ('.nii' not in basename) & (
'.mgz' not in basename) & ('.npz' not in basename):
if os.path.isfile(path_images):
raise Exception(
'Extension not supported for %s, only use: nii.gz, .nii, .mgz, or .npz'
% path_images)
path_images = utils.list_images_in_folder(path_images)
if (out_seg[-7:] == '.nii.gz') | (out_seg[-4:] == '.nii') | (
out_seg[-4:] == '.mgz') | (out_seg[-4:] == '.npz'):
raise Exception(
'Output folders cannot have extensions: .nii.gz, .nii, .mgz, or .npz, had %s'
% out_seg)
utils.mkdir(out_seg)
out_seg = [
os.path.join(out_seg, os.path.basename(image)).replace(
'.nii', '_synthseg.nii') for image in path_images
]
out_seg = [
seg_path.replace('.mgz', '_synthseg.mgz') for seg_path in out_seg
]
out_seg = [
seg_path.replace('.npz', '_synthseg.npz') for seg_path in out_seg
]
recompute_seg = [not os.path.isfile(path_seg) for path_seg in out_seg]
if out_posteriors is not None:
if (out_posteriors[-7:] == '.nii.gz') | (out_posteriors[-4:] == '.nii') | \
(out_posteriors[-4:] == '.mgz') | (out_posteriors[-4:] == '.npz'):
raise Exception('Output folders cannot have extensions: '
'.nii.gz, .nii, .mgz, or .npz, had %s' %
out_posteriors)
utils.mkdir(out_posteriors)
out_posteriors = [
os.path.join(out_posteriors, os.path.basename(image)).replace(
'.nii', '_posteriors.nii') for image in path_images
]
out_posteriors = [
posteriors_path.replace('.mgz', '_posteriors.mgz')
for posteriors_path in out_posteriors
]
out_posteriors = [
posteriors_path.replace('.npz', '_posteriors.npz')
for posteriors_path in out_posteriors
]
recompute_post = [
not os.path.isfile(path_post) for path_post in out_posteriors
]
else:
out_posteriors = [out_posteriors] * len(path_images)
recompute_post = [out_volumes is not None] * len(path_images)
if out_resampled is not None:
if (out_resampled[-7:] == '.nii.gz') | (out_resampled[-4:] == '.nii') | \
(out_resampled[-4:] == '.mgz') | (out_resampled[-4:] == '.npz'):
raise Exception('Output folders cannot have extensions: '
'.nii.gz, .nii, .mgz, or .npz, had %s' %
out_resampled)
utils.mkdir(out_resampled)
out_resampled = [
os.path.join(out_resampled, os.path.basename(image)).replace(
'.nii', '_resampled.nii') for image in path_images
]
out_resampled = [
resampled_path.replace('.mgz', '_resampled.mgz')
for resampled_path in out_resampled
]
out_resampled = [
resampled_path.replace('.npz', '_resampled.npz')
for resampled_path in out_resampled
]
recompute_resampled = [
not os.path.isfile(path_post) for path_post in out_resampled
]
else:
out_resampled = [out_resampled] * len(path_images)
recompute_resampled = [out_volumes is not None] * len(path_images)
# path_images is an image
else:
assert os.path.isfile(path_images), "file does not exist: %s \n" \
"please make sure the path and the extension are correct" % path_images
path_images = [path_images]
if ('.nii.gz' not in out_seg) & ('.nii' not in out_seg) & (
'.mgz' not in out_seg) & ('.npz' not in out_seg):
utils.mkdir(out_seg)
filename = os.path.basename(path_images[0]).replace(
'.nii', '_synthseg.nii')
filename = filename.replace('.mgz', '_synthseg.mgz')
filename = filename.replace('.npz', '_synthseg.npz')
out_seg = os.path.join(out_seg, filename)
else:
utils.mkdir(os.path.dirname(out_seg))
out_seg = [out_seg]
recompute_seg = [not os.path.isfile(out_seg[0])]
if out_posteriors is not None:
if ('.nii.gz' not in out_posteriors) & ('.nii' not in out_posteriors) &\
('.mgz' not in out_posteriors) & ('.npz' not in out_posteriors):
utils.mkdir(out_posteriors)
filename = os.path.basename(path_images[0]).replace(
'.nii', '_posteriors.nii')
filename = filename.replace('.mgz', '_posteriors.mgz')
filename = filename.replace('.npz', '_posteriors.npz')
out_posteriors = os.path.join(out_posteriors, filename)
else:
utils.mkdir(os.path.dirname(out_posteriors))
recompute_post = [not os.path.isfile(out_posteriors)]
else:
recompute_post = [out_volumes is not None]
out_posteriors = [out_posteriors]
if out_resampled is not None:
if ('.nii.gz' not in out_resampled) & ('.nii' not in out_resampled) &\
('.mgz' not in out_resampled) & ('.npz' not in out_resampled):
utils.mkdir(out_resampled)
filename = os.path.basename(path_images[0]).replace(
'.nii', '_resampled.nii')
filename = filename.replace('.mgz', '_resampled.mgz')
filename = filename.replace('.npz', '_resampled.npz')
out_resampled = os.path.join(out_resampled, filename)
else:
utils.mkdir(os.path.dirname(out_resampled))
recompute_resampled = [not os.path.isfile(out_resampled)]
else:
recompute_resampled = [out_volumes is not None]
out_resampled = [out_resampled]
recompute_list = [
recompute | re_seg | re_post | re_res
for (re_seg, re_post,
re_res) in zip(recompute_seg, recompute_post, recompute_resampled)
]
if out_volumes is not None:
if out_volumes[-4:] != '.csv':
print(
'Path for volume outputs provided without csv extension. Adding csv extension.'
)
out_volumes += '.csv'
utils.mkdir(os.path.dirname(out_volumes))
return path_images, out_seg, out_posteriors, out_resampled, out_volumes, recompute_list
def inter_rater_reproducibility_cross_val_exp(manual_seg_dir,
ref_image_dir=None,
recompute=True):
# list subjects
list_subjects = utils.list_subfolders(manual_seg_dir)
# create result directories
if ref_image_dir is not None:
realigned_seg_dir = os.path.join(os.path.dirname(manual_seg_dir),
'registered_to_t1')
list_ref_subjects = utils.list_images_in_folder(ref_image_dir)
else:
realigned_seg_dir = os.path.join(os.path.dirname(manual_seg_dir),
'realigned')
list_ref_subjects = [None] * len(list_subjects)
utils.mkdir(realigned_seg_dir)
path_dice = os.path.join(realigned_seg_dir, 'dice.npy')
# loop over subjects
dice = list()
if (not os.path.isfile(path_dice)) | recompute:
for subject_dir, ref_subject in zip(list_subjects, list_ref_subjects):
# align all images to first image
if ref_subject is not None:
ref_image = ref_subject
else:
ref_image = utils.list_images_in_folder(subject_dir)[0]
result_dir = os.path.join(realigned_seg_dir,
os.path.basename(subject_dir))
edit_volumes.mri_convert_images_in_dir(subject_dir,
result_dir,
interpolation='nearest',
reference_dir=ref_image,
same_reference=True,
recompute=recompute)
# load all volumes and compute distance maps
list_segs = [
utils.load_volume(path)
for path in utils.list_images_in_folder(result_dir)
]
list_distance_maps = [
edit_volumes.compute_distance_map(labels, crop_margin=20)
for labels in list_segs
]
distance_maps = np.stack(list_distance_maps, axis=-1)
n_raters = len(list_segs)
# compare each segmentation to the consensus of all others
tmp_dice = list()
for i, seg in enumerate(list_segs):
tmp_distance_maps = distance_maps[...,
np.arange(n_raters) != i]
tmp_distance_maps = (np.mean(tmp_distance_maps, axis=-1) >
0) * 1
seg = (seg > 0) * 1
tmp_dice.append(2 * np.sum(tmp_distance_maps * seg) /
(np.sum(tmp_distance_maps) + np.sum(seg)))
dice.append(tmp_dice)
np.save(path_dice, np.array(dice))
def prepare_hippo_training_atlases(labels_dir,
result_dir,
image_dir=None,
image_result_dir=None,
smooth=True,
crop_margin=50,
recompute=True):
"""This function prepares training label maps from CobraLab. It first crops each atlas around the right and left
hippocampi, with a margin. It then equalises the shape of these atlases by croppping them to the size of the
smallest hippocampus. Finally it realigns the obtained atlases to FS orientation axes.
:param labels_dir: path of directory with label maps to prepare
:param result_dir: path of directory where prepared atlases will be writen
:param image_dir: (optional) path of directory with images corresponding to the label maps to prepare.
This can be sued to prepare a dataset of real images for supervised training.
:param image_result_dir: (optional) path of directory where images corresponding to prepared atlases will be writen
:param smooth: (optional) whether to smooth the final cropped label maps
:param crop_margin: (optional) margin to add around hippocampi when cropping
:param recompute: (optional) whether to recompute result files even if they already exists"""
# create results dir
utils.mkdir(result_dir)
tmp_result_dir = os.path.join(result_dir, 'first_cropping')
utils.mkdir(tmp_result_dir)
if image_dir is not None:
assert image_result_dir is not None, 'image_result_dir should not be None if image_dir is specified'
utils.mkdir(image_result_dir)
tmp_image_result_dir = os.path.join(image_result_dir, 'first_cropping')
utils.mkdir(tmp_image_result_dir)
else:
tmp_image_result_dir = None
# list labels and images
labels_paths = utils.list_images_in_folder(labels_dir)
if image_dir is not None:
path_images = utils.list_images_in_folder(image_dir)
else:
path_images = [None] * len(labels_paths)
# crop all atlases around hippo
print('\ncropping around hippo')
shape_array = np.zeros((len(labels_paths)*2, 3))
for idx, (path_label, path_image) in enumerate(zip(labels_paths, path_images)):
utils.print_loop_info(idx, len(labels_paths), 1)
# crop left hippo first
path_label_first_crop_l = os.path.join(tmp_result_dir,
os.path.basename(path_label).replace('.nii', '_left.nii'))
lab, aff, h = utils.load_volume(path_label, im_only=False)
lab_l, croppping_idx, aff_l = edit_volumes.crop_volume_around_region(lab, crop_margin,
list(range(20101, 20109)), aff=aff)
if (not os.path.exists(path_label_first_crop_l)) | recompute:
utils.save_volume(lab_l, aff_l, h, path_label_first_crop_l)
else:
lab_l = utils.load_volume(path_label_first_crop_l)
if path_image is not None:
path_image_first_crop_l = os.path.join(tmp_image_result_dir,
os.path.basename(path_image).replace('.nii', '_left.nii'))
if (not os.path.exists(path_image_first_crop_l)) | recompute:
im, aff, h = utils.load_volume(path_image, im_only=False)
im, aff = edit_volumes.crop_volume_with_idx(im, croppping_idx, aff)
utils.save_volume(im, aff, h, path_image_first_crop_l)
shape_array[2*idx, :] = np.array(lab_l.shape)
# crop right hippo and flip them
path_label_first_crop_r = os.path.join(tmp_result_dir,
os.path.basename(path_label).replace('.nii', '_right_flipped.nii'))
lab, aff, h = utils.load_volume(path_label, im_only=False)
lab_r, croppping_idx, aff_r = edit_volumes.crop_volume_around_region(lab, crop_margin,
list(range(20001, 20009)), aff=aff)
if (not os.path.exists(path_label_first_crop_r)) | recompute:
lab_r = edit_volumes.flip_volume(lab_r, direction='rl', aff=aff_r)
utils.save_volume(lab_r, aff_r, h, path_label_first_crop_r)
else:
lab_r = utils.load_volume(path_label_first_crop_r)
if path_image is not None:
path_image_first_crop_r = os.path.join(tmp_image_result_dir,
os.path.basename(path_image).replace('.nii', '_right.nii'))
if (not os.path.exists(path_image_first_crop_r)) | recompute:
im, aff, h = utils.load_volume(path_image, im_only=False)
im, aff = edit_volumes.crop_volume_with_idx(im, croppping_idx, aff)
im = edit_volumes.flip_volume(im, direction='rl', aff=aff)
utils.save_volume(im, aff, h, path_image_first_crop_r)
shape_array[2*idx+1, :] = np.array(lab_r.shape)
# list croppped files
path_labels_first_cropped = utils.list_images_in_folder(tmp_result_dir)
if tmp_image_result_dir is not None:
path_images_first_cropped = utils.list_images_in_folder(tmp_image_result_dir)
else:
path_images_first_cropped = [None] * len(path_labels_first_cropped)
# crop all label maps to same size
print('\nequalising shapes')
new_shape = np.min(shape_array, axis=0).astype('int32')
for i, (path_label, path_image) in enumerate(zip(path_labels_first_cropped, path_images_first_cropped)):
utils.print_loop_info(i, len(path_labels_first_cropped), 1)
# get cropping indices
path_lab_cropped = os.path.join(result_dir, os.path.basename(path_label))
lab, aff, h = utils.load_volume(path_label, im_only=False)
lab_shape = lab.shape
min_cropping = np.array([np.maximum(int((lab_shape[i]-new_shape[i])/2), 0) for i in range(3)])
max_cropping = np.array([min_cropping[i] + new_shape[i] for i in range(3)])
# crop labels and realign on adni format
if (not os.path.exists(path_lab_cropped)) | recompute:
lab, aff = edit_volumes.crop_volume_with_idx(lab, np.concatenate([min_cropping, max_cropping]), aff)
# realign on adni format
lab = np.flip(lab, axis=2)
aff[0:3, 0:3] = np.array([[-0.6, 0, 0], [0, 0, -0.6], [0, -0.6, 0]])
utils.save_volume(lab, aff, h, path_lab_cropped)
# crop image and realign on adni format
if path_image is not None:
path_im_cropped = os.path.join(image_result_dir, os.path.basename(path_image))
if (not os.path.exists(path_im_cropped)) | recompute:
im, aff, h = utils.load_volume(path_image, im_only=False)
im, aff = edit_volumes.crop_volume_with_idx(im, np.concatenate([min_cropping, max_cropping]), aff)
im = np.flip(im, axis=2)
aff[0:3, 0:3] = np.array([[-0.6, 0, 0], [0, 0, -0.6], [0, -0.6, 0]])
im = edit_volumes.mask_volume(im, lab)
utils.save_volume(im, aff, h, path_im_cropped)
# correct all labels to left values
print('\ncorrecting labels')
list_incorrect_labels = [77, 80, 251, 252, 253, 254, 255, 29, 41, 42, 43, 44, 46, 47, 49, 50, 51, 52, 54, 58, 60,
61, 62, 63, 7012, 20001, 20002, 20004, 20005, 20006, 20007, 20008]
list_correct_labels = [2, 3, 2, 2, 2, 2, 2, 2, 2, 3, 4, 5, 7, 8, 10, 11, 12, 13, 18, 26, 28, 2, 30, 31, 20108,
20101, 20102, 20104, 20105, 20106, 20107, 20108]
edit_volumes.correct_labels_in_dir(result_dir, list_incorrect_labels, list_correct_labels, result_dir)
# smooth labels
if smooth:
print('\nsmoothing labels')
edit_volumes.smooth_labels_in_dir(result_dir, result_dir)
generation_labels, n_neutral_labels = utils.get_list_labels(generation_labels,
FS_sort=True)
########################################################################################################
# instantiate BrainGenerator object
brain_generator = BrainGenerator(labels_dir=path_label_maps,
generation_labels=generation_labels,
output_labels=segmentation_labels,
n_neutral_labels=n_neutral_labels,
output_shape=output_shape,
output_div_by_n=output_divisible_by_n,
prior_distributions=prior_distributions,
flipping=flipping)
utils.mkdir(result_folder)
for n in range(n_examples):
# generate new image and corresponding labels
start = time.time()
im, lab = brain_generator.generate_brain()
end = time.time()
print('deformation {0:d} took {1:.01f}s'.format(n, end - start))
# save image
utils.save_volume(
np.squeeze(im), brain_generator.aff, brain_generator.header,
os.path.join(result_folder, 'SynthSeg_image_%s.nii.gz' % n))
utils.save_volume(
np.squeeze(lab), brain_generator.aff, brain_generator.header,
def training(image_dir,
labels_dir,
model_dir,
path_label_list=None,
save_label_list=None,
n_neutral_labels=1,
batchsize=1,
target_res=None,
output_shape=None,
flipping=True,
flip_rl_only=False,
scaling_bounds=0.15,
rotation_bounds=15,
enable_90_rotations=False,
shearing_bounds=.012,
translation_bounds=False,
nonlin_std=3.,
nonlin_shape_factor=.04,
bias_field_std=.3,
bias_shape_factor=.025,
same_bias_for_all_channels=False,
augment_intensitites=True,
noise_std=1.,
augment_channels_separately=True,
n_levels=5,
nb_conv_per_level=2,
conv_size=3,
unet_feat_count=24,
feat_multiplier=1,
dropout=0,
activation='elu',
lr=1e-4,
lr_decay=0,
wl2_epochs=5,
dice_epochs=200,
steps_per_epoch=1000,
checkpoint=None):
"""
This function trains a neural network with aggressively augmented images. The model is implemented on the GPU and
contains three sub-model: one for augmentation, one neural network (UNet), and one for computing the loss function.
The network is pre-trained with a weighted sum of square error, in order to bring the weights in a favorable
optimisation landscape. The training then continues with a soft dice loss function.
:param image_dir: path of folder with all input images, or to a single image (if only one training example)
:param labels_dir: path of folder with all input label maps, or to a single label map (if only one training example)
labels maps and images are likend by sorting order.
:param model_dir: path of a directory where the models will be saved during training.
#---------------------------------------------- Generation parameters ----------------------------------------------
# output-related parameters
:param path_label_list: (optional) path to a numpy array containing all the label values to be segmented.
By default, this is computed by taking all the label values in the training label maps.
:param save_label_list: (optional) path where to write the computed list of segmentation labels.
:param n_neutral_labels: (optional) number of non-sided labels in label_list. This is used for determining which
label values to swap when right/left flipping the training examples. Default is 1 (to account for the background).
:param batchsize: (optional) number of images per mini-batch. Default is 1.
:param target_res: (optional) target resolution at which to produce the segmentation label maps. The training data
will be resampled to this resolution before being run through the network. If None, no resampling is performed.
Can be a number (isotropic resolution), or the path to a 1d numpy array.
:param output_shape: (optional) desired shape of the output image, obtained by randomly cropping the generated image
Can be an integer (same size in all dimensions), a sequence, a 1d numpy array, or the path to a 1d numpy array.
# Augmentation parameters
:param flipping: (optional) whether to introduce random flipping. Default is True.
:param flip_rl_only: (optional) if flipping is True, whether to flip only in the right/left axis. Default is False.
:param scaling_bounds: (optional) if apply_linear_trans is True, the scaling factor for each dimension is
sampled from a uniform distribution of predefined bounds. scaling_bounds can either be:
1) a number, in which case the scaling factor is independently sampled from the uniform distribution of bounds
(1-scaling_bounds, 1+scaling_bounds) for each dimension.
2) the path to a numpy array of shape (2, n_dims), in which case the scaling factor is sampled from the uniform
distribution of bounds (scaling_bounds[0, i], scaling_bounds[1, i]) for the i-th dimension.
3) False, in which case scaling is completely turned off.
Default is scaling_bounds = 0.15
:param rotation_bounds: (optional) same as scaling bounds but for the rotation angle, except that for case 1 the
bounds are centred on 0 rather than 1, i.e. (0+rotation_bounds[i], 0-rotation_bounds[i]).
Default is rotation_bounds = 15.
:param enable_90_rotations: (optional) wheter to rotate the input by a random angle chosen in {0, 90, 180, 270}.
This is done regardless of the value of rotation_bounds. If true, a different value is sampled for each dimension.
:param shearing_bounds: (optional) same as scaling bounds. Default is shearing_bounds = 0.012.
:param translation_bounds: (optional) same as scaling bounds. Default is translation_bounds = False, but we
encourage using it when cropping is deactivated (i.e. when output_shape=None in BrainGenerator).
:param nonlin_std: (optional) maximum value for the standard deviation of the normal distribution from which we
sample the first tensor for synthesising the deformation field. Set to 0 to completely turn it off.
:param nonlin_shape_factor: (optional) ratio between the size of the input label maps and the size of the sampled
tensor for synthesising the elastic deformation field.
:param bias_field_std: (optional) If strictly positive, this triggers the corruption of images with a bias field.
It is obtained by sampling a first small tensor from a normal distribution, resizing it to full size, and rescaling
it to positive values by taking the voxel-wise exponential. bias_field_std designates the std dev of the normal
distribution from which we sample the first tensor. Set to 0 to deactivate biad field.
:param bias_shape_factor: (optional) If bias_field_std is not False, this designates the ratio between the size
of the input label maps and the size of the first sampled tensor for synthesising the bias field.
:param same_bias_for_all_channels: (optional) If bias_field_std is not False, whether to apply the same bias field
to all channels or not.
:param augment_intensitites: (optional) whether to augment the intensities of the images with gamma augmentation.
:param noise_std: (optional) if augment_intensities is true, maximum value for the standard deviation of the normal
distribution from which we sample a Gaussian white noise. Set to False to deactivate white noise augmentation.
Default value is 1.
:param augment_channels_separately: (optional) whether to augment the intensities of each channel indenpendently.
Only applied if augment_intensity is True, and the training images have several channels. Default is True.
# ------------------------------------------ UNet architecture parameters ------------------------------------------
:param n_levels: (optional) number of level for the Unet. Default is 5.
:param nb_conv_per_level: (optional) number of convolutional layers per level. Default is 2.
:param conv_size: (optional) size of the convolution kernels. Default is 2.
:param unet_feat_count: (optional) number of feature for the first layr of the Unet. Default is 24.
:param feat_multiplier: (optional) multiply the number of feature by this nummber at each new level. Default is 1.
:param dropout: (optional) probability of dropout for the Unet. Deafult is 0, where no dropout is applied.
:param activation: (optional) activation function. Can be 'elu', 'relu'.
# ----------------------------------------------- Training parameters ----------------------------------------------
:param lr: (optional) learning rate for the training. Default is 1e-4
:param lr_decay: (optional) learing rate decay. Default is 0, where no decay is applied.
:param wl2_epochs: (optional) number of epohs for which the network (except the soft-max layer) is trained with L2
norm loss function. Default is 5.
:param dice_epochs: (optional) number of epochs with the soft Dice loss function. default is 100.
:param steps_per_epoch: (optional) number of steps per epoch. Default is 1000. Since no online validation is
possible, this is equivalent to the frequency at which the models are saved.
:param checkpoint: (optional) path of an already saved model to load before starting the training.
"""
# check epochs
assert (wl2_epochs > 0) | (dice_epochs > 0), \
'either wl2_epochs or dice_epochs must be positive, had {0} and {1}'.format(wl2_epochs, dice_epochs)
# prepare data files
path_images = utils.list_images_in_folder(image_dir)
path_label_maps = utils.list_images_in_folder(labels_dir)
assert len(path_images) == len(path_label_maps), 'not the same number of training images and label maps.'
# read info from image and get label list
im_shape, _, _, n_channels, _, image_res = utils.get_volume_info(path_images[0], aff_ref=np.eye(4))
label_list, _ = utils.get_list_labels(path_label_list, labels_dir=labels_dir, save_label_list=save_label_list)
n_labels = np.size(label_list)
# prepare model folder
utils.mkdir(model_dir)
# transformation model
augmentation_model = build_augmentation_model(im_shape=im_shape,
n_channels=n_channels,
label_list=label_list,
n_neutral_labels=n_neutral_labels,
image_res=image_res,
target_res=target_res,
output_shape=output_shape,
output_div_by_n=2 ** n_levels,
flipping=flipping,
flip_rl_only=flip_rl_only,
aff=np.eye(4),
scaling_bounds=scaling_bounds,
rotation_bounds=rotation_bounds,
enable_90_rotations=enable_90_rotations,
shearing_bounds=shearing_bounds,
translation_bounds=translation_bounds,
nonlin_std=nonlin_std,
nonlin_shape_factor=nonlin_shape_factor,
bias_field_std=bias_field_std,
bias_shape_factor=bias_shape_factor,
same_bias_for_all_channels=same_bias_for_all_channels,
apply_intensity_augmentation=augment_intensitites,
noise_std=noise_std,
augment_channels_separately=augment_channels_separately)
unet_input_shape = augmentation_model.output[0].get_shape().as_list()[1:]
# prepare the segmentation model
unet_model = nrn_models.unet(nb_features=unet_feat_count,
input_shape=unet_input_shape,
nb_levels=n_levels,
conv_size=conv_size,
nb_labels=n_labels,
feat_mult=feat_multiplier,
nb_conv_per_level=nb_conv_per_level,
conv_dropout=dropout,
batch_norm=-1,
activation=activation,
input_model=augmentation_model)
# input generator
train_example_gen = image_seg_generator(path_images=path_images,
path_labels=path_label_maps,
batchsize=batchsize,
n_channels=n_channels)
input_generator = utils.build_training_generator(train_example_gen, batchsize)
# pre-training with weighted L2, input is fit to the softmax rather than the probabilities
if wl2_epochs > 0:
wl2_model = models.Model(unet_model.inputs, [unet_model.get_layer('unet_likelihood').output])
wl2_model = metrics.metrics_model(input_model=wl2_model, metrics='wl2')
train_model(wl2_model, input_generator, lr, lr_decay, wl2_epochs, steps_per_epoch, model_dir, 'wl2', checkpoint)
checkpoint = os.path.join(model_dir, 'wl2_%03d.h5' % wl2_epochs)
# fine-tuning with dice metric
dice_model = metrics.metrics_model(input_model=unet_model, metrics='dice')
train_model(dice_model, input_generator, lr, lr_decay, dice_epochs, steps_per_epoch, model_dir, 'dice', checkpoint)
def preprocess_adni_hippo(path_t1,
path_t2,
path_aseg,
result_dir,
target_res,
padding_margin=85,
remove=False,
path_freesurfer='/usr/local/freesurfer/',
verbose=True,
recompute=True):
"""This function builds a T1+T2 multimodal image from the ADNI dataset.
It first rescales intensities of each channel between 0 and 255.
It then resamples the T2 image (which are 0.4*0.4*2.0 resolution) to target resolution.
The obtained T2 is then padded in all directions by the padding_margin param (typically large 85).
The T1 and aseg are then resampled like the T2 using mri_convert.
Now that the T1, T2 and asegs are aligned and at the same resolution, we crop them around the right and left hippo.
Finally, the T1 and T2 are concatenated into one single multimodal image.
:param path_t1: path input T1 (typically at 1mm isotropic)
:param path_t2: path input T2 (typically cropped around the hippo in sagittal axis, 0.4x0.4x2.0)
:param path_aseg: path input segmentation (typically at 1mm isotropic)
:param result_dir: path of directory where prepared images and labels will be writen.
:param target_res: resolution at which to resample the label maps, and the images.
Can be a number (isotropic resolution), a sequence, or a 1d numpy array.
:param padding_margin: (optional) margin to add around hippocampi when cropping
:param remove: (optional) whether to delete temporary files. Default is True.
:param path_freesurfer: (optional) path of FreeSurfer home, to use mri_convert
:param verbose: (optional) whether to print out mri_convert output when resampling images
:param recompute: (optional) whether to recompute result files even if they already exists
"""
# create results dir
utils.mkdir(result_dir)
path_test_im_right = os.path.join(result_dir, 'hippo_right.nii.gz')
path_test_aseg_right = os.path.join(result_dir, 'hippo_right_aseg.nii.gz')
path_test_im_left = os.path.join(result_dir, 'hippo_left.nii.gz')
path_test_aseg_left = os.path.join(result_dir, 'hippo_left_aseg.nii.gz')
if (not os.path.isfile(path_test_im_right)) | (not os.path.isfile(path_test_aseg_right)) | \
(not os.path.isfile(path_test_im_left)) | (not os.path.isfile(path_test_aseg_left)) | recompute:
# set up FreeSurfer
os.environ['FREESURFER_HOME'] = path_freesurfer
os.system(os.path.join(path_freesurfer, 'SetUpFreeSurfer.sh'))
mri_convert = os.path.join(path_freesurfer, 'bin/mri_convert.bin')
# rescale T1
path_t1_rescaled = os.path.join(result_dir, 't1_rescaled.nii.gz')
if (not os.path.isfile(path_t1_rescaled)) | recompute:
im, aff, h = utils.load_volume(path_t1, im_only=False)
im = edit_volumes.rescale_volume(im)
utils.save_volume(im, aff, h, path_t1_rescaled)
# rescale T2
path_t2_rescaled = os.path.join(result_dir, 't2_rescaled.nii.gz')
if (not os.path.isfile(path_t2_rescaled)) | recompute:
im, aff, h = utils.load_volume(path_t2, im_only=False)
im = edit_volumes.rescale_volume(im)
utils.save_volume(im, aff, h, path_t2_rescaled)
# resample T2 to target res
path_t2_resampled = os.path.join(result_dir, 't2_rescaled_resampled.nii.gz')
if (not os.path.isfile(path_t2_resampled)) | recompute:
str_res = ' '.join([str(r) for r in utils.reformat_to_list(target_res, length=3)])
cmd = mri_convert + ' ' + path_t2_rescaled + ' ' + path_t2_resampled + ' --voxsize ' + str_res
cmd += ' -odt float'
if not verbose:
cmd += ' >/dev/null 2>&1'
_ = os.system(cmd)
# pad T2
path_t2_padded = os.path.join(result_dir, 't2_rescaled_resampled_padded.nii.gz')
if (not os.path.isfile(path_t2_padded)) | recompute:
t2, aff, h = utils.load_volume(path_t2_resampled, im_only=False)
t2_padded = np.pad(t2, padding_margin, 'constant')
aff[:3, -1] = aff[:3, -1] - (aff[:3, :3] @ (padding_margin * np.ones((3, 1)))).T
utils.save_volume(t2_padded, aff, h, path_t2_padded)
# resample T1 and aseg accordingly
path_t1_resampled = os.path.join(result_dir, 't1_rescaled_resampled.nii.gz')
if (not os.path.isfile(path_t1_resampled)) | recompute:
cmd = mri_convert + ' ' + path_t1_rescaled + ' ' + path_t1_resampled + ' -rl ' + path_t2_padded
cmd += ' -odt float'
if not verbose:
cmd += ' >/dev/null 2>&1'
_ = os.system(cmd)
path_aseg_resampled = os.path.join(result_dir, 'aseg_resampled.nii.gz')
if (not os.path.isfile(path_aseg_resampled)) | recompute:
cmd = mri_convert + ' ' + path_aseg + ' ' + path_aseg_resampled + ' -rl ' + path_t2_padded
cmd += ' -rt nearest -odt float'
if not verbose:
cmd += ' >/dev/null 2>&1'
_ = os.system(cmd)
# crop images and concatenate T1 and T2
for lab, side in zip([17, 53], ['left', 'right']):
path_test_image = os.path.join(result_dir, 'hippo_{}.nii.gz'.format(side))
path_test_aseg = os.path.join(result_dir, 'hippo_{}_aseg.nii.gz'.format(side))
if (not os.path.isfile(path_test_image)) | (not os.path.isfile(path_test_aseg)) | recompute:
aseg, aff, h = utils.load_volume(path_aseg_resampled, im_only=False)
tmp_aseg, cropping, tmp_aff = edit_volumes.crop_volume_around_region(aseg,
margin=30,
masking_labels=lab,
aff=aff)
if side == 'right':
tmp_aseg = edit_volumes.flip_volume(tmp_aseg, direction='rl', aff=tmp_aff)
utils.save_volume(tmp_aseg, tmp_aff, h, path_test_aseg)
if (not os.path.isfile(path_test_image)) | recompute:
t1 = utils.load_volume(path_t1_resampled)
t1 = edit_volumes.crop_volume_with_idx(t1, crop_idx=cropping)
t1 = edit_volumes.mask_volume(t1, tmp_aseg, dilate=6, erode=5)
t2 = utils.load_volume(path_t2_padded)
t2 = edit_volumes.crop_volume_with_idx(t2, crop_idx=cropping)
t2 = edit_volumes.mask_volume(t2, tmp_aseg, dilate=6, erode=5)
if side == 'right':
t1 = edit_volumes.flip_volume(t1, direction='rl', aff=tmp_aff)
t2 = edit_volumes.flip_volume(t2, direction='rl', aff=tmp_aff)
test_image = np.stack([t1, t2], axis=-1)
utils.save_volume(test_image, tmp_aff, h, path_test_image)
# remove unnecessary files
if remove:
list_files_to_remove = [path_t1_rescaled,
path_t2_rescaled,
path_t2_resampled,
path_t2_padded,
path_t1_resampled,
path_aseg_resampled]
for path in list_files_to_remove:
os.remove(path)
def validate_training(image_dir,
gt_dir,
models_dir,
validation_main_dir,
segmentation_labels,
n_neutral_labels=None,
evaluation_labels=None,
step_eval=1,
padding=None,
cropping=None,
target_res=1.,
gradients=False,
flip=False,
topology_classes=None,
sigma_smoothing=0,
keep_biggest_component=False,
conv_size=3,
n_levels=5,
nb_conv_per_level=2,
unet_feat_count=24,
feat_multiplier=2,
activation='elu',
mask_dir=None,
compute_distances=False,
recompute=True):
"""This function validates models saved at different epochs of the same training.
All models are assumed to be in the same folder.contained in models_dir.
The results of each model are saved in a subfolder in validation_main_dir.
:param image_dir: path of the folder with validation images.
:param gt_dir: path of the folder with ground truth label maps.
These are matched to the validation images by sorting order.
:param models_dir: path of the folder with the models to validate.
:param validation_main_dir: path of the folder where all the models validation subfolders will be saved.
:param segmentation_labels: path of the numpy array containing all the segmentation labels used during training.
:param n_neutral_labels: (optional) value of n_neutral_labels used during training. Used only if flip is True.
:param evaluation_labels: (optional) label values to validate on. Must be a subset of the segmentation labels.
Can be a sequence, a 1d numpy array, or the path to a numpy 1d array. Default is the same as segmentation_label_list
:param step_eval: (optional) If step_eval > 1 skips models when validating, by validating on models step_eval apart.
:param padding: (optional) pad the images to the specified shape before predicting the segmentation maps.
Can be an int, a sequence or a 1d numpy array.
:param cropping: (optional) whether to crop the input to smaller size while being run through the network.
The result is then given in the original image space. Can be an int, a sequence, or a 1d numpy array.
:param target_res: (optional) target resolution at which the network operates (and thus resolution of the output
segmentations). This must match the resolution of the training data ! target_res is used to automatically resampled
the images with resolutions outside [target_res-0.05, target_res+0.05].
Can be a sequence, a 1d numpy array. Set to None to disable the automatic resampling. Default is 1mm.
:param flip: (optional) whether to perform test-time augmentation, where the input image is segmented along with
a right/left flipped version on it. If set to True (default), be careful because this requires more memory.
:param topology_classes: List of classes corresponding to all segmentation labels, in order to group them into
classes, for each of which we will operate a smooth version of biggest connected component.
Can be a sequence, a 1d numpy array, or the path to a numpy 1d array in the same order as segmentation_label_list.
Default is None, where no topological analysis is performed.
:param sigma_smoothing: (optional) If not None, the posteriors are smoothed with a gaussian kernel of the specified
standard deviation.
:param keep_biggest_component: (optional) whether to only keep the biggest component in the predicted segmentation.
This is applied independently of topology_classes, and it is applied to the whole segmentation
:param conv_size: (optional) size of the convolution kernels. Default is 2.
:param n_levels: (optional) number of level for the Unet. Default is 5.
:param nb_conv_per_level: (optional) number of convolutional layers per level. Default is 2.
:param unet_feat_count: (optional) number of feature maps for the first level. Default is 24.
:param feat_multiplier: (optional) multiply the number of feature by this nummber at each new level. Default is 1.
:param activation: (optional) activation function. Can be 'elu', 'relu'.
:param mask_dir: (optional) path of masks that will be used to mask out some parts of the obtained segmentations
during the evaluation. Default is None, where nothing is masked.
:param compute_distances: (optional) whether to add Hausdorff and mean surface distance evaluations to the default
Dice evaluation. Default is True.
:param recompute: (optional) whether to recompute result files even if they already exists."""
# create result folder
utils.mkdir(validation_main_dir)
# loop over models
list_models = utils.list_files(models_dir,
expr=['dice', '.h5'],
cond_type='and')[::step_eval]
# list_models = [p for p in list_models if int(os.path.basename(p)[-6:-3]) % 10 == 0]
loop_info = utils.LoopInfo(len(list_models), 1, 'validating', True)
for model_idx, path_model in enumerate(list_models):
# build names and create folders
model_val_dir = os.path.join(
validation_main_dir,
os.path.basename(path_model).replace('.h5', ''))
dice_path = os.path.join(model_val_dir, 'dice.npy')
utils.mkdir(model_val_dir)
if (not os.path.isfile(dice_path)) | recompute:
loop_info.update(model_idx)
predict(path_images=image_dir,
path_model=path_model,
segmentation_labels=segmentation_labels,
n_neutral_labels=n_neutral_labels,
path_segmentations=model_val_dir,
padding=padding,
cropping=cropping,
target_res=target_res,
gradients=gradients,
flip=flip,
topology_classes=topology_classes,
sigma_smoothing=sigma_smoothing,
keep_biggest_component=keep_biggest_component,
conv_size=conv_size,
n_levels=n_levels,
nb_conv_per_level=nb_conv_per_level,
unet_feat_count=unet_feat_count,
feat_multiplier=feat_multiplier,
activation=activation,
gt_folder=gt_dir,
mask_folder=mask_dir,
evaluation_labels=evaluation_labels,
compute_distances=compute_distances,
recompute=recompute,
verbose=False)
def prepare_anisotropic_dataset(image_dir,
list_synth_res,
downsample_image_result_dir,
resample_image_result_dir,
labels_dir=None,
downsample_labels_result_dir=None,
native_resolution=1,
slice_thickness=4,
recompute=True):
"""
This function takes as input a set of isotropic HR images (e.g. 1mm) and a list of resolutions.
The HR images will be downsampled to each resolution but only in one of the three direction at a time (e.g. 1*1*r,
then 1*r*1, then r*1*1). All downsampled images are then resampled back to native HR.
Additionally, this function can also downsample label maps corresponding to the input images.
:param image_dir: path of directory with input images
:param list_synth_res: list of low resolutions (e.g. [3, 6, 9]).
:param downsample_image_result_dir: path of directory where all versions of downsampled images will be writen.
:param resample_image_result_dir: path of directory where all versions of resampled images will be writen.
:param native_resolution: native HR (useful when all HR images are not exactly the same resolution)
:param slice_thickness: slice thickness (same for all resolution)
:param labels_dir: path of directory with label maps corresponding to input images
:param downsample_labels_result_dir: path of directory where all versions of downsampled labels will be writen.
:param recompute:
"""
# create results dir
utils.mkdir(resample_image_result_dir)
utils.mkdir(downsample_image_result_dir)
if downsample_labels_result_dir is not None:
utils.mkdir(downsample_labels_result_dir)
# define thickness, which is always the same
list_thickness = slice_thickness * np.eye(3)
list_thickness[list_thickness == 0] = native_resolution
list_thickness = list_thickness.tolist()
# loop over resolution levels
for synth_res in list_synth_res:
# define blurring res
list_data_res = synth_res * np.eye(3)
list_data_res[list_data_res == 0] = native_resolution
list_data_res = list_data_res.tolist()
# loop over resolution directions
for (data_res, thickness) in zip(list_data_res, list_thickness):
res_str = '_'.join(['%d' % r for r in data_res])
print('\npreprocess images to ' + res_str.replace('_', '*') + 'mm resolution')
# build path result folders
im_results_dir = os.path.join(resample_image_result_dir, 'images_' + res_str)
im_downsample_results_dir = os.path.join(downsample_image_result_dir, 'images_' + res_str)
if downsample_labels_result_dir is not None:
labels_results_dir = os.path.join(downsample_labels_result_dir, 'labels_' + res_str)
else:
labels_results_dir = None
# downsample datasets
edit_volumes.simulate_upsampled_anisotropic_images(image_dir,
im_downsample_results_dir,
im_results_dir,
data_res,
labels_dir=labels_dir,
downsample_labels_result_dir=labels_results_dir,
slice_thickness=thickness,
recompute=recompute)
def postprocess_samseg(list_samseg_dir,
list_gt_dir,
path_segmentation_labels,
incorrect_labels,
correct_labels,
list_posteriors_dir=None,
list_thresholds=None,
recompute=False):
""" This function processes the samseg segmentations: it corrects the labels (right/left and 99 to 77), resamples
them to the space of gt_dir, and computes the Dice scores for 1) all_subjects vs. testing subjects only, and 2) all
ROIs vs. lesions only.
It requires that all segmentations are sorted in three subfolders inside samseg_main_dir: t1, flair, and t1_flair.
IMPORTANT: Images are expected to have to following naming convention: <subject_id>.samseg.<contrast>.lesion.mgz,
where <contrast> must either be t1, flair, ***t1_flair***
:param list_samseg_dir: main samseg dir containing the three subfolders t1, flair, t1_flair
:param list_gt_dir: folder with the gt label maps for all subjects
:param path_segmentation_labels: list of segmentation labels
:param incorrect_labels: list of samseg incorrect labels
:param correct_labels: list of labels to correct the wrong one with
:param recompute: whether to recompute files
"""
if list_posteriors_dir is None:
list_posteriors_dir = [None] * len(list_samseg_dir)
for samseg_dir, gt_dir, posteriors_dir, threshold in zip(
list_samseg_dir, list_gt_dir, list_posteriors_dir,
list_thresholds):
# define result directories
samseg_corrected_dir = samseg_dir + '_corrected'
samseg_preprocessed_dir = samseg_dir + '_preprocessed'
if (not os.path.isdir(samseg_preprocessed_dir)) | recompute:
# regroup right/left labels and change 99 to 77
edit_volumes.correct_labels_in_dir(samseg_dir,
incorrect_labels,
correct_labels,
samseg_corrected_dir,
recompute=recompute)
# resample to gt format
edit_volumes.mri_convert_images_in_dir(samseg_corrected_dir,
samseg_preprocessed_dir,
interpolation='nearest',
reference_dir=gt_dir,
recompute=recompute)
# replace lesions by thresholded lesion posteriors
if posteriors_dir is not None:
# resample posteriors to gt format
posteriors_preprocessed_dir = posteriors_dir + '_preprocessed'
edit_volumes.mri_convert_images_in_dir(posteriors_dir,
posteriors_preprocessed_dir,
reference_dir=gt_dir,
recompute=recompute)
# list hard segmentations and posteriors
samseg_postprocessed_dir = samseg_dir + '_postprocessed'
utils.mkdir(samseg_postprocessed_dir)
path_segs = [
path for path in utils.list_images_in_folder(
samseg_preprocessed_dir)
]
path_posteriors = [
path for path in utils.list_images_in_folder(
posteriors_preprocessed_dir)
]
for subject_idx, (path_seg, path_post) in enumerate(
zip(path_segs, path_posteriors)):
path_result = os.path.join(samseg_postprocessed_dir,
os.path.basename(path_seg))
if (not os.path.isfile(path_result)) | recompute:
# replace segmented lesions by thresholded posteriors
seg, aff, h = utils.load_volume(path_seg, im_only=False)
posteriors = utils.load_volume(path_post)
seg[seg == 77] = 2
seg[posteriors > threshold] = 77
utils.save_volume(seg, aff, h, path_result)
else:
samseg_postprocessed_dir = samseg_preprocessed_dir
# compute dice scores with
path_dice_testing = os.path.join(samseg_postprocessed_dir, 'dice.npy')
path_dice_lesions_testing = os.path.join(samseg_postprocessed_dir,
'dice_lesions.npy')
if (not os.path.isfile(path_dice_testing)) | recompute:
dice_evaluation(gt_dir, samseg_postprocessed_dir,
path_segmentation_labels, path_dice_testing)
if (not os.path.isfile(path_dice_lesions_testing)) | recompute:
dice = np.load(path_dice_testing)
np.save(path_dice_lesions_testing, dice[4, :])
def build_intensity_stats(list_image_dir,
list_labels_dir,
result_dir,
estimation_labels,
estimation_classes=None,
max_channel=3,
rescale=True):
"""This function aims at estimating the intensity distributions of K different structure types from a set of images.
The distribution of each structure type is modelled as a Gaussian, parametrised by a mean and a standard deviation.
Because the intensity distribution of structures can vary accross images, we additionally use Gausian priors for the
parameters of each Gaussian distribution. Therefore, the intensity distribution of each structure type is described
by 4 parameters: a mean/std for the mean intensity, and a mean/std for the std deviation.
This function uses a set of images along with corresponding segmentations to estimate the 4*K parameters.
Additionally, it can estimate the 4*K parameters for several image datasets, that we call here n_datasets.
This function writes 2 numpy arrays of size (2*n_datasets, K), one with the evaluated means/std for the mean
intensities, and one for the mean/std for the standard deviations.
In these arrays, each block of two rows refer to a different dataset.
Within each block of two rows, the first row represents the mean, and the second represents the std.
:param list_image_dir: path of folders with images for intensity distribution estimation.
Can be the path of single directory (n_datasets=1), or a list of folders, each being a separate dataset.
Images can be multimodal, in which case each modality is treated as a different dataset, i.e. each modality will
have a separate block (of size (2, K)) in the result arrays.
:param list_labels_dir: path of folders with label maps corresponding to input images.
If list_image_dir is a list of several folders, list_labels_dir can either be a list of folders (one for each image
folder), or the path to a single folder, which will be used for all datasets.
If a dataset has multi-modal images, the same label map is applied to all modalities.
:param result_dir: path of directory where estimated priors will be writen.
:param estimation_labels: labels to estimate intensity statistics from.
Can be a sequence, a 1d numpy array, or the path to a 1d numpy array.
:param estimation_classes: (optional) enables to regroup structures into classes of similar intensity statistics.
Intenstites associated to regrouped labels will thus contribute to the same Gaussian during statistics estimation.
Can be a sequence, a 1d numpy array, or the path to a 1d numpy array.
It should have the same length as labels_list, and contain values between 0 and K-1, where K is the total number of
classes. Default is all labels have different classes (K=len(estimation_labels)).
:param max_channel: (optional) maximum number of channels to consider if the data is multispectral. Default is 3.
:param rescale: (optional) whether to rescale images between 0 and 255 before intensity estimation
"""
# handle results directories
utils.mkdir(result_dir)
# reformat image/labels dir into lists
list_image_dir = utils.reformat_to_list(list_image_dir)
list_labels_dir = utils.reformat_to_list(list_labels_dir, length=len(list_image_dir))
# reformat list estimation labels and classes
estimation_labels = np.array(utils.reformat_to_list(estimation_labels, load_as_numpy=True, dtype='int'))
if estimation_classes is not None:
estimation_classes = np.array(utils.reformat_to_list(estimation_classes, load_as_numpy=True, dtype='int'))
else:
estimation_classes = np.arange(estimation_labels.shape[0])
assert len(estimation_classes) == len(estimation_labels), 'estimation labels and classes should be of same length'
# get unique classes
unique_estimation_classes, unique_indices = np.unique(estimation_classes, return_index=True)
n_classes = len(unique_estimation_classes)
if not np.array_equal(unique_estimation_classes, np.arange(n_classes)):
raise ValueError('estimation_classes should only contain values between 0 and N-1, '
'where K is the total number of classes. Here N = %d' % n_classes)
# loop over dataset
list_datasets_prior_means = list()
list_datasets_prior_stds = list()
for image_dir, labels_dir in zip(list_image_dir, list_labels_dir):
# get prior stats for dataset
tmp_prior_means, tmp_prior_stds = sample_intensity_stats_from_single_dataset(image_dir,
labels_dir,
estimation_labels,
estimation_classes,
max_channel=max_channel,
rescale=rescale)
# add stats arrays to list of datasets-wise statistics
list_datasets_prior_means.append(tmp_prior_means)
list_datasets_prior_stds.append(tmp_prior_stds)
# stack all modalities together
prior_means = np.concatenate(list_datasets_prior_means, axis=0)
prior_stds = np.concatenate(list_datasets_prior_stds, axis=0)
# save files
np.save(os.path.join(result_dir, 'prior_means.npy'), prior_means)
np.save(os.path.join(result_dir, 'prior_stds.npy'), prior_stds)
return prior_means, prior_stds
def prepare_output_files(path_images, out_seg, out_posteriors, out_volumes, recompute):
assert out_seg or out_posteriors, "output segmentation (or posteriors) is required"
# convert path to absolute paths
path_images = os.path.abspath(path_images)
basename = os.path.basename(path_images)
out_seg = os.path.abspath(out_seg) if (out_seg is not None) else out_seg
out_posteriors = os.path.abspath(out_posteriors) if (out_posteriors is not None) else out_posteriors
out_volumes = os.path.abspath(out_volumes) if (out_volumes is not None) else out_volumes
# prepare input/output volumes
if ('.nii.gz' not in basename) & ('.nii' not in basename) & ('.mgz' not in basename) & ('.npz' not in basename):
if os.path.isfile(path_images):
raise Exception('extension not supported for %s, only use: nii.gz, .nii, .mgz, or .npz' % path_images)
images_to_segment = utils.list_images_in_folder(path_images)
if out_seg is not None:
utils.mkdir(out_seg)
out_seg = [os.path.join(out_seg, os.path.basename(image)).replace('.nii', '_seg.nii') for image in
images_to_segment]
out_seg = [seg_path.replace('.mgz', '_seg.mgz') for seg_path in out_seg]
out_seg = [seg_path.replace('.npz', '_seg.npz') for seg_path in out_seg]
recompute_seg = [not os.path.isfile(path_seg) for path_seg in out_seg]
else:
out_seg = [out_seg] * len(images_to_segment)
recompute_seg = [False] * len(images_to_segment)
if out_posteriors is not None:
utils.mkdir(out_posteriors)
out_posteriors = [os.path.join(out_posteriors, os.path.basename(image)).replace('.nii',
'_posteriors.nii') for image in images_to_segment]
out_posteriors = [posteriors_path.replace('.mgz', '_posteriors.mgz') for posteriors_path in out_posteriors]
out_posteriors = [posteriors_path.replace('.npz', '_posteriors.npz') for posteriors_path in out_posteriors]
recompute_post = [not os.path.isfile(path_post) for path_post in out_posteriors]
else:
out_posteriors = [out_posteriors] * len(images_to_segment)
recompute_post = [out_volumes is not None] * len(images_to_segment)
else:
assert os.path.isfile(path_images), "files does not exist: %s " \
"\nplease make sure the path and the extension are correct" % path_images
images_to_segment = [path_images]
if out_seg is not None:
if ('.nii.gz' not in out_seg) & ('.nii' not in out_seg) & ('.mgz' not in out_seg) & ('.npz' not in out_seg):
utils.mkdir(out_seg)
filename = os.path.basename(path_images).replace('.nii', '_seg.nii')
filename = filename.replace('mgz', '_seg.mgz')
filename = filename.replace('.npz', '_seg.npz')
out_seg = [os.path.join(out_seg, filename)]
else:
utils.mkdir(os.path.dirname(out_seg))
out_seg = [out_seg]
recompute_seg = [not os.path.isfile(out_seg[0])]
else:
out_seg = [out_seg]
recompute_seg = [False]
if out_posteriors is not None:
if ('.nii.gz' not in out_posteriors) & ('.nii' not in out_posteriors) & ('.mgz' not in out_posteriors) & \
('.npz' not in out_posteriors):
utils.mkdir(out_posteriors)
filename = os.path.basename(path_images).replace('.nii', '_posteriors.nii')
filename = filename.replace('mgz', '_posteriors.mgz')
filename = filename.replace('.npz', '_posteriors.npz')
out_posteriors = [os.path.join(out_posteriors, filename)]
else:
utils.mkdir(os.path.dirname(out_posteriors))
out_posteriors = [out_posteriors]
recompute_post = [not os.path.isfile(out_posteriors[0])]
else:
out_posteriors = [out_posteriors]
recompute_post = [out_volumes is not None]
recompute_list = [recompute | re_seg | re_post for (re_seg, re_post) in zip(recompute_seg, recompute_post)]
if out_volumes is not None:
if out_volumes[-4:] != '.csv':
print('out_volumes provided without csv extension. Adding csv extension to output_volumes.')
out_volumes += '.csv'
utils.mkdir(os.path.dirname(out_volumes))
return images_to_segment, out_seg, out_posteriors, out_volumes, recompute_list
# Prepare list of images to process
path_images = os.path.abspath(args['path_images'])
basename = os.path.basename(path_images)
path_predictions = os.path.abspath(args['path_predictions'])
# prepare input/output volumes
# First case: you're providing directories
if ('.nii.gz' not in basename) & ('.nii' not in basename) & (
'.mgz' not in basename) & ('.npz' not in basename):
if os.path.isfile(path_images):
raise Exception(
'extension not supported for %s, only use: nii.gz, .nii, .mgz, or .npz'
% path_images)
images_to_segment = utils.list_images_in_folder(path_images)
utils.mkdir(path_predictions)
path_predictions = [
os.path.join(path_predictions,
os.path.basename(image)).replace('.nii', '_SynthSR.nii')
for image in images_to_segment
]
path_predictions = [
seg_path.replace('.mgz', '_SynthSR.mgz')
for seg_path in path_predictions
]
path_predictions = [
seg_path.replace('.npz', '_SynthSR.npz')
for seg_path in path_predictions
]
else:
def dice_evaluation(gt_dir,
seg_dir,
label_list,
compute_distances=False,
compute_score_whole_structure=False,
path_dice=None,
path_hausdorff=None,
path_mean_distance=None,
crop_margin_around_gt=10,
recompute=True,
verbose=True):
"""This function computes Dice scores between two sets of labels maps in gt_dir (ground truth) and seg_dir
(typically predictions). Labels maps in both folders are matched by sorting order.
:param gt_dir: path of directory with gt label maps
:param seg_dir: path of directory with label maps to compare to gt_dir. Matched to gt label maps by sorting order.
:param label_list: list of label values for which to compute evaluation metrics. Can be a sequence, a 1d numpy
array, or the path to such array.
:param compute_distances: (optional) whether to compute distances (Hausdorff and mean distance) between the surfaces
of GT and predicted labels. Default is False.
:param compute_score_whole_structure: (optional) whether to also compute the selected scores for the whole segmented
structure (i.e. scores are computed for a single structure obtained by regrouping all non-zero values). If True, the
resulting scores are added as an extra row to the result matrices. Default is False.
:param path_dice: path where the resulting Dice will be writen as numpy array.
Default is None, where the array is not saved.
:param path_hausdorff: path where the resulting Hausdorff distances will be writen as numpy array (only if
compute_distances is True). Default is None, where the array is not saved.
:param path_mean_distance: path where the resulting mean distances will be writen as numpy array (only if
compute_distances is True). Default is None, where the array is not saved.
:param crop_margin_around_gt: (optional) margin by which to crop around the gt volumes, in order to copute the
scores more efficiently. If None, no cropping is performed.
:param recompute: (optional) whether to recompute the already existing results. Default is True.
:param verbose: (optional) whether to print out info about the remaining number of cases.
:return: numpy array containing all Dice scores (labels in rows, subjects in columns). Also returns numpy arrays
with the same structures for Hausdorff and mean distances if compute_distances is True.
"""
# check whether to recompute
compute_dice = not os.path.isfile(path_dice) if (path_dice
is not None) else True
if compute_distances:
compute_hausdorff = not os.path.isfile(path_hausdorff) if (
path_hausdorff is not None) else True
compute_mean_dist = not os.path.isfile(path_mean_distance) if (
path_mean_distance is not None) else True
else:
compute_hausdorff = compute_mean_dist = False
if compute_dice | compute_hausdorff | compute_mean_dist | recompute:
# get list label maps to compare
path_gt_labels = utils.list_images_in_folder(gt_dir)
path_segs = utils.list_images_in_folder(seg_dir)
if len(path_gt_labels) != len(path_segs):
print(
'gt and segmentation folders must have the same amount of label maps.'
)
# load labels list
label_list, _ = utils.get_list_labels(label_list=label_list,
FS_sort=True,
labels_dir=gt_dir)
n_labels = len(label_list)
# initialise result matrices
if compute_score_whole_structure:
max_dists = np.zeros((n_labels + 1, len(path_segs)))
mean_dists = np.zeros((n_labels + 1, len(path_segs)))
dice_coefs = np.zeros((n_labels + 1, len(path_segs)))
else:
max_dists = np.zeros((n_labels, len(path_segs)))
mean_dists = np.zeros((n_labels, len(path_segs)))
dice_coefs = np.zeros((n_labels, len(path_segs)))
# loop over segmentations
loop_info = utils.LoopInfo(len(path_segs), 10, 'evaluating')
for idx, (path_gt,
path_seg) in enumerate(zip(path_gt_labels, path_segs)):
if verbose:
loop_info.update(idx)
# load gt labels and segmentation
gt_labels = utils.load_volume(path_gt, dtype='int')
seg = utils.load_volume(path_seg, dtype='int')
# crop images
if crop_margin_around_gt is not None:
gt_labels, cropping = edit_volumes.crop_volume_around_region(
gt_labels, margin=crop_margin_around_gt)
seg = edit_volumes.crop_volume_with_idx(seg, cropping)
# compute Dice scores
dice_coefs[:n_labels, idx] = fast_dice(gt_labels, seg, label_list)
# compute Dice scores for whole structures
if compute_score_whole_structure:
temp_gt = (gt_labels > 0) * 1
temp_seg = (seg > 0) * 1
dice_coefs[-1, idx] = dice(temp_gt, temp_seg)
else:
temp_gt = temp_seg = None
# compute average and Hausdorff distances
if compute_distances:
# compute unique label values
unique_gt_labels = np.unique(gt_labels)
unique_seg_labels = np.unique(seg)
# compute max/mean surface distances for all labels
for index, label in enumerate(label_list):
if (label in unique_gt_labels) & (label
in unique_seg_labels):
mask_gt = np.where(gt_labels == label, True, False)
mask_seg = np.where(seg == label, True, False)
max_dists[index,
idx], mean_dists[index,
idx] = surface_distances(
mask_gt, mask_seg)
else:
max_dists[index, idx] = max(gt_labels.shape)
mean_dists[index, idx] = max(gt_labels.shape)
# compute max/mean distances for whole structure
if compute_score_whole_structure:
max_dists[-1, idx], mean_dists[-1,
idx] = surface_distances(
temp_gt, temp_seg)
# write results
if path_dice is not None:
utils.mkdir(os.path.dirname(path_dice))
np.save(path_dice, dice_coefs)
if compute_distances and path_hausdorff is not None:
utils.mkdir(os.path.dirname(path_hausdorff))
np.save(path_hausdorff, max_dists)
if compute_distances and path_mean_distance is not None:
utils.mkdir(os.path.dirname(path_mean_distance))
np.save(path_mean_distance, mean_dists)
else:
dice_coefs = np.load(path_dice)
if compute_distances:
max_dists = np.load(path_hausdorff)
mean_dists = np.load(path_mean_distance)
else:
max_dists = mean_dists = None
if compute_distances:
return dice_coefs, max_dists, mean_dists
else:
return dice_coefs, None, None
def training(labels_dir,
model_dir,
path_generation_labels=None,
path_segmentation_labels=None,
save_generation_labels=None,
batchsize=1,
n_channels=1,
target_res=None,
output_shape=None,
path_generation_classes=None,
prior_distributions='uniform',
prior_means=None,
prior_stds=None,
use_specific_stats_for_channel=False,
mix_prior_and_random=False,
flipping=True,
scaling_bounds=None,
rotation_bounds=None,
shearing_bounds=None,
nonlin_std=3.,
nonlin_shape_factor=.04,
blur_background=True,
data_res=None,
thickness=None,
downsample=False,
blur_range=1.15,
crop_channel_2=None,
bias_field_std=.3,
bias_shape_factor=.025,
n_levels=5,
nb_conv_per_level=2,
conv_size=3,
unet_feat_count=24,
feat_multiplier=2,
dropout=0,
no_batch_norm=False,
activation='elu',
lr=1e-4,
lr_decay=0,
wl2_epochs=5,
dice_epochs=100,
steps_per_epoch=1000,
background_weight=1e-4,
include_background=True,
loss_cropping=None,
load_model_file=None,
initial_epoch_wl2=0,
initial_epoch_dice=0):
"""
This function trains a Unet to segment MRI images with synthetic scans generated by sampling a GMM conditioned on
label maps. We regroup the parameters in three categories: Generation, Architecture, Training.
# IMPORTANT !!!
# Each time we provide a parameter with separate values for each axis (e.g. with a numpy array or a sequence),
# these values refer to the RAS axes.
:param labels_dir: path of folder with all input label maps, or to a single label map (if only one training example)
:param model_dir: path of a directory where the models will be saved during training.
#---------------------------------------------- Generation parameters ----------------------------------------------
# label maps parameters
:param path_generation_labels: (optional) list of all possible label values in the input label maps.
Default is None, where the label values are directly gotten from the provided label maps.
If not None, must be the path to a 1d numpy array, which should be organised as follows: background label first,
then non-sided labels (e.g. CSF, brainstem, etc.), then all the structures of the same hemisphere (can be left or
right), and finally all the corresponding contralateral structures (in the same order).
:param path_segmentation_labels: (optional) path to a numpy array of all the label values to keep in the output
label maps, in no particular order. Should be a subset of the values contained in path_generation_labels.
Labels that are in path_generation_labels but not in path_segmentation_labels are reset to zero.
By default segmentation labels are equal to generation labels.
:param save_generation_labels: (optional) path where to write the computed list of generation labels.
# output-related parameters
:param batchsize: (optional) number of images to generate per mini-batch. Default is 1.
:param n_channels: (optional) number of channels to be synthetised. Default is 1.
:param target_res: (optional) target resolution of the generated images and corresponding label maps.
If None, the outputs will have the same resolution as the input label maps.
Can be a number (isotropic resolution), or the path to a 1d numpy array.
:param output_shape: (optional) desired shape of the output image, obtained by randomly cropping the generated image
Can be an integer (same size in all dimensions), a sequence, a 1d numpy array, or the path to a 1d numpy array.
# GMM-sampling parameters
:param path_generation_classes: (optional) Indices regrouping generation labels into classes of same intensity
distribution. Regouped labels will thus share the same Gaussian when samling a new image. Should be the path to a 1d
numpy array with the same length as generation_labels. and contain values between 0 and K-1, where K is the total
number of classes. Default is all labels have different classes.
:param prior_distributions: (optional) type of distribution from which we sample the GMM parameters.
Can either be 'uniform', or 'normal'. Default is 'uniform'.
:param prior_means: (optional) hyperparameters controlling the prior distributions of the GMM means. Because
these prior distributions are uniform or normal, they require by 2 hyperparameters. Can be:
1) an array of shape (2, K), where K is the number of classes (K=len(generation_labels) if generation_classes is
not given). The mean of the Gaussian distribution associated to class k in [0, ...K-1] is sampled at each mini-batch
from U(prior_means[0,k], prior_means[1,k]) if prior_distributions is uniform, and from
N(prior_means[0,k], prior_means[1,k]) if prior_distributions is normal.
2) an array of shape (2*n_mod, K), where each block of two rows is associated to hyperparameters derived
from different modalities. In this case, if use_specific_stats_for_channel is False, we first randomly select a
modality from the n_mod possibilities, and we sample the GMM means like in 2).
If use_specific_stats_for_channel is True, each block of two rows correspond to a different channel
(n_mod=n_channels), thus we select the corresponding block to each channel rather than randomly drawing it.
Default is None, which corresponds all GMM means sampled from uniform distribution U(25, 225).
:param prior_stds: (optional) same as prior_means but for the standard deviations of the GMM.
Default is None, which corresponds to U(5, 25).
:param use_specific_stats_for_channel: (optional) whether the i-th block of two rows in the prior arrays must be
only used to generate the i-th channel. If True, n_mod should be equal to n_channels. Default is False.
:param mix_prior_and_random: (optional) if prior_means is not None, enables to reset the priors to their default
values for half of thes cases, and thus generate images of random contrast.
# spatial deformation parameters
:param flipping: (optional) whether to introduce right/left random flipping. Default is True.
:param scaling_bounds: (optional) if apply_linear_trans is True, the scaling factor for each dimension is
sampled from a uniform distribution of predefined bounds. Can either be:
1) a number, in which case the scaling factor is independently sampled from the uniform distribution of bounds
(1-scaling_bounds, 1+scaling_bounds) for each dimension.
2) the path to a numpy array of shape (2, n_dims), in which case the scaling factor in dimension i is sampled from
the uniform distribution of bounds (scaling_bounds[0, i], scaling_bounds[1, i]) for the i-th dimension.
If None (default), scaling_range = 0.15
:param rotation_bounds: (optional) same as scaling bounds but for the rotation angle, except that for case 1 the
bounds are centred on 0 rather than 1, i.e. (0+rotation_bounds[i], 0-rotation_bounds[i]).
If None (default), rotation_bounds = 15.
:param shearing_bounds: (optional) same as scaling bounds. If None (default), shearing_bounds = 0.01.
:param nonlin_std: (optional) Standard deviation of the normal distribution from which we sample the first
tensor for synthesising the deformation field.
:param nonlin_shape_factor: (optional) Ratio between the size of the input label maps and the size of the sampled
tensor for synthesising the elastic deformation field.
# blurring/resampling parameters
:param blur_background: (optional) If True, the background is blurred with the other labels, and can be reset to
zero with a probability of 0.2. If False, the background is not blurred (we apply an edge blurring correction),
and can be replaced by a low-intensity background with a probability of 0.5.
:param data_res: (optional) acquisition resolution to mimick. If provided, the images sampled from the GMM are
blurred to mimick data that would be: 1) acquired at the given acquisition resolution, and 2) resample at
target_resolution.
Default is None, where images are isotropically blurred to introduce some spatial correlation between voxels.
If the generated images are uni-modal, data_res can be a number (isotropic acquisition resolution), or the path to a
1d numpy array. In the multi-modal case, it should be given as a numpy array (or a path) of size (n_mod, n_dims),
where each row is the acquisition resolution of the correspionding chanel.
:param thickness: optional) if data_res is provided, we can further specify the slice thickness of the low
resolution images to mimick.
If the generated images are uni-modal, data_res can be a number (isotropic acquisition resolution), or the path to a
1d numy array. In the multi-modal case, it should be given as a numpy array (or a path) of size (n_mod, n_dims),
where each row is the acquisition resolution of the correspionding chanel.
:param downsample: (optional) whether to actually downsample the volume image to data_res (i.e. the acquisition
resolution). Default is False.
:param blur_range: (optional) Randomise the standard deviation of the blurring kernels, (whether data_res is given
or not). At each mini_batch, the standard deviation of the blurring kernels are multiplied by a coefficient sampled
from a uniform distribution with bounds [1/blur_range, blur_range]. If None, no randomisation. Default is 1.15.
:param crop_channel_2: (optional) stats for cropping second channel along the anterior-posterior axis.
Should be the path to a 1d numpy array of length 4, with bounds of uniform distribution for cropping the front and
back of the image (in percentage). None is no croppping.
# bias field parameters
:param bias_field_std: (optional) Standard deviation of the normal distribution from which we sample the first
tensor for synthesising the bias field.
:param bias_shape_factor: (optional) Ratio between the size of the input label maps and the size of the sampled
tensor for synthesising the bias field.
# ------------------------------------------ UNet architecture parameters ------------------------------------------
:param n_levels: (optional) number of level for the Unet. Default is 5.
:param nb_conv_per_level: (optional) number of convolutional layers per level. Default is 2.
:param conv_size: (optional) size of the convolution kernels. Default is 2.
:param unet_feat_count: (optional) number of feature for the first layr of the Unet. Default is 24.
:param feat_multiplier: (optional) multiply the number of feature by this nummber at each new level. Default is 2.
:param dropout: (optional) probability of drpout for the Unet. Deafult is 0, where no dropout is applied.
:param no_batch_norm: (optional) wheter to remove batch normalisation. Default is False, where BN is applied.
:param activation: (optional) activation function. Can be 'elu', 'relu'.
# ----------------------------------------------- Training parameters ----------------------------------------------
:param lr: (optional) learning rate for the training. Default is 1e-4
:param lr_decay: (optional) learing rate decay. Default is 0, where no decay is applied.
:param wl2_epochs: (optional) number of epohs for which the network (except the soft-max layer) is trained with L2
norm loss function. Default is 5.
:param dice_epochs: (optional) number of epochs with the soft Dice loss function. default is 100.
:param steps_per_epoch: (optional) number of steps per epoch. Default is 1000. Since no online validation is
possible, this is equivalent to the frequency at which the models are saved.
:param background_weight: (optional) weight of the background when computing loss. Default is 1e-4.
:param include_background: (optional) whether to include Dice of background when evaluating the loss function.
:param loss_cropping: (optional) margin by which to crop the posteriors when evaluating the loss function.
Can be an int, or the path to a 1d numpy array.
:param load_model_file: (optional) path of an already saved model to load before starting the training.
:param initial_epoch_wl2: (optional) initial epoch for wl2 training. Useful for resuming training.
:param initial_epoch_dice: (optional) initial epoch for dice training. Useful for resuming training.
"""
# check epochs
assert (wl2_epochs > 0) | (dice_epochs > 0), \
'either wl2_epochs or dice_epochs must be positive, had {0} and {1}'.format(wl2_epochs, dice_epochs)
# get label lists
generation_labels, n_neutral_labels = utils.get_list_labels(
label_list=path_generation_labels,
labels_dir=labels_dir,
save_label_list=save_generation_labels,
FS_sort=True)
if path_segmentation_labels is not None:
segmentation_labels, _ = utils.get_list_labels(
label_list=path_segmentation_labels, FS_sort=True)
else:
segmentation_labels = generation_labels
n_segmentation_labels = np.size(segmentation_labels)
# prepare model folder
utils.mkdir(model_dir)
# prepare log folder
log_dir = os.path.join(model_dir, 'logs')
utils.mkdir(log_dir)
# compute padding_margin
if loss_cropping is not None:
padding_margin = utils.get_padding_margin(output_shape, loss_cropping)
else:
padding_margin = None
# instantiate BrainGenerator object
brain_generator = BrainGenerator(
labels_dir=labels_dir,
generation_labels=generation_labels,
output_labels=segmentation_labels,
n_neutral_labels=n_neutral_labels,
padding_margin=padding_margin,
batchsize=batchsize,
n_channels=n_channels,
target_res=target_res,
output_shape=output_shape,
output_div_by_n=2**n_levels,
generation_classes=path_generation_classes,
prior_distributions=prior_distributions,
prior_means=prior_means,
prior_stds=prior_stds,
use_specific_stats_for_channel=use_specific_stats_for_channel,
mix_prior_and_random=mix_prior_and_random,
flipping=flipping,
scaling_bounds=scaling_bounds,
rotation_bounds=rotation_bounds,
shearing_bounds=shearing_bounds,
nonlin_std=nonlin_std,
nonlin_shape_factor=nonlin_shape_factor,
blur_background=blur_background,
data_res=data_res,
thickness=thickness,
downsample=downsample,
blur_range=blur_range,
crop_channel_2=crop_channel_2,
bias_field_std=bias_field_std,
bias_shape_factor=bias_shape_factor)
# transformation model
labels_to_image_model = brain_generator.labels_to_image_model
unet_input_shape = brain_generator.model_output_shape
# prepare the segmentation model
if no_batch_norm:
batch_norm_dim = None
else:
batch_norm_dim = -1
unet_model = nrn_models.unet(nb_features=unet_feat_count,
input_shape=unet_input_shape,
nb_levels=n_levels,
conv_size=conv_size,
nb_labels=n_segmentation_labels,
feat_mult=feat_multiplier,
nb_conv_per_level=nb_conv_per_level,
conv_dropout=dropout,
batch_norm=batch_norm_dim,
activation=activation,
input_model=labels_to_image_model)
# input generator
train_example_gen = brain_generator.model_inputs_generator
training_generator = utils.build_training_generator(
train_example_gen, batchsize)
# pre-training with weighted L2, input is fit to the softmax rather than the probabilities
if wl2_epochs > 0:
wl2_model = Model(unet_model.inputs,
[unet_model.get_layer('unet_likelihood').output])
wl2_model = metrics_model.metrics_model(
input_shape=unet_input_shape[:-1] + [n_segmentation_labels],
segmentation_label_list=segmentation_labels,
input_model=wl2_model,
loss_cropping=loss_cropping,
metrics='wl2',
weight_background=background_weight,
name='metrics_model')
if load_model_file is not None:
print('loading', load_model_file)
wl2_model.load_weights(load_model_file)
train_model(wl2_model, training_generator, lr, lr_decay, wl2_epochs,
steps_per_epoch, model_dir, log_dir, 'wl2',
initial_epoch_wl2)
# fine-tuning with dice metric
if dice_epochs > 0:
dice_model = metrics_model.metrics_model(
input_shape=unet_input_shape[:-1] + [n_segmentation_labels],
segmentation_label_list=segmentation_labels,
input_model=unet_model,
loss_cropping=loss_cropping,
include_background=include_background,
name='metrics_model')
if wl2_epochs > 0:
last_wl2_model_name = os.path.join(model_dir,
'wl2_%03d.h5' % wl2_epochs)
dice_model.load_weights(last_wl2_model_name, by_name=True)
elif load_model_file is not None:
print('loading', load_model_file)
dice_model.load_weights(load_model_file)
train_model(dice_model, training_generator, lr, lr_decay, dice_epochs,
steps_per_epoch, model_dir, log_dir, 'dice',
initial_epoch_dice)
def validate_training(image_dir,
gt_dir,
models_dir,
validation_main_dir,
segmentation_label_list,
evaluation_label_list=None,
dist_map=False,
step_eval=1,
aff_ref='FS',
sigma_smoothing=0,
keep_biggest_component=False,
padding=None,
cropping=None,
conv_size=3,
n_levels=5,
nb_conv_per_level=2,
unet_feat_count=24,
feat_multiplier=2,
activation='elu',
compute_distances=False,
recompute=True):
"""This function validates models saved at different epochs of the same training.
All models are assumed to be in the same folder.contained in models_dir.
The results of each model are saved in a subfolder in validation_main_dir.
:param image_dir: path of the folder with validation images.
:param gt_dir: path of the folder with ground truth label maps.
These are matched to the validation images by sorting order.
:param models_dir: path of the folder with the models to validate.
:param validation_main_dir: path of the folder where all the models validation subfolders will be saved.
:param segmentation_label_list: path of the numpy array containing all the segmentation labels used during training.
:param evaluation_label_list: (optional) label values to validate on. Must be a subset of the segmentation labels.
Can be a sequence, a 1d numpy array, or the path to a numpy 1d array. Default is the same as segmentation_label_list
:param dist_map: (optional) whether the input will contain distance maps channels (between each intenisty channels)
Default is False.
:param step_eval: (optional) If step_eval > 1 skips models when validating, by validating on models step_eval apart.
:param aff_ref: (optional) affine matrix with which the models were trained. Can be 'FS' (default), or 'identity.
:param sigma_smoothing: (optional) If not None, the posteriors are smoothed with a gaussian kernel of the specified
standard deviation.
:param keep_biggest_component: (optional) whether to only keep the biggest component in the predicted segmentation.
:param padding: (optional) pad the images to the specified shape before predicting the segmentation maps.
Can be an int, a sequence or a 1d numpy array.
:param cropping: (optional) whether to crop the input to smaller size while being run through the network.
The result is then given in the original image space. Can be an int, a sequence, or a 1d numpy array.
:param n_levels: (optional) number of level for the Unet. Default is 5.
:param nb_conv_per_level: (optional) number of convolutional layers per level. Default is 2.
:param conv_size: (optional) size of the convolution kernels. Default is 2.
:param unet_feat_count: (optional) number of feature maps for the first level. Default is 24.
:param feat_multiplier: (optional) multiply the number of feature by this nummber at each new level. Default is 1.
:param activation: (optional) activation function. Can be 'elu', 'relu'.
:param compute_distances: (optional) whether to compute the Haussdorf and mean surface distance.
:param recompute: (optional) whether to recompute result files even if they already exists."""
# create result folder
utils.mkdir(validation_main_dir)
# loop over models
list_models = utils.list_files(models_dir,
expr=['dice', 'h5'],
cond_type='and')[::step_eval]
loop_info = utils.LoopInfo(len(list_models), 1, 'validating', True)
for model_idx, path_model in enumerate(list_models):
# build names and create folders
model_val_dir = os.path.join(
validation_main_dir,
os.path.basename(path_model).replace('.h5', ''))
dice_path = os.path.join(model_val_dir, 'dice.npy')
utils.mkdir(model_val_dir)
if (not os.path.isfile(dice_path)) | recompute:
loop_info.update(model_idx)
predict(path_images=image_dir,
path_model=path_model,
segmentation_label_list=segmentation_label_list,
dist_map=dist_map,
path_segmentations=model_val_dir,
padding=padding,
cropping=cropping,
aff_ref=aff_ref,
sigma_smoothing=sigma_smoothing,
keep_biggest_component=keep_biggest_component,
conv_size=conv_size,
n_levels=n_levels,
nb_conv_per_level=nb_conv_per_level,
unet_feat_count=unet_feat_count,
feat_multiplier=feat_multiplier,
activation=activation,
gt_folder=gt_dir,
evaluation_label_list=evaluation_label_list,
compute_distances=compute_distances,
recompute=recompute,
verbose=False)
def evaluation(gt_dir,
seg_dir,
label_list,
mask_dir=None,
compute_score_whole_structure=False,
path_dice=None,
path_hausdorff=None,
path_hausdorff_99=None,
path_hausdorff_95=None,
path_mean_distance=None,
crop_margin_around_gt=10,
list_incorrect_labels=None,
list_correct_labels=None,
use_nearest_label=False,
recompute=True,
verbose=True):
"""This function computes Dice scores, as well as surface distances, between two sets of labels maps in gt_dir
(ground truth) and seg_dir (typically predictions). Labels maps in both folders are matched by sorting order.
The resulting scores are saved at the specified locations.
:param gt_dir: path of directory with gt label maps
:param seg_dir: path of directory with label maps to compare to gt_dir. Matched to gt label maps by sorting order.
:param label_list: list of label values for which to compute evaluation metrics. Can be a sequence, a 1d numpy
array, or the path to such array.
:param mask_dir: (optional) path of directory with masks of areas to ignore for each evaluated segmentation.
Matched to gt label maps by sorting order. Default is None, where nothing is masked.
:param compute_score_whole_structure: (optional) whether to also compute the selected scores for the whole segmented
structure (i.e. scores are computed for a single structure obtained by regrouping all non-zero values). If True, the
resulting scores are added as an extra row to the result matrices. Default is False.
:param path_dice: path where the resulting Dice will be writen as numpy array.
Default is None, where the array is not saved.
:param path_hausdorff: path where the resulting Hausdorff distances will be writen as numpy array (only if
compute_distances is True). Default is None, where the array is not saved.
:param path_hausdorff_99: same as for path_hausdorff but for the 99th percentile of the boundary distance.
:param path_hausdorff_95: same as for path_hausdorff but for the 95th percentile of the boundary distance.
:param path_mean_distance: path where the resulting mean distances will be writen as numpy array (only if
compute_distances is True). Default is None, where the array is not saved.
:param crop_margin_around_gt: (optional) margin by which to crop around the gt volumes, in order to copute the
scores more efficiently. If None, no cropping is performed.
:param list_incorrect_labels: (optional) this option enables to replace some label values in the maps in seg_dir by
other label values. Can be a list, a 1d numpy array, or the path to such an array.
The incorrect labels can then be replaced either by specified values, or by the nearest value (see below).
:param list_correct_labels: (optional) list of values to correct the labels specified in list_incorrect_labels.
Correct values must have the same order as their corresponding value in list_incorrect_labels.
:param use_nearest_label: (optional) whether to correct the incorrect lavel values with the nearest labels.
:param recompute: (optional) whether to recompute the already existing results. Default is True.
:param verbose: (optional) whether to print out info about the remaining number of cases.
"""
# check whether to recompute
compute_dice = not os.path.isfile(path_dice) if (path_dice
is not None) else True
compute_hausdorff = not os.path.isfile(path_hausdorff) if (
path_hausdorff is not None) else False
compute_hausdorff_99 = not os.path.isfile(path_hausdorff_99) if (
path_hausdorff_99 is not None) else False
compute_hausdorff_95 = not os.path.isfile(path_hausdorff_95) if (
path_hausdorff_95 is not None) else False
compute_mean_dist = not os.path.isfile(path_mean_distance) if (
path_mean_distance is not None) else False
compute_hd = [
compute_hausdorff, compute_hausdorff_99, compute_hausdorff_95
]
if compute_dice | any(compute_hd) | compute_mean_dist | recompute:
# get list label maps to compare
path_gt_labels = utils.list_images_in_folder(gt_dir)
path_segs = utils.list_images_in_folder(seg_dir)
path_gt_labels = utils.reformat_to_list(path_gt_labels,
length=len(path_segs))
if len(path_gt_labels) != len(path_segs):
print(
'gt and segmentation folders must have the same amount of label maps.'
)
if mask_dir is not None:
path_masks = utils.list_images_in_folder(mask_dir)
if len(path_masks) != len(path_segs):
print('not the same amount of masks and segmentations.')
else:
path_masks = [None] * len(path_segs)
# load labels list
label_list, _ = utils.get_list_labels(label_list=label_list,
FS_sort=True,
labels_dir=gt_dir)
n_labels = len(label_list)
max_label = np.max(label_list) + 1
# initialise result matrices
if compute_score_whole_structure:
max_dists = np.zeros((n_labels + 1, len(path_segs), 3))
mean_dists = np.zeros((n_labels + 1, len(path_segs)))
dice_coefs = np.zeros((n_labels + 1, len(path_segs)))
else:
max_dists = np.zeros((n_labels, len(path_segs), 3))
mean_dists = np.zeros((n_labels, len(path_segs)))
dice_coefs = np.zeros((n_labels, len(path_segs)))
# loop over segmentations
loop_info = utils.LoopInfo(len(path_segs),
10,
'evaluating',
print_time=True)
for idx, (path_gt, path_seg, path_mask) in enumerate(
zip(path_gt_labels, path_segs, path_masks)):
if verbose:
loop_info.update(idx)
# load gt labels and segmentation
gt_labels = utils.load_volume(path_gt, dtype='int')
seg = utils.load_volume(path_seg, dtype='int')
if path_mask is not None:
mask = utils.load_volume(path_mask, dtype='bool')
gt_labels[mask] = max_label
seg[mask] = max_label
# crop images
if crop_margin_around_gt is not None:
gt_labels, cropping = edit_volumes.crop_volume_around_region(
gt_labels, margin=crop_margin_around_gt)
seg = edit_volumes.crop_volume_with_idx(seg, cropping)
if list_incorrect_labels is not None:
seg = edit_volumes.correct_label_map(seg,
list_incorrect_labels,
list_correct_labels,
use_nearest_label)
# compute Dice scores
dice_coefs[:n_labels, idx] = fast_dice(gt_labels, seg, label_list)
# compute Dice scores for whole structures
if compute_score_whole_structure:
temp_gt = (gt_labels > 0) * 1
temp_seg = (seg > 0) * 1
dice_coefs[-1, idx] = dice(temp_gt, temp_seg)
else:
temp_gt = temp_seg = None
# compute average and Hausdorff distances
if any(compute_hd) | compute_mean_dist:
# compute unique label values
unique_gt_labels = np.unique(gt_labels)
unique_seg_labels = np.unique(seg)
# compute max/mean surface distances for all labels
for index, label in enumerate(label_list):
if (label in unique_gt_labels) & (label
in unique_seg_labels):
mask_gt = np.where(gt_labels == label, True, False)
mask_seg = np.where(seg == label, True, False)
tmp_max_dists, mean_dists[index,
idx] = surface_distances(
mask_gt, mask_seg,
[100, 99, 95])
max_dists[index, idx, :] = np.array(tmp_max_dists)
else:
mean_dists[index, idx] = max(gt_labels.shape)
max_dists[index, idx, :] = np.array(
[max(gt_labels.shape)] * 3)
# compute max/mean distances for whole structure
if compute_score_whole_structure:
tmp_max_dists, mean_dists[-1, idx] = surface_distances(
temp_gt, temp_seg, [100, 99, 95])
max_dists[-1, idx, :] = np.array(tmp_max_dists)
# write results
if path_dice is not None:
utils.mkdir(os.path.dirname(path_dice))
np.save(path_dice, dice_coefs)
if path_hausdorff is not None:
utils.mkdir(os.path.dirname(path_hausdorff))
np.save(path_hausdorff, max_dists[..., 0])
if path_hausdorff_99 is not None:
utils.mkdir(os.path.dirname(path_hausdorff_99))
np.save(path_hausdorff_99, max_dists[..., 1])
if path_hausdorff_95 is not None:
utils.mkdir(os.path.dirname(path_hausdorff_95))
np.save(path_hausdorff_95, max_dists[..., 2])
if path_mean_distance is not None:
utils.mkdir(os.path.dirname(path_mean_distance))
np.save(path_mean_distance, max_dists[..., 2])
def dice_evaluation(gt_dir,
seg_dir,
path_label_list,
path_result_dice_array,
cropping_margin_around_gt=10,
verbose=True):
"""Computes Dice scores for all labels contained in path_segmentation_label_list. Files in gt_folder and seg_folder
are matched by sorting order.
:param gt_dir: folder containing ground truth files.
:param seg_dir: folder containing evaluation files.
:param path_label_list: path of numpy vector containing all labels to compute the Dice for.
:param path_result_dice_array: path where the resulting Dice will be writen as numpy array.
:param cropping_margin_around_gt: (optional) margin by which to crop around the gt volumes.
If None, no cropping is performed.
:param verbose: (optional) whether to print out info about the remaining number of cases.
:return: numpy array containing all dice scores (labels in rows, subjects in columns).
"""
# create result folder
utils.mkdir(os.path.dirname(path_result_dice_array))
# get list label maps to compare
path_gt_labels = utils.list_images_in_folder(gt_dir)
path_segs = utils.list_images_in_folder(seg_dir)
if len(path_gt_labels) != len(path_segs):
print(
'different number of files in data folders, had {} and {}'.format(
len(path_gt_labels), len(path_segs)))
# load labels list
label_list, neutral_labels = utils.get_list_labels(
label_list=path_label_list, FS_sort=True, labels_dir=gt_dir)
label_list_sorted = np.sort(label_list)
# initialise result matrix
dice_coefs = np.zeros((label_list.shape[0], len(path_segs)))
# loop over segmentations
for idx, (path_gt, path_seg) in enumerate(zip(path_gt_labels, path_segs)):
if verbose:
utils.print_loop_info(idx, len(path_segs), 10)
# load gt labels and segmentation
gt_labels = utils.load_volume(path_gt, dtype='int')
seg = utils.load_volume(path_seg, dtype='int')
# crop images
if cropping_margin_around_gt is not None:
gt_labels, cropping = edit_volumes.crop_volume_around_region(
gt_labels, margin=cropping_margin_around_gt)
seg = edit_volumes.crop_volume_with_idx(seg, cropping)
# compute dice scores
tmp_dice = fast_dice(gt_labels, seg, label_list_sorted)
if len(label_list_sorted) > 1:
dice_coefs[:,
idx] = tmp_dice[np.searchsorted(label_list_sorted,
label_list)]
else:
dice_coefs[:, idx] = tmp_dice
# write dice results
np.save(path_result_dice_array, dice_coefs)
return dice_coefs
