def _get_subjects(self, data_folder):
dirs = sorted([
os.path.basename(f)
for f in glob.glob(os.path.join(data_folder, '*'))
])
subjects = []
for directory in dirs:
for resolution in self._resolutions_to_use:
subject_pjoin = mdt.make_path_joiner(data_folder, directory,
resolution)
if os.path.exists(subject_pjoin()):
niftis = glob.glob(subject_pjoin('*.nii*'))
dwi_fname = list(
filter(
lambda v: '_mask' not in v and 'grad_dev' not in v,
niftis))[0]
mask_fname = list(
sorted(filter(lambda v: '_mask' in v, niftis)))[0]
protocol_fname = glob.glob(subject_pjoin('*prtcl'))[0]
protocol_loader = BatchFitProtocolLoader(
subject_pjoin(), protocol_fname=protocol_fname)
subjects.append(
SimpleSubjectInfo(data_folder, subject_pjoin(),
directory + '_' + resolution,
dwi_fname, protocol_loader,
mask_fname))
return subjects
def _get_subjects(self):
subjects = []
for subject_id in sorted([os.path.basename(f) for f in glob.glob(os.path.join(self._root_dir, '*'))]):
pjoin = mdt.make_path_joiner(self._root_dir, subject_id)
subject_info = self._get_subject_in_directory(subject_id, pjoin)
if subject_info:
subjects.append(subject_info)
return subjects
def _run_b1k_b2k_analysis(cls):
pjoin = mdt.make_path_joiner(
os.path.join(cls._tmp_dir, cls._tmp_dir_subdir, 'b1k_b2k'))
input_data = mdt.load_input_data(
pjoin('b1k_b2k_example_slices_24_38'), pjoin('b1k_b2k.prtcl'),
pjoin('b1k_b2k_example_slices_24_38_mask'))
for model_name in ['BallStick_r1', 'Tensor', 'NODDI']:
mdt.fit_model(model_name, input_data,
pjoin('output', 'b1k_b2k_example_slices_24_38_mask'))
def _run_b6k_analysis(cls):
pjoin = mdt.make_path_joiner(
os.path.join(cls._tmp_dir, cls._tmp_dir_subdir,
'multishell_b6k_max'))
input_data = mdt.load_input_data(
pjoin('multishell_b6k_max_example_slices_24_38'),
pjoin('multishell_b6k_max.prtcl'),
pjoin('multishell_b6k_max_example_slices_24_38_mask'))
for model_name in ['CHARMED_r1', 'CHARMED_r2', 'CHARMED_r3']:
mdt.fit_model(
model_name, input_data,
pjoin('output',
'multishell_b6k_max_example_slices_24_38_mask'))
def test_lls_multishell_b6k_max(self):
known_values = {
'CHARMED_r1': {
'LogLikelihood': {
'mean': -8038.29248046875,
'std': 4703.6005859375
}
},
'CHARMED_r2': {
'LogLikelihood': {
'mean': -7001.50927734375,
'std': 4062.600341796875
}
},
'CHARMED_r3': {
'LogLikelihood': {
'mean': -6514.17529296875,
'std': 3659.7138671875
}
}
}
for model_name in ['CHARMED_r1', 'CHARMED_r2', 'CHARMED_r3']:
pjoin = mdt.make_path_joiner(
os.path.join(self._tmp_dir, self._tmp_dir_subdir,
'multishell_b6k_max'))
user_volumes = mdt.load_volume_maps(
pjoin('output', 'multishell_b6k_max_example_slices_24_38_mask',
model_name))
msg_prefix = 'b1k_b2k - {}'.format(model_name)
roi = mdt.create_roi(
user_volumes['LogLikelihood'],
pjoin('multishell_b6k_max_example_slices_24_38_mask'))
for map_name, test_values in known_values[model_name].items():
np.testing.assert_allclose(test_values['mean'],
np.mean(roi),
rtol=1e-4,
err_msg='{} - {} - mean'.format(
msg_prefix, map_name))
np.testing.assert_allclose(test_values['std'],
np.std(roi),
rtol=1e-4,
err_msg='{} - {} - std'.format(
msg_prefix, map_name))
def test_lls_multishell_b6k_max(self):
known_values = {
'CHARMED_r1': {
'LogLikelihood': {
'mean': -447.95446,
'std': 36.09088
}
},
'CHARMED_r2': {
'LogLikelihood': {
'mean': -436.13861,
'std': 25.13305
}
},
'CHARMED_r3': {
'LogLikelihood': {
'mean': -432.15878,
'std': 21.87481
}
}
}
for model_name in ['CHARMED_r1', 'CHARMED_r2', 'CHARMED_r3']:
pjoin = mdt.make_path_joiner(
os.path.join(self._tmp_dir, self._tmp_dir_subdir,
'multishell_b6k_max'))
user_volumes = mdt.load_volume_maps(
pjoin('output', 'multishell_b6k_max_example_slices_24_38_mask',
model_name))
msg_prefix = 'b1k_b2k - {}'.format(model_name)
roi = mdt.create_roi(
user_volumes['LogLikelihood'],
pjoin('multishell_b6k_max_example_slices_24_38_mask'))
for map_name, test_values in known_values[model_name].items():
np.testing.assert_allclose(test_values['mean'],
np.mean(roi),
rtol=1e-4,
err_msg='{} - {} - mean'.format(
msg_prefix, map_name))
np.testing.assert_allclose(test_values['std'],
np.std(roi),
rtol=1e-4,
err_msg='{} - {} - std'.format(
msg_prefix, map_name))
def test_lls_b1k_b2k(self):
known_values = {
'BallStick_r1': {
'LogLikelihood': {
'mean': -1215.52355,
'std': 924.27117
}
},
'Tensor': {
'LogLikelihood': {
'mean': -182.73164,
'std': 20.02311
}
},
'NODDI': {
'LogLikelihood': {
'mean': -451.16198,
'std': 37.77514
}
}
}
for model_name in ['BallStick_r1', 'Tensor', 'NODDI']:
pjoin = mdt.make_path_joiner(
os.path.join(self._tmp_dir, self._tmp_dir_subdir, 'b1k_b2k'))
user_volumes = mdt.load_volume_maps(
pjoin('output', 'b1k_b2k_example_slices_24_38_mask',
model_name))
msg_prefix = 'b1k_b2k - {}'.format(model_name)
roi = mdt.create_roi(user_volumes['LogLikelihood'],
pjoin('b1k_b2k_example_slices_24_38_mask'))
for map_name, test_values in known_values[model_name].items():
np.testing.assert_allclose(test_values['mean'],
np.mean(roi),
rtol=1e-4,
err_msg='{} - {} - mean'.format(
msg_prefix, map_name))
np.testing.assert_allclose(test_values['std'],
np.std(roi),
rtol=1e-4,
err_msg='{} - {} - std'.format(
msg_prefix, map_name))
def test_lls_b1k_b2k(self):
known_values = {
'BallStick_r1': {
'LogLikelihood': {
'mean': -327297.53125,
'std': 359791.5625
}
},
'Tensor': {
'LogLikelihood': {
'mean': -5891.28515625,
'std': 7691.93408203125
}
},
'NODDI': {
'LogLikelihood': {
'mean': -19352.4609375,
'std': 14489.8623046875
}
}
}
for model_name in ['BallStick_r1', 'Tensor', 'NODDI']:
pjoin = mdt.make_path_joiner(
os.path.join(self._tmp_dir, self._tmp_dir_subdir, 'b1k_b2k'))
user_volumes = mdt.load_volume_maps(
pjoin('output', 'b1k_b2k_example_slices_24_38_mask',
model_name))
msg_prefix = 'b1k_b2k - {}'.format(model_name)
roi = mdt.create_roi(user_volumes['LogLikelihood'],
pjoin('b1k_b2k_example_slices_24_38_mask'))
for map_name, test_values in known_values[model_name].items():
np.testing.assert_allclose(test_values['mean'],
np.mean(roi),
rtol=1e-4,
err_msg='{} - {} - mean'.format(
msg_prefix, map_name))
np.testing.assert_allclose(test_values['std'],
np.std(roi),
rtol=1e-4,
err_msg='{} - {} - std'.format(
msg_prefix, map_name))
def single_subject_results(subject_info):
output_folder = subject_info.data_folder[:-1] + '_output'
subject_id = subject_info.subject_id
input_data = subject_info.get_input_data()
wm_mask = output_folder + '/' + subject_id + '/wm_mask'
snrs = estimate_snr_wm_mask(input_data, wm_mask)
results_pjoin = mdt.make_path_joiner(output_folder + '/' + subject_id)
results = {}
for model in model_names:
results_per_method = {}
for method in ['sample', 'optimization']:
results_per_snr = {}
if method == 'sample':
std_data = get_mcmc_std(results_pjoin, model)
else:
std_data = get_mle_std(results_pjoin, model)
std_data = mdt.create_roi(std_data, wm_mask)
for snr in noise_snrs:
bin_width = 2.5
std_values_in_snr = std_data[np.where(
(snr - bin_width <= snrs) & (snrs <= snr + bin_width))[0]]
cutoff = 1
std_values_in_snr = std_values_in_snr[
np.isfinite(std_values_in_snr)
& (std_values_in_snr < cutoff) & (std_values_in_snr > 0)]
results_per_snr.update({snr: np.mean(std_values_in_snr)})
results_per_method[method] = results_per_snr
results[model] = results_per_method
return results
import mdt
import matplotlib.pyplot as plt
from mdt.lib.post_processing import DTIMeasures
import numpy as np
import seaborn
seaborn.set()
__author__ = 'Robbert Harms'
__date__ = '2018-11-02'
__maintainer__ = 'Robbert Harms'
__email__ = '*****@*****.**'
__licence__ = 'LGPL v3'
input_pjoin = mdt.make_path_joiner('/home/robbert/programming/python/uncertainty_paper/data/single_slice/')
output_base_pjoin = mdt.make_path_joiner('/home/robbert/phd-data/papers/uncertainty_paper/single_slice/')
figure_output_pjoin = mdt.make_path_joiner('/tmp/uncertainty_paper/mean_mode_comparison')
def set_matplotlib_font_size(font_size):
import matplotlib.pyplot as plt
plt.rc('font', size=font_size) # controls default text sizes
plt.rc('axes', titlesize=font_size) # fontsize of the axes title
plt.rc('axes', labelsize=font_size) # fontsize of the x and y labels
plt.rc('xtick', labelsize=font_size) # fontsize of the tick labels
plt.rc('ytick', labelsize=font_size) # fontsize of the tick labels
plt.rc('legend', fontsize=font_size) # legend fontsize
plt.rc('figure', titlesize=font_size)
def _get_subjects(self):
pjoin = mdt.make_path_joiner(self._root_dir)
files = [os.path.basename(f) for f in glob.glob(pjoin('*'))]
basenames = sorted(list({split_image_path(f)[1] for f in files}))
subjects = []
protocol_options = ['TE', 'TR', 'Delta', 'delta', 'maxG']
default_mask = None
if list(glob.glob(pjoin('mask.nii*'))):
default_mask = list(glob.glob(pjoin('mask.nii*')))[0]
for basename in basenames:
dwi_fname = None
if basename + '.nii' in files:
dwi_fname = pjoin(basename + '.nii')
elif basename + '.nii.gz' in files:
dwi_fname = pjoin(basename + '.nii.gz')
elif basename + '.hdr' in files and basename + '.img' in files:
dwi_fname = pjoin(basename + '.hdr')
noise_std = self._autoload_noise_std(basename,
file_path=pjoin(basename +
'.noise_std'))
prtcl_fname = None
if basename + '.prtcl' in files:
prtcl_fname = pjoin(basename + '.prtcl')
bval_fname = None
if basename + '.bval' in files:
bval_fname = pjoin(basename + '.bval')
bvec_fname = None
if basename + '.bvec' in files:
bvec_fname = pjoin(basename + '.bvec')
mask_fname = default_mask
if basename + '_mask.nii' in files:
mask_fname = pjoin(basename + '_mask.nii')
elif basename + '_mask.nii.gz' in files:
mask_fname = pjoin(basename + '_mask.nii.gz')
extra_cols_from_file = {}
for option in protocol_options:
if basename + '.' + option in files:
extra_cols_from_file.update(
{option: pjoin(basename + '.' + option)})
if dwi_fname and (prtcl_fname or (bval_fname and bvec_fname)):
protocol_loader = BatchFitProtocolLoader(
pjoin(),
protocol_fname=prtcl_fname,
bvec_fname=bvec_fname,
bval_fname=bval_fname,
protocol_columns=extra_cols_from_file)
output_dir = self._get_subject_output_dir(
basename, mask_fname, pjoin(self.output_base_dir,
basename))
subjects.append(
SimpleSubjectInfo(basename,
dwi_fname,
protocol_loader,
mask_fname,
output_dir,
noise_std=noise_std))
return subjects
from scipy.ndimage import binary_erosion
import mdt
__author__ = 'Robbert Harms'
__date__ = '2018-12-20'
__maintainer__ = 'Robbert Harms'
__email__ = '*****@*****.**'
__licence__ = 'LGPL v3'
output_pjoin = mdt.make_path_joiner(
'/home/robbert/phd-data/papers/uncertainty_paper/registration/')
mask = mdt.load_brain_mask(
'/usr/share/data/fsl-mni152-templates/FMRIB58_FA_1mm.nii.gz')
mask = binary_erosion(mask, iterations=1)
maps = {}
subjects_to_load = ['mgh_1005', 'mgh_1016', 'mgh_1017']
for subject in subjects_to_load:
point_map = mdt.load_nifti(
output_pjoin(subject, 'warped_BinghamNODDI_r1_w_in0.w')).get_data()
std_map = mdt.load_nifti(
output_pjoin(subject,
'warped_BinghamNODDI_r1_w_in0.w.std')).get_data()
maps[subject + '.std'] = std_map
maps[subject] = point_map
mdt.apply_mask(maps, mask)
import glob
import mdt
import os
from mdt.lib.batch_utils import BatchFitProtocolLoader, SimpleSubjectInfo
import matplotlib.pyplot as plt
import numpy as np
import seaborn
seaborn.set()
from mdt.lib.batch_utils import SimpleBatchProfile
from matplotlib.ticker import FuncFormatter, MaxNLocator, FixedLocator
figure_output_pjoin = mdt.make_path_joiner('/tmp/uncertainty_paper/std_snr/',
make_dirs=True)
def set_matplotlib_font_size(font_size):
import matplotlib.pyplot as plt
plt.rc('font', size=font_size) # controls default text sizes
plt.rc('axes', titlesize=font_size) # fontsize of the axes title
plt.rc('axes', labelsize=font_size) # fontsize of the x and y labels
plt.rc('xtick', labelsize=font_size) # fontsize of the tick labels
plt.rc('ytick', labelsize=font_size) # fontsize of the tick labels
plt.rc('legend', fontsize=font_size) # legend fontsize
plt.rc('figure', titlesize=font_size)
class RheinLandBatchProfile(SimpleBatchProfile):
def __init__(self, *args, resolutions_to_use=None, **kwargs):
"""Construct the Rheinland study batch profile.
Args:
from matplotlib.ticker import ScalarFormatter, FuncFormatter, MaxNLocator
import pylab
import mdt
from mdt.lib.post_processing import DTIMeasures
import numpy as np
import matplotlib.pyplot as plt
import seaborn
seaborn.set()
__author__ = 'Robbert Harms'
__date__ = '2018-11-09'
__maintainer__ = 'Robbert Harms'
__email__ = '*****@*****.**'
__licence__ = 'LGPL v3'
input_pjoin = mdt.make_path_joiner(
'/home/robbert/programming/python/uncertainty_paper/data/snr_simulations/')
output_base_pjoin = mdt.make_path_joiner(
'/home/robbert/phd-data/papers/uncertainty_paper/snr_simulations/')
figure_output_pjoin = mdt.make_path_joiner(
'/tmp/uncertainty_paper/snr_simulations/', make_dirs=True)
nmr_trials = 2
simulations_unweighted_signal_height = 1e4
# noise_snrs = [2, 5, 10, 20, 30, 40, 50]
noise_snrs = [5, 10, 20, 30, 40, 50]
protocols = ['hcp_mgh_1003', 'rheinland_v3a_1_2mm']
model_names = [
'BallStick_r1',
__date__ = '2018-01-01'
__maintainer__ = 'Robbert Harms'
__email__ = '*****@*****.**'
__licence__ = 'LGPL v3'
"""
This script is meant to generate the simulation data needed for figure 4 and 5 of the article.
Before running this script, please copy the protocol files "hcp_mgh_1003.prtcl" and "rheinland_v3a_1_2mm.prtcl"
to the directory where you wish to store the simulation data and results. Set the same path here in this script.
"""
# the path to where you stored the protocol files.
data_storage_path = r'/home/robbert/phd-data/papers/uncertainty_paper/snr_simulations/'
# an utility for easily appending to paths
pjoin = mdt.make_path_joiner(data_storage_path)
# The names of the protocol files we wish to simulate data for
protocols = ['hcp_mgh_1003', 'rheinland_v3a_1_2mm']
# The models for which we want to simulate the data
models = [
'BallStick_r1', 'BallStick_r2', 'BallStick_r3', 'NODDI', 'BinghamNODDI_r1',
'Tensor', 'CHARMED_r1', 'CHARMED_r2', 'CHARMED_r3'
]
# The different SNR's we wish to simulate. Each additional SNR will generate a copy of the simulated data with
# noise corresponding to the desired SNR.
noise_snrs = [2, 5, 10, 20, 30, 40, 50]
# The b0 signal intensity, fixed such that we can easily generate the noise (noise_std = unweighted_signal_height / SNR)
def _get_subjects(self):
dirs = sorted([
os.path.basename(f)
for f in glob.glob(os.path.join(self._root_dir, '*'))
])
subjects = []
for subject_id in dirs:
pjoin = mdt.make_path_joiner(self._root_dir, subject_id, 'diff',
'preproc')
if os.path.isdir(pjoin()):
dwi_fname = list(glob.glob(pjoin('mri',
'diff_preproc.nii*')))[0]
noise_std = self._autoload_noise_std(
subject_id, file_path=pjoin('noise_std'))
bval_fname = pjoin('bvals.txt')
if os.path.isfile(pjoin('diff_preproc.bval')):
bval_fname = pjoin('diff_preproc.bval')
bvec_fname = pjoin('bvecs_fsl_moco_norm.txt')
if os.path.isfile(pjoin('diff_preproc.bvec')):
bvec_fname = pjoin('diff_preproc.bvec')
prtcl_fname = None
if os.path.isfile(pjoin('diff_preproc.prtcl')):
prtcl_fname = pjoin('diff_preproc.prtcl')
mask_fname = None
if list(glob.glob(pjoin('diff_preproc_mask.nii*'))):
mask_fname = list(
glob.glob(pjoin('diff_preproc_mask.nii*')))[0]
if mask_fname is None:
if list(glob.glob(pjoin('mri', 'diff_preproc_mask.nii*'))):
mask_fname = list(
glob.glob(pjoin('mri',
'diff_preproc_mask.nii*')))[0]
protocol_loader = BatchFitProtocolLoader(
pjoin(),
protocol_fname=prtcl_fname,
bvec_fname=bvec_fname,
bval_fname=bval_fname,
protocol_columns={
'Delta': 21.8e-3,
'delta': 12.9e-3,
'TR': 8800e-3,
'TE': 57e-3
})
output_dir = self._get_subject_output_dir(
subject_id, mask_fname)
subjects.append(
SimpleSubjectInfo(subject_id,
dwi_fname,
protocol_loader,
mask_fname,
output_dir,
noise_std=noise_std))
return subjects
import mdt
__author__ = 'Robbert Harms'
__date__ = "2018-08-14"
__maintainer__ = "Robbert Harms"
__email__ = "*****@*****.**"
pjoin = mdt.make_path_joiner(
'/home/robbert/phd-data/papers/uncertainty_paper/snr_simulations/')
nmr_trials = 10
simulations_unweighted_signal_height = 1e4
nmr_samples = {
'BallStick_r1': 11000,
'BallStick_r2': 15000,
'BallStick_r3': 25000,
'NODDI': 15000,
'BinghamNODDI_r1': 20000,
'Tensor': 13000,
'CHARMED_r1': 17000,
'CHARMED_r2': 25000,
'CHARMED_r3': 25000
}
protocols = ['hcp_mgh_1003', 'rheinland_v3a_1_2mm']
noise_snrs = [2, 5, 10, 20, 30, 40, 50]
model_names = [
'BallStick_r1', 'BallStick_r2', 'BallStick_r3', 'NODDI', 'BinghamNODDI_r1',
'Tensor', 'CHARMED_r1', 'CHARMED_r2', 'CHARMED_r3'
]
model_titles = {
'BallStick_r1': 'BallStick_in1',
'BallStick_r2': 'BallStick_in2',
'BallStick_r3': 'BallStick_in3',
'Tensor': 'Tensor',
'NODDI': 'NODDI',
'BinghamNODDI_r1': 'BinghamNODDI',
'CHARMED_r1': 'CHARMED_in1',
'CHARMED_r2': 'CHARMED_in2',
'CHARMED_r3': 'CHARMED_in3'
}
output_base_pjoin = mdt.make_path_joiner('/home/robbert/phd-data/papers/uncertainty_paper/single_subject/')
# output_base_pjoin = mdt.make_path_joiner('/home/robbert/Downloads/single_subject/')
figure_output_pjoin = mdt.make_path_joiner('/tmp/uncertainty_paper/mle_mcmc_scatter/', make_dirs=True)
goc_output_pjoin = mdt.make_path_joiner('/home/robbert/phd-data/papers/uncertainty_paper/single_subject_gocs/')
def get_mle_results(dataset_name, model_name):
maps = mdt.load_volume_maps(output_base_pjoin(dataset_name, model_name))
if model_name == 'Tensor':
return maps['Tensor.FA'], maps['Tensor.FA.std']
elif model_name == 'BallStick_r1':
return maps['w_stick0.w'], maps['w_stick0.w.std']
elif model_name == 'BallStick_r2':
return maps['FS'], maps['FS.std']
