def create_signal_estimates(model, input_data, parameters):
"""Create the signals estimates for your estimated model parameters.
This function is typically used to obtain signal estimates from optimization results.
This function evaluates the model as it is in the model fitting and sample. That is, this method includes
the gradient deviations (if set in the input data) and loads all static and fixed parameters maps.
Args:
model (str or model): the model or the name of the model to use for estimating the signals
input_data (mdt.utils.MRIInputData): the input data object, we will set this to the model
parameters (str or dict): either a directory file name or a dictionary containing optimization results
Each element is assumed to be a 4d volume with the voxels we are using for the simulations.
Returns:
ndarray: the 4d array with the signal estimates per voxel
"""
if isinstance(model, str):
model = get_model(model)()
model.set_input_data(input_data)
build_model = model.build()
if isinstance(parameters, str):
parameters = get_all_nifti_data(parameters)
parameters = create_roi(parameters, input_data.mask)
parameters = model.param_dict_to_array(parameters)
kernel_data = {
'data':
build_model.get_kernel_data(),
'parameters':
Array(parameters, ctype='mot_float_type'),
'estimates':
Zeros((parameters.shape[0], build_model.get_nmr_observations()),
'mot_float_type')
}
_get_simulate_function(build_model).evaluate(kernel_data,
parameters.shape[0])
results = kernel_data['estimates'].get_data()
return restore_volumes(results, input_data.mask)
def simulate_signals(model, protocol, parameters):
"""Estimate the signals of a given model for the given combination of protocol and parameters.
In contrast to the function :func:`create_signal_estimates`, this function does not incorporate the gradient
deviations. Furthermore, this function expects a two dimensional list of parameters and this function will
simply evaluate the model for each set of parameters.
Args:
model (str or model): the model or the name of the model to use for estimating the signals
protocol (mdt.protocols.Protocol): the protocol we will use for the signal simulation
parameters (dict or ndarray): the parameters for which to simulate the signal. It can either be a matrix with
for every row every model parameter, or a dictionary with for every parameter a 1d array.
Returns:
ndarray: a 2d array with for every parameter combination the simulated model signal
"""
if isinstance(model, str):
model = get_model(model)()
model.set_input_data(MockMRIInputData(protocol=protocol))
build_model = model.build()
if isinstance(parameters, collections.Mapping):
parameters = model.param_dict_to_array(parameters)
nmr_problems = parameters.shape[0]
kernel_data = {
'data':
build_model.get_kernel_data(),
'parameters':
Array(parameters, ctype='mot_float_type'),
'estimates':
Zeros((nmr_problems, build_model.get_nmr_observations()),
'mot_float_type')
}
_get_simulate_function(build_model).evaluate(kernel_data, nmr_problems)
return kernel_data['estimates'].get_data()
def get_init_data(model_name):
inits = {}
free_parameters = get_model(model_name)().get_free_param_names()
if 'S0.s0' in free_parameters and input_data.has_input_data('b'):
if input_data.get_input_data(
'b').shape[0] == input_data.nmr_voxels:
masked_observations = ma.masked_array(
input_data.observations,
input_data.get_input_data('b') < 250e6)
inits['S0.s0'] = restore_volumes(
np.mean(masked_observations, axis=1), input_data.mask)
else:
unweighted_locations = np.where(
input_data.get_input_data('b') < 250e6)[0]
inits['S0.s0'] = np.mean(
input_data.signal4d[..., unweighted_locations], axis=-1)
if model_name.startswith('BallStick_r2'):
inits.update(
get_subset(free_parameters, get_model_fit('BallStick_r1')))
inits['w_stick1.w'] = np.minimum(inits['w_stick0.w'], 0.05)
elif model_name.startswith('BallStick_r3'):
inits.update(
get_subset(free_parameters, get_model_fit('BallStick_r2')))
inits['w_stick2.w'] = np.minimum(inits['w_stick1.w'], 0.05)
elif model_name.startswith('Tensor'):
fit_results = get_model_fit('BallStick_r1')
inits.update(get_subset(free_parameters, fit_results))
inits['Tensor.theta'] = fit_results['Stick0.theta']
inits['Tensor.phi'] = fit_results['Stick0.phi']
elif model_name.startswith('NODDI'):
fit_results = get_model_fit('BallStick_r1')
inits.update(get_subset(free_parameters, fit_results))
inits['w_ic.w'] = fit_results['w_stick0.w'] / 2.0
inits['w_ec.w'] = fit_results['w_stick0.w'] / 2.0
inits['w_csf.w'] = fit_results['w_ball.w']
inits['NODDI_IC.theta'] = fit_results['Stick0.theta']
inits['NODDI_IC.phi'] = fit_results['Stick0.phi']
elif model_name.startswith('BinghamNODDI_r1'):
noddi_results = get_model_fit('NODDI')
inits.update(get_subset(free_parameters, noddi_results))
inits['w_in0.w'] = noddi_results['w_ic.w']
inits['w_en0.w'] = noddi_results['w_ec.w']
inits['w_csf.w'] = noddi_results['w_csf.w']
inits['BinghamNODDI_IN0.theta'] = noddi_results['NODDI_IC.theta']
inits['BinghamNODDI_IN0.phi'] = noddi_results['NODDI_IC.phi']
inits['BinghamNODDI_IN0.k1'] = noddi_results['NODDI_IC.kappa']
elif model_name.startswith('BinghamNODDI_r2'):
bs2_results = get_model_fit('BallStick_r2')
inits.update(get_subset(free_parameters, bs2_results))
inits.update(
get_subset(free_parameters, get_model_fit('BinghamNODDI_r1')))
inits['BinghamNODDI_IN1.theta'] = bs2_results['Stick1.theta']
inits['BinghamNODDI_IN1.phi'] = bs2_results['Stick1.phi']
elif model_name.startswith('Kurtosis'):
fit_results = get_model_fit('Tensor')
inits.update(get_subset(free_parameters, fit_results))
inits.update({
'KurtosisTensor.' + key: fit_results['Tensor.' + key]
for key in ['theta', 'phi', 'psi', 'd', 'dperp0', 'dperp1']
})
elif model_name.startswith('CHARMED_r'):
nmr_dir = model_name[len('CHARMED_r'):len('CHARMED_r') + 1]
fit_results = get_model_fit('BallStick_r' + nmr_dir)
inits.update(get_subset(free_parameters, fit_results))
inits['Tensor.theta'] = fit_results['Stick0.theta']
inits['Tensor.phi'] = fit_results['Stick0.phi']
for dir_ind in range(int(nmr_dir)):
inits['w_res{}.w'.format(dir_ind)] = fit_results[
'w_stick{}.w'.format(dir_ind)]
inits['CHARMEDRestricted{}.theta'.format(
dir_ind)] = fit_results['Stick{}.theta'.format(dir_ind)]
inits['CHARMEDRestricted{}.phi'.format(dir_ind)] = fit_results[
'Stick{}.phi'.format(dir_ind)]
elif model_name.startswith('BallRacket_r'):
nmr_dir = model_name[len('BallRacket_r'):len('BallRacket_r') + 1]
fit_results = get_model_fit('BallStick_r' + nmr_dir)
inits.update(get_subset(free_parameters, fit_results))
for dir_ind in range(int(nmr_dir)):
inits['w_res{}.w'.format(dir_ind)] = fit_results[
'w_stick{}.w'.format(dir_ind)]
inits['Racket{}.theta'.format(dir_ind)] = fit_results[
'Stick{}.theta'.format(dir_ind)]
inits['Racket{}.phi'.format(dir_ind)] = fit_results[
'Stick{}.phi'.format(dir_ind)]
elif model_name.startswith('AxCaliber'):
fit_results = get_model_fit('BallStick_r1')
inits.update(get_subset(free_parameters, fit_results))
inits['GDRCylinders.theta'] = fit_results['Stick0.theta']
inits['GDRCylinders.phi'] = fit_results['Stick0.phi']
elif model_name.startswith('ActiveAx'):
fit_results = get_model_fit('BallStick_r1')
inits.update(get_subset(free_parameters, fit_results))
inits['w_ic.w'] = fit_results['w_stick0.w'] / 2.0
inits['w_ec.w'] = fit_results['w_stick0.w'] / 2.0
inits['w_csf.w'] = fit_results['w_ball.w']
inits['CylinderGPD.theta'] = fit_results['Stick0.theta']
inits['CylinderGPD.phi'] = fit_results['Stick0.phi']
elif model_name.startswith('QMT_ReducedRamani'):
inits['S0.s0'] = np.mean(input_data.signal4d, axis=-1)
return inits
def fit_model(model,
input_data,
output_folder,
method=None,
recalculate=False,
only_recalculate_last=False,
cl_device_ind=None,
double_precision=False,
tmp_results_dir=True,
initialization_data=None,
use_cascaded_inits=True,
post_processing=None,
optimizer_options=None):
"""Run the optimizer on the given model.
Since version 0.17.2 fitting cascade models has been deprecated in favor of a slightly more manual setup by
using the :func:`get_optimization_inits` function.
Args:
model (str or :class:`~mdt.models.composite.DMRICompositeModel` or :class:`~mdt.models.cascade.DMRICascadeModelInterface`):
An implementation of an AbstractModel that contains the model we want to optimize or the name of
an model.
input_data (:class:`~mdt.utils.MRIInputData`): the input data object containing all
the info needed for the model fitting.
output_folder (string): The path to the folder where to place the output, we will make a subdir with the
model name in it.
method (str): The optimization method to use, one of:
- 'Levenberg-Marquardt'
- 'Nelder-Mead'
- 'Powell'
- 'Subplex'
If not given, defaults to 'Powell'.
recalculate (boolean): If we want to recalculate the results if they are already present.
only_recalculate_last (boolean):
This is only of importance when dealing with CascadeModels.
If set to true we only recalculate the last element in the chain (if recalculate is set to True, that is).
If set to false, we recalculate everything. This only holds for the first level of the cascade.
cl_device_ind (int or list): the index of the CL device to use. The index is from the list from the function
utils.get_cl_devices(). This can also be a list of device indices.
double_precision (boolean): if we would like to do the calculations in double precision
tmp_results_dir (str, True or None): The temporary dir for the calculations. Set to a string to use
that path directly, set to True to use the config value, set to None to disable.
initialization_data (dict): provides (extra) initialization data to
use during model fitting. If we are optimizing a cascade model this data only applies to the last model
in the cascade. This dictionary can contain the following elements:
* ``inits``: dictionary with per parameter an initialization point
* ``fixes``: dictionary with per parameter a fixed point, this will remove that parameter from the fitting
* ``lower_bounds``: dictionary with per parameter a lower bound
* ``upper_bounds``: dictionary with per parameter a upper bound
* ``unfix``: a list of parameters to unfix
For example::
initialization_data = {
'fixes': {'Stick0.theta: np.array(...), ...},
'inits': {...}
}
use_cascaded_inits (boolean): if set, we initialize the model parameters using :func:`get_optimization_inits`.
You can still overwrite these initializations using the ``initialization_data`` attribute.
Please note that this only works for non-cascade models.
post_processing (dict): a dictionary with flags for post-processing options to enable or disable.
For valid elements, please see the configuration file settings for ``optimization``
under ``post_processing``. Valid input for this parameter is for example: {'covariance': False}
to disable automatic calculation of the covariance from the Hessian.
optimizer_options (dict): extra options passed to the optimization routines.
Returns:
dict: The result maps for the given composite model or the last model in the cascade.
This returns the results as 3d/4d volumes for every output map.
"""
import mdt.utils
from mdt.lib.model_fitting import ModelFit
if not mdt.utils.check_user_components():
init_user_settings(pass_if_exists=True)
if cl_device_ind is not None and not isinstance(cl_device_ind,
collections.Iterable):
cl_device_ind = [cl_device_ind]
if isinstance(model, str):
model_name = model
model_instance = get_model(model)()
else:
model_name = model.name
model_instance = model
if isinstance(model_instance, DMRICascadeModelInterface):
warnings.warn(
dedent('''
Fitting cascade models has been deprecated in favor of specifying 'initialization_data' directly.
To replicate old fit results, use the function mdt.get_optimization_inits()
Old (deprecated) example:
fit_model('NODDI (Cascade)', ...)
New example:
fit_model('NODDI', ..., use_cascaded_inits=True)
Since ``use_cascaded_inits`` is True by default, you can also just use:
fit_model('NODDI', ...)
'''), FutureWarning)
else:
if use_cascaded_inits:
if initialization_data is None:
initialization_data = {}
initialization_data['inits'] = initialization_data.get('inits', {})
inits = get_optimization_inits(model_name,
input_data,
output_folder,
cl_device_ind=cl_device_ind)
inits.update(initialization_data['inits'])
initialization_data['inits'] = inits
model_fit = ModelFit(model,
input_data,
output_folder,
method=method,
optimizer_options=optimizer_options,
recalculate=recalculate,
only_recalculate_last=only_recalculate_last,
cl_device_ind=cl_device_ind,
double_precision=double_precision,
tmp_results_dir=tmp_results_dir,
initialization_data=initialization_data,
post_processing=post_processing)
return model_fit.run()
def sample_model(model,
input_data,
output_folder,
nmr_samples=None,
burnin=None,
thinning=None,
method=None,
recalculate=False,
cl_device_ind=None,
double_precision=False,
store_samples=True,
sample_items_to_save=None,
tmp_results_dir=True,
initialization_data=None,
post_processing=None,
post_sampling_cb=None,
sampler_options=None):
"""Sample a composite model using Markov Chain Monte Carlo sampling.
Args:
model (:class:`~mdt.models.composite.DMRICompositeModel` or str): the model to sample
input_data (:class:`~mdt.utils.MRIInputData`): the input data object containing all
the info needed for the model fitting.
output_folder (string): The path to the folder where to place the output, we will make a subdir with the
model name in it (for the optimization results) and then a subdir with the samples output.
nmr_samples (int): the number of samples we would like to return.
burnin (int): the number of samples to burn-in, that is, to discard before returning the desired
number of samples
thinning (int): how many sample we wait before storing a new one. This will draw extra samples such that
the total number of samples generated is ``nmr_samples * (thinning)`` and the number of samples stored
is ``nmr_samples``. If set to one or lower we store every sample after the burn in.
method (str): The sampling method to use, one of:
- 'AMWG', for the Adaptive Metropolis-Within-Gibbs
- 'SCAM', for the Single Component Adaptive Metropolis
- 'FSL', for the sampling method used in the FSL toolbox
- 'MWG', for the Metropolis-Within-Gibbs (simple random walk metropolis without updates)
If not given, defaults to 'AMWG'.
recalculate (boolean): If we want to recalculate the results if they are already present.
cl_device_ind (int): the index of the CL device to use. The index is from the list from the function
utils.get_cl_devices().
double_precision (boolean): if we would like to do the calculations in double precision
store_samples (boolean): determines if we store any of the samples. If set to False we will store none
of the samples.
sample_items_to_save (list): list of output names we want to store the samples of. If given, we only
store the items specified in this list. Valid items are the free parameter names of the model and the
items 'LogLikelihood' and 'LogPrior'.
tmp_results_dir (str, True or None): The temporary dir for the calculations. Set to a string to use
that path directly, set to True to use the config value, set to None to disable.
initialization_data (:class:`~mdt.utils.InitializationData` or dict): provides (extra) initialization data to
use during model fitting. If we are optimizing a cascade model this data only applies to the last model
in the cascade. If a dictionary is given we will load the elements as arguments to the
:class:`mdt.utils.SimpleInitializationData` class. For example::
initialization_data = {'fixes': {...}, 'inits': {...}}
is transformed into::
initialization_data = SimpleInitializationData(fixes={...}, inits={...})
post_processing (dict): a dictionary with flags for post-processing options to enable or disable.
For valid elements, please see the configuration file settings for ``sample`` under ``post_processing``.
Valid input for this parameter is for example: {'sample_statistics': True} to enable automatic calculation
of the sample statistics.
post_sampling_cb (Callable[
[mot.sample.base.SamplingOutput, mdt.models.composite.DMRICompositeModel], Optional[Dict]]):
additional post-processing called after sampling. This function can optionally return a (nested)
dictionary with as keys dir-/file-names and as values maps to be stored in the results directory.
sampler_options (dict): specific options for the MCMC routine. These will be provided to the sampling routine
as additional keyword arguments to the constructor.
Returns:
dict: if store_samples is True then we return the samples per parameter as a numpy memmap. If store_samples
is False we return None
"""
import mdt.utils
from mdt.lib.model_sampling import sample_composite_model
from mdt.models.cascade import DMRICascadeModelInterface
import mot.configuration
settings = mdt.configuration.get_general_sampling_settings()
if nmr_samples is None:
nmr_samples = settings['nmr_samples']
if burnin is None:
burnin = settings['burnin']
if thinning is None:
thinning = settings['thinning']
if not isinstance(initialization_data,
InitializationData) and initialization_data is not None:
initialization_data = SimpleInitializationData(**initialization_data)
if not mdt.utils.check_user_components():
init_user_settings(pass_if_exists=True)
if isinstance(model, str):
model = get_model(model)()
if post_processing:
model.update_active_post_processing('sampling', post_processing)
if isinstance(model, DMRICascadeModelInterface):
raise ValueError(
'The function \'sample_model()\' does not accept cascade models.')
if cl_device_ind is None:
cl_context_action = mot.configuration.VoidConfigurationAction()
else:
cl_context_action = mot.configuration.RuntimeConfigurationAction(
cl_environments=get_cl_devices(cl_device_ind),
double_precision=double_precision)
with mot.configuration.config_context(cl_context_action):
base_dir = os.path.join(output_folder, model.name, 'samples')
if not os.path.isdir(base_dir):
os.makedirs(base_dir)
if recalculate:
shutil.rmtree(base_dir)
logger = logging.getLogger(__name__)
logger.info('Using MDT version {}'.format(__version__))
logger.info('Preparing for model {0}'.format(model.name))
logger.info('The {0} parameters we will sample are: {1}'.format(
len(model.get_free_param_names()), model.get_free_param_names()))
return sample_composite_model(
model,
input_data,
base_dir,
nmr_samples,
thinning,
burnin,
get_temporary_results_dir(tmp_results_dir),
method=method,
recalculate=recalculate,
store_samples=store_samples,
sample_items_to_save=sample_items_to_save,
initialization_data=initialization_data,
post_sampling_cb=post_sampling_cb,
sampler_options=sampler_options)
def __init__(self,
model,
input_data,
output_folder,
method=None,
optimizer_options=None,
recalculate=False,
only_recalculate_last=False,
cl_device_ind=None,
double_precision=False,
tmp_results_dir=True,
initialization_data=None,
post_processing=None):
"""Setup model fitting for the given input model and data.
To actually fit the model call run().
Args:
model (str or :class:`~mdt.models.composite.DMRICompositeModel` or :class:`~mdt.models.cascade.DMRICascadeModelInterface`):
the model we want to optimize.
input_data (:class:`~mdt.utils.MRIInputData`): the input data object containing
all the info needed for the model fitting.
output_folder (string): The full path to the folder where to place the output
method (str): The optimization method to use, one of:
- 'Levenberg-Marquardt'
- 'Nelder-Mead'
- 'Powell'
- 'Subplex'
If not given, defaults to 'Powell'.
optimizer_options (dict): extra options passed to the optimization routines.
recalculate (boolean): If we want to recalculate the results if they are already present.
only_recalculate_last (boolean): If we want to recalculate all the models.
This is only of importance when dealing with CascadeModels. If set to true we only recalculate
the last element in the chain (if recalculate is set to True, that is). If set to false,
we recalculate everything. This only holds for the first level of the cascade.
cl_device_ind (int or list): the index of the CL device to use. The index is from the list from the function
get_cl_devices(). This can also be a list of device indices.
double_precision (boolean): if we would like to do the calculations in double precision
tmp_results_dir (str, True or None): The temporary dir for the calculations. Set to a string to use
that path directly, set to True to use the config value, set to None to disable.
initialization_data (dict or :class:`~mdt.utils.InitializationData`): extra initialization data to use
during model fitting. If we are optimizing a cascade model this data only applies to the last model in
the cascade.
post_processing (dict): a dictionary with flags for post-processing options to enable or disable.
For valid elements, please see the configuration file settings for ``optimization``
under ``post_processing``. Valid input for this parameter is for example: {'covariance': False}
to disable automatic calculation of the covariance from the Hessian.
"""
if isinstance(model, str):
model = get_model(model)()
if post_processing:
model.update_active_post_processing('optimization',
post_processing)
self._model = model
self._input_data = input_data
self._output_folder = output_folder
self._method = method
self._optimizer_options = optimizer_options
self._recalculate = recalculate
self._only_recalculate_last = only_recalculate_last
self._logger = logging.getLogger(__name__)
self._model_names_list = []
self._tmp_results_dir = get_temporary_results_dir(tmp_results_dir)
if initialization_data is not None and not isinstance(
initialization_data, InitializationData):
self._initialization_data = SimpleInitializationData(
**initialization_data)
else:
self._initialization_data = initialization_data
if cl_device_ind is not None:
self._cl_runtime_info = CLRuntimeInfo(
cl_environments=get_cl_devices(cl_device_ind),
double_precision=double_precision)
else:
self._cl_runtime_info = CLRuntimeInfo(
double_precision=double_precision)
if not model.is_input_data_sufficient(self._input_data):
raise InsufficientProtocolError(
'The provided protocol is insufficient for this model. '
'The reported errors where: {}'.format(
self._model.get_input_data_problems(self._input_data)))
def __call__(self, subject_info):
logger.info(
'Going to process subject {}, ({} of {}, we are at {:.2%})'.
format(subject_info.subject_id, self._index_counter + 1,
total_nmr_subjects,
self._index_counter / total_nmr_subjects))
self._index_counter += 1
output_dir = os.path.join(output_folder, subject_info.subject_id)
if all(
model_output_exists(model, output_dir)
for model in models_to_fit) and not recalculate:
logger.info('Skipping subject {0}, output exists'.format(
subject_info.subject_id))
return
logger.info(
'Loading the data (DWI, mask and protocol) of subject {0}'.
format(subject_info.subject_id))
input_data = subject_info.get_input_data(use_gradient_deviations)
with timer(subject_info.subject_id):
for model_name in models_to_fit:
if isinstance(model_name, str):
model_instance = get_model(model_name)()
else:
model_instance = model_name
if isinstance(model_instance, DMRICascadeModelInterface):
warnings.warn(
dedent('''
Fitting cascade models has been deprecated, MDT now by default tries to find a suitable initialization point for your specified model.
As an example, instead of specifying 'NODDI (Cascade)', now just specify 'NODDI' and MDT will do its best to get the best model fit possible.
'''), FutureWarning)
logger.info('Going to fit model {0} on subject {1}'.format(
model_name, subject_info.subject_id))
try:
if not isinstance(model_instance,
DMRICascadeModelInterface):
inits = get_optimization_inits(
model_name,
input_data,
output_dir,
cl_device_ind=cl_device_ind)
else:
inits = {}
model_fit = ModelFit(
model_name,
input_data,
output_dir,
recalculate=recalculate,
cl_device_ind=cl_device_ind,
double_precision=double_precision,
tmp_results_dir=tmp_results_dir,
initialization_data={'inits': inits})
model_fit.run()
except InsufficientProtocolError as ex:
logger.info('Could not fit model {0} on subject {1} '
'due to protocol problems. {2}'.format(
model_name, subject_info.subject_id,
ex))
else:
logger.info(
'Done fitting model {0} on subject {1}'.format(
model_name, subject_info.subject_id))
def fit_model(model,
input_data,
output_folder,
method=None,
recalculate=False,
cl_device_ind=None,
double_precision=False,
tmp_results_dir=True,
initialization_data=None,
use_cascaded_inits=True,
post_processing=None,
optimizer_options=None):
"""Run the optimizer on the given model.
Args:
model (str or :class:`~mdt.models.base.EstimableModel`):
The name of a composite model or an implementation of a composite model.
input_data (:class:`~mdt.utils.MRIInputData`): the input data object containing all
the info needed for the model fitting.
output_folder (string): The path to the folder where to place the output, we will make a subdir with the
model name in it.
method (str): The optimization method to use, one of:
- 'Levenberg-Marquardt'
- 'Nelder-Mead'
- 'Powell'
- 'Subplex'
If not given, defaults to 'Powell'.
recalculate (boolean): If we want to recalculate the results if they are already present.
cl_device_ind (int or list): the index of the CL device to use. The index is from the list from the function
utils.get_cl_devices(). This can also be a list of device indices.
double_precision (boolean): if we would like to do the calculations in double precision
tmp_results_dir (str, True or None): The temporary dir for the calculations. Set to a string to use
that path directly, set to True to use the config value, set to None to disable.
initialization_data (dict): provides (extra) initialization data to
use during model fitting. This dictionary can contain the following elements:
* ``inits``: dictionary with per parameter an initialization point
* ``fixes``: dictionary with per parameter a fixed point, this will remove that parameter from the fitting
* ``lower_bounds``: dictionary with per parameter a lower bound
* ``upper_bounds``: dictionary with per parameter a upper bound
* ``unfix``: a list of parameters to unfix
For example::
initialization_data = {
'fixes': {'Stick0.theta: np.array(...), ...},
'inits': {...}
}
use_cascaded_inits (boolean): if set, we initialize the model parameters using :func:`get_optimization_inits`.
You can also overrule the default initializations using the ``initialization_data`` attribute.
post_processing (dict): a dictionary with flags for post-processing options to enable or disable.
For valid elements, please see the configuration file settings for ``optimization``
under ``post_processing``. Valid input for this parameter is for example: {'covariance': False}
to disable automatic calculation of the covariance from the Hessian.
optimizer_options (dict): extra options passed to the optimization routines.
Returns:
dict: The result maps for the given composite model or the last model in the cascade.
This returns the results as 3d/4d volumes for every output map.
"""
logger = logging.getLogger(__name__)
if not check_user_components():
init_user_settings(pass_if_exists=True)
if cl_device_ind is not None:
if not isinstance(cl_device_ind, collections.Iterable):
cl_device_ind = [cl_device_ind]
cl_runtime_info = CLRuntimeInfo(
cl_environments=get_cl_devices(cl_device_ind),
double_precision=double_precision)
else:
cl_runtime_info = CLRuntimeInfo(double_precision=double_precision)
if isinstance(model, str):
model_name = model
model_instance = get_model(model)()
else:
model_name = model.name
model_instance = model
if not model_instance.is_input_data_sufficient(input_data):
raise InsufficientProtocolError(
'The provided protocol is insufficient for this model. '
'The reported errors where: {}'.format(
model_instance.get_input_data_problems(input_data)))
if post_processing:
model_instance.update_active_post_processing('optimization',
post_processing)
if use_cascaded_inits:
if initialization_data is None:
initialization_data = {}
initialization_data['inits'] = initialization_data.get('inits', {})
inits = get_optimization_inits(model_name,
input_data,
output_folder,
cl_device_ind=cl_device_ind)
inits.update(initialization_data['inits'])
initialization_data['inits'] = inits
initialization_data = SimpleInitializationData(**initialization_data)
initialization_data.apply_to_model(model_instance, input_data)
logger.info('Preparing {0} with the cascaded initializations.'.format(
model_name))
if method is None:
method, optimizer_options = get_optimizer_for_model(model_name)
with mot.configuration.config_context(CLRuntimeAction(cl_runtime_info)):
fit_composite_model(model_instance,
input_data,
output_folder,
method,
get_temporary_results_dir(tmp_results_dir),
recalculate=recalculate,
optimizer_options=optimizer_options)
return get_all_nifti_data(os.path.join(output_folder, model_name))
def bootstrap_model(model, input_data, optimization_results, output_folder, bootstrap_method=None,
bootstrap_options=None, nmr_samples=None, optimization_method=None, optimizer_options=None,
recalculate=False, cl_device_ind=None, double_precision=False, keep_samples=True,
tmp_results_dir=True, initialization_data=None):
"""Resample the model using residual bootstrapping.
This is typically used to construct confidence intervals on the optimized parameters.
Args:
model (str or :class:`~mdt.models.base.EstimableModel`): the model to sample
input_data (:class:`~mdt.lib.input_data.MRIInputData`): the input data object containing all
the info needed for the model fitting.
optimization_results (dict or str): the optimization results, either a dictionary with results or the
path to a folder.
output_folder (string): The path to the folder where to place the output, we will make a subdir with the
model name in it (for the optimization results) and then a subdir with the samples output.
bootstrap_method (str): the bootstrap method we want to use, 'residual', or 'wild'. Defaults to 'wild'.
bootstrap_options (dict): bootstrapping options specific for the bootstrap method in use
nmr_samples (int): the number of samples we would like to compute. Defaults to 1000.
optimization_method (str): The optimization method to use, one of:
- 'Levenberg-Marquardt'
- 'Nelder-Mead'
- 'Powell'
- 'Subplex'
If not given, defaults to 'Powell'.
optimizer_options (dict): extra options passed to the optimization routines.
recalculate (boolean): If we want to recalculate the results if they are already present.
cl_device_ind (int): the index of the CL device to use. The index is from the list from the function
utils.get_cl_devices().
double_precision (boolean): if we would like to do the calculations in double precision
keep_samples (boolean): determines if we keep any of the chains. If set to False, the chains will
be discarded after generating the mean and standard deviations.
tmp_results_dir (str, True or None): The temporary dir for the calculations. Set to a string to use
that path directly, set to True to use the config value, set to None to disable.
initialization_data (dict): provides (extra) initialization data to
use during model fitting. This dictionary can contain the following elements:
* ``inits``: dictionary with per parameter an initialization point
* ``fixes``: dictionary with per parameter a fixed point, this will remove that parameter from the fitting
* ``lower_bounds``: dictionary with per parameter a lower bound
* ``upper_bounds``: dictionary with per parameter a upper bound
* ``unfix``: a list of parameters to unfix
For example::
initialization_data = {
'fixes': {'Stick0.theta: np.array(...), ...},
'inits': {...}
}
Returns:
dict: if keep_samples is True we return the samples per parameter as a numpy memmap.
If store_samples is False we return None
"""
initialization_data = initialization_data or {}
nmr_samples = nmr_samples or 1000
bootstrap_method = bootstrap_method or 'wild'
if not check_user_components():
init_user_settings(pass_if_exists=True)
if cl_device_ind is None:
cl_context_action = mot.configuration.VoidConfigurationAction()
else:
cl_context_action = mot.configuration.RuntimeConfigurationAction(
cl_environments=get_cl_devices(cl_device_ind),
double_precision=double_precision)
if isinstance(model, str):
model_name = model
model_instance = get_model(model)()
else:
model_name = model.name
model_instance = model
model_instance.update_active_post_processing('optimization', {'uncertainties': False, 'll_and_ic': False})
initialization_data = SimpleInitializationData(**initialization_data)
initialization_data.apply_to_model(model_instance, input_data)
if optimization_method is None:
optimization_method, optimizer_options = get_optimizer_for_model(model_name)
with mot.configuration.config_context(cl_context_action):
from mdt.lib.processing.model_bootstrapping import compute_bootstrap
return compute_bootstrap(model_instance, input_data, optimization_results,
output_folder, bootstrap_method, optimization_method, nmr_samples,
get_temporary_results_dir(tmp_results_dir),
recalculate=recalculate,
keep_samples=keep_samples,
optimizer_options=optimizer_options,
bootstrap_options=bootstrap_options)
def compute_fim(model, input_data, optimization_results, output_folder=None, cl_device_ind=None, cl_load_balancer=None,
initialization_data=None):
"""Compute the Fisher Information Matrix (FIM).
This is typically done as post-processing step during the model fitting process, but can also be performed
separately after optimization.
Since the FIM depends on which parameters were optimized, results will change if different parameters are fixed.
That is, this function will compute the FIM for every estimable parameter (free-non-fixed parameters). If you want
to have the exact same FIM results as when you computed the FIM as optimization post-processing it is important
to have exactly the same maps fixed.
Contrary to the post-processing of the optimization maps, all FIM results are written to a single sub-folder in the
provided output folder.
Args:
model (str or :class:`~mdt.models.base.EstimableModel`):
The name of a composite model or an implementation of a composite model.
input_data (:class:`~mdt.lib.input_data.MRIInputData`): the input data object containing all
the info needed for the model fitting.
optimization_results (dict or str): the optimization results, either a dictionary with results or the
path to a folder.
output_folder (string): Optionally, the path to the folder where to place the output
cl_device_ind (List[Union[mot.lib.cl_environments.CLEnvironment, int]]
or mot.lib.cl_environments.CLEnvironment or int): the CL devices to use.
Either provide MOT CLEnvironment's or indices from into the list from the function mdt.get_cl_devices().
cl_load_balancer (mot.lib.load_balancers.LoadBalancer or Tuple[float]): the load balancer to use. Can also
be an array of fractions (summing to 1) with one fraction per device. For example, for two devices one
can specify ``cl_load_balancer = [0.3, 0.7]`` to let one device to more work than another.
initialization_data (dict): provides (extra) initialization data to
use during model fitting. This dictionary can contain the following elements:
* ``inits``: dictionary with per parameter an initialization point
* ``fixes``: dictionary with per parameter a fixed point, this will remove that parameter from the fitting
* ``lower_bounds``: dictionary with per parameter a lower bound
* ``upper_bounds``: dictionary with per parameter a upper bound
* ``unfix``: a list of parameters to unfix
For example::
initialization_data = {
'fixes': {'Stick0.theta: np.array(...), ...},
'inits': {...}
}
Returns:
dict: all the computed FIM maps in a flattened dictionary.
"""
initialization_data = initialization_data or {}
if isinstance(optimization_results, str):
optimization_results = get_all_nifti_data(optimization_results)
if not check_user_components():
init_user_settings(pass_if_exists=True)
cl_runtime_info = CLRuntimeInfo(cl_environments=cl_device_ind,
double_precision=True,
load_balancer=cl_load_balancer)
if isinstance(model, str):
model_name = model
model_instance = get_model(model)()
else:
model_name = model.name
model_instance = model
model_instance.set_input_data(input_data)
initialization_data = SimpleInitializationData(**initialization_data)
initialization_data.apply_to_model(model_instance, input_data)
with mot.configuration.config_context(CLRuntimeAction(cl_runtime_info)):
opt_points = create_roi(optimization_results, input_data.mask)
opt_array = combine_dict_to_array(opt_points, model_instance.get_free_param_names())
covars = model_instance.compute_covariance_matrix(opt_array)
covariance_names = model_instance.get_covariance_output_names()
return_results = {}
for ind, name in enumerate(covariance_names):
if name.endswith('.std'):
return_results[name] = np.nan_to_num(np.sqrt(covars[..., ind]))
else:
return_results[name] = covars[..., ind]
return_results = restore_volumes(return_results, input_data.mask)
write_volume_maps(return_results, os.path.join(output_folder, model_name, 'FIM'))
return return_results
def sample_model(model, input_data, output_folder, nmr_samples=None, burnin=None, thinning=None,
method=None, recalculate=False, cl_device_ind=None, cl_load_balancer=None, double_precision=False,
store_samples=True, sample_items_to_save=None, tmp_results_dir=True,
initialization_data=None, post_processing=None, post_sampling_cb=None,
sampler_options=None):
"""Sample a composite model using Markov Chain Monte Carlo sampling.
Args:
model (str or :class:`~mdt.models.base.EstimableModel`): the model to sample
input_data (:class:`~mdt.lib.input_data.MRIInputData`): the input data object containing all
the info needed for the model fitting.
output_folder (string): The path to the folder where to place the output, we will make a subdir with the
model name in it (for the optimization results) and then a subdir with the samples output.
nmr_samples (int): the number of samples we would like to return.
burnin (int): the number of samples to burn-in, that is, to discard before returning the desired
number of samples
thinning (int): how many sample we wait before storing a new one. This will draw extra samples such that
the total number of samples generated is ``nmr_samples * (thinning)`` and the number of samples stored
is ``nmr_samples``. If set to one or lower we store every sample after the burn in.
method (str): The sampling method to use, one of:
- 'AMWG', for the Adaptive Metropolis-Within-Gibbs
- 'SCAM', for the Single Component Adaptive Metropolis
- 'FSL', for the sampling method used in the FSL toolbox
- 'MWG', for the Metropolis-Within-Gibbs (simple random walk metropolis without updates)
If not given, defaults to 'AMWG'.
recalculate (boolean): If we want to recalculate the results if they are already present.
cl_device_ind (List[Union[mot.lib.cl_environments.CLEnvironment, int]]
or mot.lib.cl_environments.CLEnvironment or int): the CL devices to use.
Either provide MOT CLEnvironment's or indices from into the list from the function mdt.get_cl_devices().
cl_load_balancer (mot.lib.load_balancers.LoadBalancer or Tuple[float]): the load balancer to use. Can also
be an array of fractions (summing to 1) with one fraction per device. For example, for two devices one
can specify ``cl_load_balancer = [0.3, 0.7]`` to let one device to more work than another.
double_precision (boolean): if we would like to do the calculations in double precision
store_samples (boolean): determines if we store any of the samples. If set to False we will store none
of the samples.
sample_items_to_save (list): list of output names we want to store the samples of. If given, we only
store the items specified in this list. Valid items are the free parameter names of the model and the
items 'LogLikelihood' and 'LogPrior'.
tmp_results_dir (str, True or None): The temporary dir for the calculations. Set to a string to use
that path directly, set to True to use the config value, set to None to disable.
initialization_data (dict): provides (extra) initialization data to
use during model fitting. This dictionary can contain the following elements:
* ``inits``: dictionary with per parameter an initialization point
* ``fixes``: dictionary with per parameter a fixed point, this will remove that parameter from the fitting
* ``lower_bounds``: dictionary with per parameter a lower bound
* ``upper_bounds``: dictionary with per parameter a upper bound
* ``unfix``: a list of parameters to unfix
For example::
initialization_data = {
'fixes': {'Stick0.theta: np.array(...), ...},
'inits': {...}
}
post_processing (dict): a dictionary with flags for post-processing options to enable or disable.
For valid elements, please see the configuration file settings for ``sample`` under ``post_processing``.
Valid input for this parameter is for example: {'univariate_normal': True} to enable automatic calculation
of the univariate normal distribution for the model parameters.
post_sampling_cb (Callable[
[mot.sample.base.SamplingOutput, mdt.models.composite.DMRICompositeModel], Optional[Dict]]):
additional post-processing called after sampling. This function can optionally return a (nested)
dictionary with as keys dir-/file-names and as values maps to be stored in the results directory.
sampler_options (dict): specific options for the MCMC routine. These will be provided to the sampling routine
as additional keyword arguments to the constructor.
Returns:
dict: if store_samples is True then we return the samples per parameter as a numpy memmap. If store_samples
is False we return None
"""
initialization_data = initialization_data or {}
if not check_user_components():
init_user_settings(pass_if_exists=True)
cl_runtime_info = CLRuntimeInfo(cl_environments=cl_device_ind,
double_precision=double_precision,
load_balancer=cl_load_balancer)
settings = get_general_sampling_settings()
if nmr_samples is None:
nmr_samples = settings['nmr_samples']
if burnin is None:
burnin = settings['burnin']
if thinning is None:
thinning = settings['thinning']
if isinstance(model, str):
model_instance = get_model(model)()
else:
model_instance = model
initialization_data = SimpleInitializationData(**initialization_data)
initialization_data.apply_to_model(model_instance, input_data)
if post_processing:
model_instance.update_active_post_processing('sampling', post_processing)
with mot.configuration.config_context(CLRuntimeAction(cl_runtime_info)):
from mdt.lib.processing.model_sampling import sample_composite_model
return sample_composite_model(model_instance, input_data, output_folder, nmr_samples, thinning, burnin,
get_temporary_results_dir(tmp_results_dir),
method=method, recalculate=recalculate,
store_samples=store_samples,
sample_items_to_save=sample_items_to_save,
post_sampling_cb=post_sampling_cb,
sampler_options=sampler_options)