def _run_composite_model(self,
model,
recalculate,
model_names,
apply_user_provided_initialization=False):
with mot.configuration.config_context(
CLRuntimeAction(self._cl_runtime_info)):
if apply_user_provided_initialization:
self._apply_user_provided_initialization_data(model)
method = self._method or get_optimizer_for_model(model_names)
results = fit_composite_model(
model,
self._input_data,
self._output_folder,
method,
self._tmp_results_dir,
recalculate=recalculate,
cascade_names=model_names,
optimizer_options=self._optimizer_options)
map_results = get_all_nifti_data(
os.path.join(self._output_folder, model.name))
return results, map_results
def combine(self):
super().combine()
for subdir in self._subdirs:
self._combine_volumes(self._output_dir,
self._tmp_storage_dir,
self._nifti_header,
maps_subdir=subdir)
return create_roi(get_all_nifti_data(self._output_dir), self._mask)
def sort_orientations(data_input, weight_names, extra_sortable_maps):
"""Sort the orientations of multi-direction models voxel-wise.
This expects as input 3d/4d volumes. Do not use this with 2d arrays.
This can be used to sort, for example, simulations of the BallStick_r3 model (with three Sticks).
There is no voxel-wise order over Sticks since for the model they are all equal compartments.
However, when using optimization or ARD with sample, there is order within the compartments since the ARD is
commonly placed on the second and third Sticks meaning these Sticks and there corresponding orientations are
compressed to zero if they are not supported. In that case, the Stick with the primary orientation of diffusion
has to be the first.
This method accepts as input results from (MDT) model fitting and is able to sort all the maps belonging to
a given set of equal compartments per voxel.
Example::
sort_orientations('./output/BallStick_r3',
['w_stick0.w', 'w_stick1.w', 'w_stick2.w'],
[['Stick0.theta', 'Stick1.theta', 'Stick2.theta'],
['Stick0.phi', 'Stick1.phi', 'Stick2.phi'], ...])
Args:
data_input (str or dict): either a directory or a dictionary containing the maps
weight_names (iterable of str): The names of the maps we use for sorting all other maps. These will be sorted
as well.
extra_sortable_maps (iterable of iterable): the list of additional maps to sort. Every element in the given
list should be another list with the names of the maps. The length of these second layer of lists should
match the length of the ``weight_names``.
Returns:
dict: the sorted results in a new dictionary. This returns all input maps with some of them sorted.
"""
if isinstance(data_input, str):
input_maps = get_all_nifti_data(data_input)
result_maps = input_maps
else:
input_maps = data_input
result_maps = copy(input_maps)
weight_names = list(weight_names)
sortable_maps = copy(extra_sortable_maps)
sortable_maps.append(weight_names)
sort_index_matrix = create_sort_matrix(
[input_maps[k] for k in weight_names], reversed_sort=True)
for sortable_map_names in sortable_maps:
sorted = dict(
zip(
sortable_map_names,
sort_volumes_per_voxel(
[input_maps[k] for k in sortable_map_names],
sort_index_matrix)))
result_maps.update(sorted)
return result_maps
def fit_composite_model(model, input_data, output_folder, method, tmp_results_dir,
recalculate=False, cascade_names=None, optimizer_options=None):
"""Fits the composite model and returns the results as ROI lists per map.
Args:
model (:class:`~mdt.models.composite.DMRICompositeModel`): An implementation of an composite model
that contains the model we want to optimize.
input_data (:class:`~mdt.utils.MRIInputData`): The input data object for the model.
output_folder (string): The path to the folder where to place the output.
The resulting maps are placed in a subdirectory (named after the model name) in this output folder.
method (str): The optimization routine to use.
tmp_results_dir (str): the main directory to use for the temporary results
recalculate (boolean): If we want to recalculate the results if they are already present.
cascade_names (list): the list of cascade names, meant for logging
optimizer_options (dict): the additional optimization options
"""
logger = logging.getLogger(__name__)
output_path = os.path.join(output_folder, model.name)
if not model.is_input_data_sufficient(input_data):
raise InsufficientProtocolError(
'The given protocol is insufficient for this model. '
'The reported errors where: {}'.format(model.get_input_data_problems(input_data)))
if not recalculate and model_output_exists(model, output_folder):
maps = get_all_nifti_data(output_path)
logger.info('Not recalculating {} model'.format(model.name))
return create_roi(maps, input_data.mask)
with per_model_logging_context(output_path):
logger.info('Using MDT version {}'.format(__version__))
logger.info('Preparing for model {0}'.format(model.name))
logger.info('Current cascade: {0}'.format(cascade_names))
model.set_input_data(input_data)
if recalculate:
if os.path.exists(output_path):
list(map(os.remove, glob.glob(os.path.join(output_path, '*.nii*'))))
if os.path.exists(os.path.join(output_path + 'covariances')):
shutil.rmtree(os.path.join(output_path + 'covariances'))
if not os.path.exists(output_path):
os.makedirs(output_path)
with _model_fit_logging(logger, model.name, model.get_free_param_names()):
tmp_dir = get_full_tmp_results_path(output_path, tmp_results_dir)
logger.info('Saving temporary results in {}.'.format(tmp_dir))
worker = FittingProcessor(method, model, input_data.mask,
input_data.nifti_header, output_path,
tmp_dir, recalculate, optimizer_options=optimizer_options)
processing_strategy = get_processing_strategy('optimization')
return processing_strategy.process(worker)
def create_signal_estimates(model, input_data, parameters):
"""Create the signals estimates for your estimated model parameters.
This function is typically used to obtain signal estimates from optimization results.
This function evaluates the model as it is in the model fitting and sample. That is, this method includes
the gradient deviations (if set in the input data) and loads all static and fixed parameters maps.
Args:
model (str or model): the model or the name of the model to use for estimating the signals
input_data (mdt.utils.MRIInputData): the input data object, we will set this to the model
parameters (str or dict): either a directory file name or a dictionary containing optimization results
Each element is assumed to be a 4d volume with the voxels we are using for the simulations.
Returns:
ndarray: the 4d array with the signal estimates per voxel
"""
if isinstance(model, str):
model = get_model(model)()
model.set_input_data(input_data)
build_model = model.build()
if isinstance(parameters, str):
parameters = get_all_nifti_data(parameters)
parameters = create_roi(parameters, input_data.mask)
parameters = model.param_dict_to_array(parameters)
kernel_data = {
'data':
build_model.get_kernel_data(),
'parameters':
Array(parameters, ctype='mot_float_type'),
'estimates':
Zeros((parameters.shape[0], build_model.get_nmr_observations()),
'mot_float_type')
}
_get_simulate_function(build_model).evaluate(kernel_data,
parameters.shape[0])
results = kernel_data['estimates'].get_data()
return restore_volumes(results, input_data.mask)
def fit_model(model,
input_data,
output_folder,
method=None,
recalculate=False,
cl_device_ind=None,
double_precision=False,
tmp_results_dir=True,
initialization_data=None,
use_cascaded_inits=True,
post_processing=None,
optimizer_options=None):
"""Run the optimizer on the given model.
Args:
model (str or :class:`~mdt.models.base.EstimableModel`):
The name of a composite model or an implementation of a composite model.
input_data (:class:`~mdt.utils.MRIInputData`): the input data object containing all
the info needed for the model fitting.
output_folder (string): The path to the folder where to place the output, we will make a subdir with the
model name in it.
method (str): The optimization method to use, one of:
- 'Levenberg-Marquardt'
- 'Nelder-Mead'
- 'Powell'
- 'Subplex'
If not given, defaults to 'Powell'.
recalculate (boolean): If we want to recalculate the results if they are already present.
cl_device_ind (int or list): the index of the CL device to use. The index is from the list from the function
utils.get_cl_devices(). This can also be a list of device indices.
double_precision (boolean): if we would like to do the calculations in double precision
tmp_results_dir (str, True or None): The temporary dir for the calculations. Set to a string to use
that path directly, set to True to use the config value, set to None to disable.
initialization_data (dict): provides (extra) initialization data to
use during model fitting. This dictionary can contain the following elements:
* ``inits``: dictionary with per parameter an initialization point
* ``fixes``: dictionary with per parameter a fixed point, this will remove that parameter from the fitting
* ``lower_bounds``: dictionary with per parameter a lower bound
* ``upper_bounds``: dictionary with per parameter a upper bound
* ``unfix``: a list of parameters to unfix
For example::
initialization_data = {
'fixes': {'Stick0.theta: np.array(...), ...},
'inits': {...}
}
use_cascaded_inits (boolean): if set, we initialize the model parameters using :func:`get_optimization_inits`.
You can also overrule the default initializations using the ``initialization_data`` attribute.
post_processing (dict): a dictionary with flags for post-processing options to enable or disable.
For valid elements, please see the configuration file settings for ``optimization``
under ``post_processing``. Valid input for this parameter is for example: {'covariance': False}
to disable automatic calculation of the covariance from the Hessian.
optimizer_options (dict): extra options passed to the optimization routines.
Returns:
dict: The result maps for the given composite model or the last model in the cascade.
This returns the results as 3d/4d volumes for every output map.
"""
logger = logging.getLogger(__name__)
if not check_user_components():
init_user_settings(pass_if_exists=True)
if cl_device_ind is not None:
if not isinstance(cl_device_ind, collections.Iterable):
cl_device_ind = [cl_device_ind]
cl_runtime_info = CLRuntimeInfo(
cl_environments=get_cl_devices(cl_device_ind),
double_precision=double_precision)
else:
cl_runtime_info = CLRuntimeInfo(double_precision=double_precision)
if isinstance(model, str):
model_name = model
model_instance = get_model(model)()
else:
model_name = model.name
model_instance = model
if not model_instance.is_input_data_sufficient(input_data):
raise InsufficientProtocolError(
'The provided protocol is insufficient for this model. '
'The reported errors where: {}'.format(
model_instance.get_input_data_problems(input_data)))
if post_processing:
model_instance.update_active_post_processing('optimization',
post_processing)
if use_cascaded_inits:
if initialization_data is None:
initialization_data = {}
initialization_data['inits'] = initialization_data.get('inits', {})
inits = get_optimization_inits(model_name,
input_data,
output_folder,
cl_device_ind=cl_device_ind)
inits.update(initialization_data['inits'])
initialization_data['inits'] = inits
initialization_data = SimpleInitializationData(**initialization_data)
initialization_data.apply_to_model(model_instance, input_data)
logger.info('Preparing {0} with the cascaded initializations.'.format(
model_name))
if method is None:
method, optimizer_options = get_optimizer_for_model(model_name)
with mot.configuration.config_context(CLRuntimeAction(cl_runtime_info)):
fit_composite_model(model_instance,
input_data,
output_folder,
method,
get_temporary_results_dir(tmp_results_dir),
recalculate=recalculate,
optimizer_options=optimizer_options)
return get_all_nifti_data(os.path.join(output_folder, model_name))
def compute_fim(model, input_data, optimization_results, output_folder=None, cl_device_ind=None, cl_load_balancer=None,
initialization_data=None):
"""Compute the Fisher Information Matrix (FIM).
This is typically done as post-processing step during the model fitting process, but can also be performed
separately after optimization.
Since the FIM depends on which parameters were optimized, results will change if different parameters are fixed.
That is, this function will compute the FIM for every estimable parameter (free-non-fixed parameters). If you want
to have the exact same FIM results as when you computed the FIM as optimization post-processing it is important
to have exactly the same maps fixed.
Contrary to the post-processing of the optimization maps, all FIM results are written to a single sub-folder in the
provided output folder.
Args:
model (str or :class:`~mdt.models.base.EstimableModel`):
The name of a composite model or an implementation of a composite model.
input_data (:class:`~mdt.lib.input_data.MRIInputData`): the input data object containing all
the info needed for the model fitting.
optimization_results (dict or str): the optimization results, either a dictionary with results or the
path to a folder.
output_folder (string): Optionally, the path to the folder where to place the output
cl_device_ind (List[Union[mot.lib.cl_environments.CLEnvironment, int]]
or mot.lib.cl_environments.CLEnvironment or int): the CL devices to use.
Either provide MOT CLEnvironment's or indices from into the list from the function mdt.get_cl_devices().
cl_load_balancer (mot.lib.load_balancers.LoadBalancer or Tuple[float]): the load balancer to use. Can also
be an array of fractions (summing to 1) with one fraction per device. For example, for two devices one
can specify ``cl_load_balancer = [0.3, 0.7]`` to let one device to more work than another.
initialization_data (dict): provides (extra) initialization data to
use during model fitting. This dictionary can contain the following elements:
* ``inits``: dictionary with per parameter an initialization point
* ``fixes``: dictionary with per parameter a fixed point, this will remove that parameter from the fitting
* ``lower_bounds``: dictionary with per parameter a lower bound
* ``upper_bounds``: dictionary with per parameter a upper bound
* ``unfix``: a list of parameters to unfix
For example::
initialization_data = {
'fixes': {'Stick0.theta: np.array(...), ...},
'inits': {...}
}
Returns:
dict: all the computed FIM maps in a flattened dictionary.
"""
initialization_data = initialization_data or {}
if isinstance(optimization_results, str):
optimization_results = get_all_nifti_data(optimization_results)
if not check_user_components():
init_user_settings(pass_if_exists=True)
cl_runtime_info = CLRuntimeInfo(cl_environments=cl_device_ind,
double_precision=True,
load_balancer=cl_load_balancer)
if isinstance(model, str):
model_name = model
model_instance = get_model(model)()
else:
model_name = model.name
model_instance = model
model_instance.set_input_data(input_data)
initialization_data = SimpleInitializationData(**initialization_data)
initialization_data.apply_to_model(model_instance, input_data)
with mot.configuration.config_context(CLRuntimeAction(cl_runtime_info)):
opt_points = create_roi(optimization_results, input_data.mask)
opt_array = combine_dict_to_array(opt_points, model_instance.get_free_param_names())
covars = model_instance.compute_covariance_matrix(opt_array)
covariance_names = model_instance.get_covariance_output_names()
return_results = {}
for ind, name in enumerate(covariance_names):
if name.endswith('.std'):
return_results[name] = np.nan_to_num(np.sqrt(covars[..., ind]))
else:
return_results[name] = covars[..., ind]
return_results = restore_volumes(return_results, input_data.mask)
write_volume_maps(return_results, os.path.join(output_folder, model_name, 'FIM'))
return return_results