def run(self):
"""Fits the composite model."""
with per_model_logging_context(self._output_path):
self._model.set_problem_data(self._problem_data)
if self.recalculate:
if os.path.exists(self._output_path):
list(
map(
os.remove,
glob.glob(os.path.join(self._output_path,
'*.nii*'))))
else:
if model_output_exists(self._model, self._output_folder):
maps = get_all_image_data(self._output_path)
self._logger.info('Not recalculating {} model'.format(
self._model.name))
return create_roi(maps, self._problem_data.mask)
if not os.path.exists(self._output_path):
os.makedirs(self._output_path)
with self._logging():
results = self._processing_strategy.run(
self._model, self._problem_data, self._output_path,
self.recalculate,
SimpleModelProcessingWorkerGenerator(
lambda *args: FittingProcessingWorker(
self._optimizer, *args)))
self._write_protocol()
return results
def noise_std(self):
def _compute_noise_std():
if self._noise_std is None:
try:
return estimate_noise_std(self)
except NoiseStdEstimationNotPossible:
self._logger.warning('Failed to obtain a noise std for this subject. '
'We will continue with an std of 1.')
return 1
if isinstance(self._noise_std, (numbers.Number, np.ndarray)):
return self._noise_std
if isinstance(self._noise_std, str):
filename = str(self._noise_std)
if filename[-4:] == '.txt':
with open(filename, 'r') as f:
return float(f.read())
return load_nifti(filename).get_data()
self._logger.warning('Failed to obtain a noise std for this subject. We will continue with an std of 1.')
return 1
self._noise_std = _compute_noise_std()
if is_scalar(self._noise_std):
return self._noise_std
else:
return create_roi(self._noise_std, self.mask)
def combine(self):
super(FittingProcessor, self).combine()
for subdir in self._subdirs:
self._combine_volumes(self._output_dir,
self._tmp_storage_dir,
self._nifti_header,
maps_subdir=subdir)
return create_roi(get_all_nifti_data(self._output_dir), self._mask)
def extra_protocol(self):
return_values = {}
for key, value in self._extra_protocol.items():
value = np.array(value, copy=False)
if len(value.shape) < 3:
return_values[key] = value
return_values[key] = create_roi(value, self.mask)
return return_values
def __init__(self,
optimization_method,
input_data,
optimization_results,
nmr_samples,
model,
mask,
nifti_header,
output_dir,
tmp_storage_dir,
recalculate,
keep_samples=True,
optimizer_options=None):
"""The processing worker for model sample.
Args:
optimization_method: the optimization routine to use
optimization_results (dict): the starting point for the bootstrapping method
nmr_samples (int): the number of samples we would like to return.
"""
super().__init__(mask, nifti_header, output_dir, tmp_storage_dir,
recalculate)
self._logger = logging.getLogger(__name__)
self._optimization_method = optimization_method
self._input_data = input_data.get_subset(
volumes_to_keep=model.get_used_volumes(input_data))
self._roi_input_data = ROIMRIInputData.from_input_data(
self._input_data)
self._optimization_results = optimization_results
self._nmr_samples = nmr_samples
self._model = model
self._write_volumes_gzipped = gzip_sampling_results()
self._keep_samples = keep_samples
self._logger = logging.getLogger(__name__)
self._optimizer_options = optimizer_options
self._sample_storage = None
self._model.set_input_data(input_data)
self._x_opt_array = self._model.param_dict_to_array(
DeferredActionDict(
lambda _, v: create_roi(v, self._input_data.mask),
self._optimization_results))
self._cl_runtime_info = CLRuntimeInfo()
self._codec = self._model.get_mle_codec()
self._lower_bounds, self._upper_bounds = self._codec.encode_bounds(
self._model.get_lower_bounds(), self._model.get_upper_bounds())
self._wrapper = ParameterDecodingWrapper(
self._model.get_nmr_parameters(),
self._codec.get_decode_function())
self._objective_func = self._wrapper.wrap_objective_function(
self._model.get_objective_function())
self._constraints_func = self._wrapper.wrap_constraints_function(
self._model.get_constraints_function())
def fit_composite_model(model, input_data, output_folder, method, tmp_results_dir,
recalculate=False, cascade_names=None, optimizer_options=None):
"""Fits the composite model and returns the results as ROI lists per map.
Args:
model (:class:`~mdt.models.composite.DMRICompositeModel`): An implementation of an composite model
that contains the model we want to optimize.
input_data (:class:`~mdt.utils.MRIInputData`): The input data object for the model.
output_folder (string): The path to the folder where to place the output.
The resulting maps are placed in a subdirectory (named after the model name) in this output folder.
method (str): The optimization routine to use.
tmp_results_dir (str): the main directory to use for the temporary results
recalculate (boolean): If we want to recalculate the results if they are already present.
cascade_names (list): the list of cascade names, meant for logging
optimizer_options (dict): the additional optimization options
"""
logger = logging.getLogger(__name__)
output_path = os.path.join(output_folder, model.name)
if not model.is_input_data_sufficient(input_data):
raise InsufficientProtocolError(
'The given protocol is insufficient for this model. '
'The reported errors where: {}'.format(model.get_input_data_problems(input_data)))
if not recalculate and model_output_exists(model, output_folder):
maps = get_all_nifti_data(output_path)
logger.info('Not recalculating {} model'.format(model.name))
return create_roi(maps, input_data.mask)
with per_model_logging_context(output_path):
logger.info('Using MDT version {}'.format(__version__))
logger.info('Preparing for model {0}'.format(model.name))
logger.info('Current cascade: {0}'.format(cascade_names))
model.set_input_data(input_data)
if recalculate:
if os.path.exists(output_path):
list(map(os.remove, glob.glob(os.path.join(output_path, '*.nii*'))))
if os.path.exists(os.path.join(output_path + 'covariances')):
shutil.rmtree(os.path.join(output_path + 'covariances'))
if not os.path.exists(output_path):
os.makedirs(output_path)
with _model_fit_logging(logger, model.name, model.get_free_param_names()):
tmp_dir = get_full_tmp_results_path(output_path, tmp_results_dir)
logger.info('Saving temporary results in {}.'.format(tmp_dir))
worker = FittingProcessor(method, model, input_data.mask,
input_data.nifti_header, output_path,
tmp_dir, recalculate, optimizer_options=optimizer_options)
processing_strategy = get_processing_strategy('optimization')
return processing_strategy.process(worker)
def observations(self):
if self._observation_list is None:
self._observation_list = create_roi(self.signal4d, self._mask)
signal_max = np.max(self._observation_list)
if signal_max < 10:
logger = logging.getLogger(__name__)
logger.warning(
'Maximum signal intensity is quite low ({}), analysis results are sometimes improved with '
'a higher signal intensity (for example, scale input data by 1e5).'.format(signal_max))
return self._observation_list
def gradient_deviations(self):
if self._gradient_deviations is None:
return None
if self._gradient_deviations_list is None:
grad_dev = create_roi(self._gradient_deviations, self.mask)
if grad_dev.shape[-1] == 9: # HCP WUMINN format, Fortran major. Also adds the identity matrix as specified.
grad_dev = np.reshape(grad_dev, (-1, 3, 3), order='F') + np.eye(3)
self._gradient_deviations_list = grad_dev
return self._gradient_deviations_list
def get_voxels_to_compute(self):
"""By default this will return the indices of all the voxels we have not yet computed.
In the case that recalculate is set to False and we have some intermediate results lying about, this
function will only return the indices of the voxels we have not yet processed.
"""
roi_list = np.arange(0, self._total_nmr_voxels)
processed_voxels_path = os.path.join(self._processing_tmp_dir, 'processed_voxels.npy')
if os.path.exists(processed_voxels_path):
return roi_list[np.logical_not(np.squeeze(create_roi(np.load(processed_voxels_path, mmap_mode='r'),
self._mask)[roi_list]))]
return roi_list
def run(self):
"""Fits the composite model and returns the results as ROI lists per map."""
if not self.recalculate and model_output_exists(
self._model, self._output_folder):
maps = get_all_nifti_data(self._output_path)
self._logger.info('Not recalculating {} model'.format(
self._model.name))
return create_roi(maps, self._input_data.mask)
with per_model_logging_context(self._output_path):
self._logger.info('Using MDT version {}'.format(__version__))
self._logger.info('Preparing for model {0}'.format(
self._model.name))
self._logger.info('Current cascade: {0}'.format(
self._cascade_names))
self._model.set_input_data(self._input_data)
if self.recalculate:
if os.path.exists(self._output_path):
list(
map(
os.remove,
glob.glob(os.path.join(self._output_path,
'*.nii*'))))
if not os.path.exists(self._output_path):
os.makedirs(self._output_path)
with self._logging():
tmp_dir = get_full_tmp_results_path(self._output_path,
self._tmp_results_dir)
self._logger.info(
'Saving temporary results in {}.'.format(tmp_dir))
worker = FittingProcessor(self._optimizer, self._model,
self._input_data.mask,
self._input_data.nifti_header,
self._output_path, tmp_dir,
self.recalculate)
processing_strategy = get_processing_strategy('optimization')
results = processing_strategy.process(worker)
self._write_protocol(self._model.get_input_data().protocol)
return results
def _get_air_voxels(self, problem_data, border_offset=3):
"""Get a two dimensional list with all the voxels in the air.
Returns:
ndarray: The first dimension is the list of voxels, the second the signal per voxel.
"""
mask = np.copy(problem_data.mask)
mask = binary_dilation(mask, iterations=1)
mask[0:border_offset] = True
mask[-border_offset:] = True
mask[:, 0:border_offset, :] = True
mask[:, -border_offset:, :] = True
mask[..., 0:border_offset] = True
mask[..., -border_offset:] = True
return create_roi(problem_data.dwi_volume, np.invert(mask))
def get_input_data(self, parameter_name):
if parameter_name in self._extra_protocol:
value = np.array(self._extra_protocol[parameter_name], copy=False)
if all_elements_equal(value):
return get_single_value(value)
if value.ndim >= 3:
value = create_roi(value, self.mask)
return value
if parameter_name in self._protocol:
if all_elements_equal(self._protocol[parameter_name]):
return get_single_value(self._protocol[parameter_name])
return self._protocol[parameter_name]
raise ValueError('No input data could be find for the parameter "{}".'.format(parameter_name))
def create_signal_estimates(model, input_data, parameters):
"""Create the signals estimates for your estimated model parameters.
This function is typically used to obtain signal estimates from optimization results.
This function evaluates the model as it is in the model fitting and sample. That is, this method includes
the gradient deviations (if set in the input data) and loads all static and fixed parameters maps.
Args:
model (str or model): the model or the name of the model to use for estimating the signals
input_data (mdt.utils.MRIInputData): the input data object, we will set this to the model
parameters (str or dict): either a directory file name or a dictionary containing optimization results
Each element is assumed to be a 4d volume with the voxels we are using for the simulations.
Returns:
ndarray: the 4d array with the signal estimates per voxel
"""
if isinstance(model, str):
model = get_model(model)()
model.set_input_data(input_data)
build_model = model.build()
if isinstance(parameters, str):
parameters = get_all_nifti_data(parameters)
parameters = create_roi(parameters, input_data.mask)
parameters = model.param_dict_to_array(parameters)
kernel_data = {
'data':
build_model.get_kernel_data(),
'parameters':
Array(parameters, ctype='mot_float_type'),
'estimates':
Zeros((parameters.shape[0], build_model.get_nmr_observations()),
'mot_float_type')
}
_get_simulate_function(build_model).evaluate(kernel_data,
parameters.shape[0])
results = kernel_data['estimates'].get_data()
return restore_volumes(results, input_data.mask)
def get_voxels_to_compute(self):
"""Get the ROI indices of the voxels we need to compute.
This should either return an entire list with all the ROI indices for the given brain mask, or a list
with the specific roi indices we want the strategy to compute.
Returns:
ndarray: the list of ROI indices (indexing the current mask) with the voxels we want to compute.
"""
if self._honor_voxels_to_analyze and self._model.problems_to_analyze:
roi_list = self._model.problems_to_analyze
else:
roi_list = np.arange(0, np.count_nonzero(self._problem_data.mask))
mask_path = os.path.join(self._tmp_storage_dir, '{}.npy'.format(self._used_mask_name))
if os.path.exists(mask_path):
return roi_list[np.logical_not(np.squeeze(create_roi(np.load(mask_path, mmap_mode='r'),
self._problem_data.mask)[roi_list]))]
return roi_list
def _preload_extra_protocol_items(self, extra_protocol):
"""Load all the extra protocol items that were defined by a filename."""
if extra_protocol is None:
return {}
return_items = {}
for key, value in extra_protocol.items():
if isinstance(value, str):
if value.endswith('.nii') or value.endswith('.nii.gz'):
loaded_val = load_nifti(value).get_data()
else:
loaded_val = np.genfromtxt(value)
else:
loaded_val = value
if len(loaded_val.shape) > 3:
loaded_val = create_roi(loaded_val, self._mask)
return_items[key] = loaded_val
return return_items
def create_signal_estimates(model, input_data, parameters):
"""Create the signals estimates for your estimated model parameters.
This function is typically used to obtain signal estimates from optimization results.
This function evaluates the model as it is in the model fitting and sampling. That is, this method includes
the gradient deviations (if set in the input data) and loads all static and fixed parameters maps.
Args:
model (str or model): the model or the name of the model to use for estimating the signals
input_data (mdt.utils.MRIInputData): the input data object, we will set this to the model
parameters (str or dict): either a directory file name or a dictionary containing optimization results
Each element is assumed to be a 4d volume with the voxels we are using for the simulations.
Returns:
ndarray: the 4d array with the signal estimates per voxel
"""
if isinstance(model, string_types):
model = get_model(model)()
model.set_input_data(input_data)
if isinstance(parameters, string_types):
parameters = get_all_nifti_data(parameters)
parameters = create_roi(parameters, input_data.mask)
parameters = model.param_dict_to_array(parameters)
build_model = model.build()
if parameters.shape[0] != build_model.get_nmr_problems():
raise ValueError(
'The number of voxels in the parameters does not match those in the model.'
)
calculator = CalculateModelEstimates()
results = calculator.calculate(model.build(), parameters)
return restore_volumes(results, input_data.mask)
def _get_variable_data(self):
var_data_dict = super(DMRICompositeModel, self)._get_variable_data()
if self._problem_data.gradient_deviations is not None:
if len(self._problem_data.gradient_deviations.shape) > 2:
grad_dev = create_roi(self._problem_data.gradient_deviations, self._problem_data.mask)
else:
grad_dev = np.copy(self._problem_data.gradient_deviations)
self._logger.info('Using the gradient deviations in the model optimization.')
# adds the eye(3) matrix to every grad dev, so we don't have to do it in the kernel.
# Flattening an eye(3) matrix gives the same result with F and C ordering, I nevertheless put the ordering
# here to emphasize that the gradient deviations matrix is in Fortran (column-major) order.
grad_dev += np.eye(3).flatten(order='F')
if self.problems_to_analyze is not None:
grad_dev = grad_dev[self.problems_to_analyze, ...]
adapter = SimpleDataAdapter(grad_dev, CLDataType.from_string('mot_float_type*'), self._get_mot_float_type())
var_data_dict.update({'gradient_deviations': adapter})
return var_data_dict
def _get_air_voxels(self, problem_data, border_offset=3):
"""Get a two dimensional list with all the voxels in the air.
Returns:
ndarray: The first dimension is the list of voxels, the second the signal per voxel.
"""
indices = np.where(problem_data.mask > 0)
max_dims = np.max(indices, axis=1)
min_dims = np.min(indices, axis=1)
mask = np.copy(problem_data.mask)
mask[min_dims[0]:max_dims[0]] = True
mask[:, min_dims[1]:max_dims[1], :] = True
mask[..., min_dims[2]:max_dims[2]] = True
mask[0:border_offset] = True
mask[-border_offset:] = True
mask[:, 0:border_offset, :] = True
mask[:, -border_offset:, :] = True
mask[..., 0:border_offset] = True
mask[..., -border_offset:] = True
return create_roi(problem_data.dwi_volume, np.invert(mask))
def compute_fim(model, input_data, optimization_results, output_folder=None, cl_device_ind=None, cl_load_balancer=None,
initialization_data=None):
"""Compute the Fisher Information Matrix (FIM).
This is typically done as post-processing step during the model fitting process, but can also be performed
separately after optimization.
Since the FIM depends on which parameters were optimized, results will change if different parameters are fixed.
That is, this function will compute the FIM for every estimable parameter (free-non-fixed parameters). If you want
to have the exact same FIM results as when you computed the FIM as optimization post-processing it is important
to have exactly the same maps fixed.
Contrary to the post-processing of the optimization maps, all FIM results are written to a single sub-folder in the
provided output folder.
Args:
model (str or :class:`~mdt.models.base.EstimableModel`):
The name of a composite model or an implementation of a composite model.
input_data (:class:`~mdt.lib.input_data.MRIInputData`): the input data object containing all
the info needed for the model fitting.
optimization_results (dict or str): the optimization results, either a dictionary with results or the
path to a folder.
output_folder (string): Optionally, the path to the folder where to place the output
cl_device_ind (List[Union[mot.lib.cl_environments.CLEnvironment, int]]
or mot.lib.cl_environments.CLEnvironment or int): the CL devices to use.
Either provide MOT CLEnvironment's or indices from into the list from the function mdt.get_cl_devices().
cl_load_balancer (mot.lib.load_balancers.LoadBalancer or Tuple[float]): the load balancer to use. Can also
be an array of fractions (summing to 1) with one fraction per device. For example, for two devices one
can specify ``cl_load_balancer = [0.3, 0.7]`` to let one device to more work than another.
initialization_data (dict): provides (extra) initialization data to
use during model fitting. This dictionary can contain the following elements:
* ``inits``: dictionary with per parameter an initialization point
* ``fixes``: dictionary with per parameter a fixed point, this will remove that parameter from the fitting
* ``lower_bounds``: dictionary with per parameter a lower bound
* ``upper_bounds``: dictionary with per parameter a upper bound
* ``unfix``: a list of parameters to unfix
For example::
initialization_data = {
'fixes': {'Stick0.theta: np.array(...), ...},
'inits': {...}
}
Returns:
dict: all the computed FIM maps in a flattened dictionary.
"""
initialization_data = initialization_data or {}
if isinstance(optimization_results, str):
optimization_results = get_all_nifti_data(optimization_results)
if not check_user_components():
init_user_settings(pass_if_exists=True)
cl_runtime_info = CLRuntimeInfo(cl_environments=cl_device_ind,
double_precision=True,
load_balancer=cl_load_balancer)
if isinstance(model, str):
model_name = model
model_instance = get_model(model)()
else:
model_name = model.name
model_instance = model
model_instance.set_input_data(input_data)
initialization_data = SimpleInitializationData(**initialization_data)
initialization_data.apply_to_model(model_instance, input_data)
with mot.configuration.config_context(CLRuntimeAction(cl_runtime_info)):
opt_points = create_roi(optimization_results, input_data.mask)
opt_array = combine_dict_to_array(opt_points, model_instance.get_free_param_names())
covars = model_instance.compute_covariance_matrix(opt_array)
covariance_names = model_instance.get_covariance_output_names()
return_results = {}
for ind, name in enumerate(covariance_names):
if name.endswith('.std'):
return_results[name] = np.nan_to_num(np.sqrt(covars[..., ind]))
else:
return_results[name] = covars[..., ind]
return_results = restore_volumes(return_results, input_data.mask)
write_volume_maps(return_results, os.path.join(output_folder, model_name, 'FIM'))
return return_results
def volume_weights(self):
if self._volume_weights is None:
return None
if self._volume_weights_list is None:
self._volume_weights_list = create_roi(self._volume_weights, self.mask).astype(np.float16)
return self._volume_weights_list
def combine(self):
super(FittingProcessingWorker, self).combine()
self._combine_volumes(self._output_dir, self._tmp_storage_dir, self._problem_data.volume_header)
return create_roi(get_all_image_data(self._output_dir), self._problem_data.mask)
def sample_model(model,
problem_data,
output_folder,
sampler=None,
recalculate=False,
cl_device_ind=None,
double_precision=False,
store_samples=True,
tmp_results_dir=True,
save_user_script_info=True,
initialization_maps=None):
"""Sample a composite model using the given cascading strategy.
Args:
model (:class:`~mdt.models.composite.DMRICompositeModel` or str): the model to sample
problem_data (:class:`~mdt.utils.DMRIProblemData`): the problem data object
output_folder (string): The path to the folder where to place the output, we will make a subdir with the
model name in it (for the optimization results) and then a subdir with the samples output.
sampler (:class:`mot.cl_routines.sampling.base.AbstractSampler`): the sampler to use
recalculate (boolean): If we want to recalculate the results if they are already present.
cl_device_ind (int): the index of the CL device to use. The index is from the list from the function
utils.get_cl_devices().
double_precision (boolean): if we would like to do the calculations in double precision
store_samples (boolean): if set to False we will store none of the samples. Use this
if you are only interested in the volume maps and not in the entire sample chain.
tmp_results_dir (str, True or None): The temporary dir for the calculations. Set to a string to use
that path directly, set to True to use the config value, set to None to disable.
save_user_script_info (boolean, str or SaveUserScriptInfo): The info we need to save about the script the
user is currently executing. If True (default) we use the stack to lookup the script the user is executing
and save that using a SaveFromScript saver. If a string is given we use that filename again for the
SaveFromScript saver. If False or None, we do not write any information. If a SaveUserScriptInfo is
given we use that directly.
initialization_maps (dict): 4d maps to initialize the sampling with. Per default this is None,
common practice is to use the maps from an optimization as starting point
Returns:
dict: the samples per parameter as a numpy memmap if store_samples is True
"""
import mdt.utils
from mot.load_balance_strategies import EvenDistribution
from mdt.model_sampling import sample_composite_model
from mdt.models.cascade import DMRICascadeModelInterface
import mot.configuration
if not mdt.utils.check_user_components():
init_user_settings(pass_if_exists=True)
if isinstance(model, string_types):
model = get_model(model)
if isinstance(model, DMRICascadeModelInterface):
raise ValueError(
'The function \'sample_model()\' does not accept cascade models.')
if not model.is_protocol_sufficient(problem_data.protocol):
raise InsufficientProtocolError(
'The given protocol is insufficient for this model. '
'The reported errors where: {}'.format(
model.get_protocol_problems(problem_data.protocol)))
if cl_device_ind is not None and not isinstance(cl_device_ind,
collections.Iterable):
cl_device_ind = [cl_device_ind]
if cl_device_ind is None:
cl_context_action = mot.configuration.VoidConfigurationAction()
else:
cl_context_action = mot.configuration.RuntimeConfigurationAction(
cl_environments=[get_cl_devices()[ind] for ind in cl_device_ind],
load_balancer=EvenDistribution())
with mot.configuration.config_context(cl_context_action):
if sampler is None:
sampler = configuration.get_sampler()
processing_strategy = get_processing_strategy('sampling',
model_names=model.name)
processing_strategy.set_tmp_dir(
get_temporary_results_dir(tmp_results_dir))
output_folder = os.path.join(output_folder, model.name, 'samples')
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
with per_model_logging_context(output_folder, overwrite=recalculate):
logger = logging.getLogger(__name__)
logger.info('Using MDT version {}'.format(__version__))
logger.info('Preparing for model {0}'.format(model.name))
if initialization_maps:
model.set_initial_parameters(
create_roi(initialization_maps, problem_data.mask))
model.double_precision = double_precision
results = sample_composite_model(model,
problem_data,
output_folder,
sampler,
processing_strategy,
recalculate=recalculate,
store_samples=store_samples)
easy_save_user_script_info(save_user_script_info,
output_folder + '/used_scripts.py',
stack()[1][0].f_globals.get('__file__'))
return results
