def run(self):
"""Fits the composite model."""
with per_model_logging_context(self._output_path):
self._model.set_problem_data(self._problem_data)
if self.recalculate:
if os.path.exists(self._output_path):
list(
map(
os.remove,
glob.glob(os.path.join(self._output_path,
'*.nii*'))))
else:
if model_output_exists(self._model, self._output_folder):
maps = get_all_image_data(self._output_path)
self._logger.info('Not recalculating {} model'.format(
self._model.name))
return create_roi(maps, self._problem_data.mask)
if not os.path.exists(self._output_path):
os.makedirs(self._output_path)
with self._logging():
results = self._processing_strategy.run(
self._model, self._problem_data, self._output_path,
self.recalculate,
SimpleModelProcessingWorkerGenerator(
lambda *args: FittingProcessingWorker(
self._optimizer, *args)))
self._write_protocol()
return results
def _run_composite_model(self, model, recalculate, model_names):
with mot.configuration.config_context(
RuntimeConfigurationAction(cl_environments=self._cl_envs,
load_balancer=self._load_balancer)):
with per_model_logging_context(
os.path.join(self._output_folder, model.name)):
self._logger.info('Using MDT version {}'.format(__version__))
self._logger.info('Preparing for model {0}'.format(model.name))
self._logger.info('Current cascade: {0}'.format(model_names))
optimizer = self._optimizer or get_optimizer_for_model(
model_names)
if self._cl_device_indices is not None:
all_devices = get_cl_devices()
optimizer.cl_environments = [
all_devices[ind] for ind in self._cl_device_indices
]
optimizer.load_balancer = EvenDistribution()
processing_strategy = get_processing_strategy(
'optimization', model_names=model_names)
processing_strategy.set_tmp_dir(self._tmp_results_dir)
fitter = SingleModelFit(model,
self._problem_data,
self._output_folder,
optimizer,
processing_strategy,
recalculate=recalculate)
results = fitter.run()
return results
def fit_composite_model(model, input_data, output_folder, method, tmp_results_dir,
recalculate=False, cascade_names=None, optimizer_options=None):
"""Fits the composite model and returns the results as ROI lists per map.
Args:
model (:class:`~mdt.models.composite.DMRICompositeModel`): An implementation of an composite model
that contains the model we want to optimize.
input_data (:class:`~mdt.utils.MRIInputData`): The input data object for the model.
output_folder (string): The path to the folder where to place the output.
The resulting maps are placed in a subdirectory (named after the model name) in this output folder.
method (str): The optimization routine to use.
tmp_results_dir (str): the main directory to use for the temporary results
recalculate (boolean): If we want to recalculate the results if they are already present.
cascade_names (list): the list of cascade names, meant for logging
optimizer_options (dict): the additional optimization options
"""
logger = logging.getLogger(__name__)
output_path = os.path.join(output_folder, model.name)
if not model.is_input_data_sufficient(input_data):
raise InsufficientProtocolError(
'The given protocol is insufficient for this model. '
'The reported errors where: {}'.format(model.get_input_data_problems(input_data)))
if not recalculate and model_output_exists(model, output_folder):
maps = get_all_nifti_data(output_path)
logger.info('Not recalculating {} model'.format(model.name))
return create_roi(maps, input_data.mask)
with per_model_logging_context(output_path):
logger.info('Using MDT version {}'.format(__version__))
logger.info('Preparing for model {0}'.format(model.name))
logger.info('Current cascade: {0}'.format(cascade_names))
model.set_input_data(input_data)
if recalculate:
if os.path.exists(output_path):
list(map(os.remove, glob.glob(os.path.join(output_path, '*.nii*'))))
if os.path.exists(os.path.join(output_path + 'covariances')):
shutil.rmtree(os.path.join(output_path + 'covariances'))
if not os.path.exists(output_path):
os.makedirs(output_path)
with _model_fit_logging(logger, model.name, model.get_free_param_names()):
tmp_dir = get_full_tmp_results_path(output_path, tmp_results_dir)
logger.info('Saving temporary results in {}.'.format(tmp_dir))
worker = FittingProcessor(method, model, input_data.mask,
input_data.nifti_header, output_path,
tmp_dir, recalculate, optimizer_options=optimizer_options)
processing_strategy = get_processing_strategy('optimization')
return processing_strategy.process(worker)
def run(self):
"""Fits the composite model and returns the results as ROI lists per map."""
if not self.recalculate and model_output_exists(
self._model, self._output_folder):
maps = get_all_nifti_data(self._output_path)
self._logger.info('Not recalculating {} model'.format(
self._model.name))
return create_roi(maps, self._input_data.mask)
with per_model_logging_context(self._output_path):
self._logger.info('Using MDT version {}'.format(__version__))
self._logger.info('Preparing for model {0}'.format(
self._model.name))
self._logger.info('Current cascade: {0}'.format(
self._cascade_names))
self._model.set_input_data(self._input_data)
if self.recalculate:
if os.path.exists(self._output_path):
list(
map(
os.remove,
glob.glob(os.path.join(self._output_path,
'*.nii*'))))
if not os.path.exists(self._output_path):
os.makedirs(self._output_path)
with self._logging():
tmp_dir = get_full_tmp_results_path(self._output_path,
self._tmp_results_dir)
self._logger.info(
'Saving temporary results in {}.'.format(tmp_dir))
worker = FittingProcessor(self._optimizer, self._model,
self._input_data.mask,
self._input_data.nifti_header,
self._output_path, tmp_dir,
self.recalculate)
processing_strategy = get_processing_strategy('optimization')
results = processing_strategy.process(worker)
self._write_protocol(self._model.get_input_data().protocol)
return results
def sample_composite_model(model, input_data, output_folder, nmr_samples, thinning, burnin, tmp_dir,
method=None, recalculate=False, store_samples=True, sample_items_to_save=None,
initialization_data=None, post_sampling_cb=None, sampler_options=None):
"""Sample a composite model.
Args:
model (:class:`~mdt.models.composite.DMRICompositeModel`): a composite model to sample
input_data (:class:`~mdt.utils.MRIInputData`): The input data object with which the model
is initialized before running
output_folder (string): The full path to the folder where to place the output
nmr_samples (int): the number of samples we would like to return.
burnin (int): the number of samples to burn-in, that is, to discard before returning the desired
number of samples
thinning (int): how many sample we wait before storing a new one. This will draw extra samples such that
the total number of samples generated is ``nmr_samples * (thinning)`` and the number of samples stored
is ``nmr_samples``. If set to one or lower we store every sample after the burn in.
tmp_dir (str): the preferred temporary storage dir
method (str): The sampling method to use, one of:
- 'AMWG', for the Adaptive Metropolis-Within-Gibbs method
- 'SCAM', for the Single Component Adaptive Metropolis
- 'FSL', for the sampling method used in the FSL toolbox
- 'MWG', for the Metropolis-Within-Gibbs (simple random walk metropolis without updates)
If not given, defaults to 'AMWG'.
recalculate (boolean): If we want to recalculate the results if they are already present.
store_samples (boolean, sequence or :class:`mdt.lib.processing_strategies.SamplesStorageStrategy`): if set to
False, we will store none of the samples. If set to True we will save all samples. If set to a sequence we
expect a sequence of integer numbers with sample positions to store. Finally, you can also give a subclass
instance of :class:`~mdt.lib.processing_strategies.SamplesStorageStrategy` (it is then typically set to
a :class:`mdt.lib.processing_strategies.SaveThinnedSamples` instance).
sample_items_to_save (list): list of output names we want to store the samples of. If given, we only
store the items specified in this list. Valid items are the free parameter names of the model and the
items 'LogLikelihood' and 'LogPrior'.
initialization_data (:class:`~mdt.utils.InitializationData`): provides (extra) initialization data to use
during model fitting. If we are optimizing a cascade model this data only applies to the last model in the
cascade.
post_sampling_cb (Callable[
[mot.sample.base.SamplingOutput, mdt.models.composite.DMRICompositeModel], Optional[Dict]]):
additional post-processing called after sampling. This function can optionally return a (nested)
dictionary with as keys dir-/file-names and as values maps to be stored in the results directory.
sampler_options (dict): specific options for the MCMC routine. These will be provided to the sampling routine
as additional keyword arguments to the constructor.
"""
samples_storage_strategy = SaveAllSamples()
if store_samples:
if sample_items_to_save:
samples_storage_strategy = SaveSpecificMaps(included=sample_items_to_save)
else:
samples_storage_strategy = SaveNoSamples()
if not model.is_input_data_sufficient(input_data):
raise InsufficientProtocolError(
'The provided protocol is insufficient for this model. '
'The reported errors where: {}'.format(model.get_input_data_problems(input_data)))
logger = logging.getLogger(__name__)
if not recalculate:
if os.path.exists(os.path.join(output_folder, 'UsedMask.nii.gz')) \
or os.path.exists(os.path.join(output_folder, 'UsedMask.nii')):
logger.info('Not recalculating {} model'.format(model.name))
return load_samples(output_folder)
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
model.set_input_data(input_data)
with per_model_logging_context(output_folder, overwrite=recalculate):
if initialization_data:
logger.info('Preparing the model with the user provided initialization data.')
initialization_data.apply_to_model(model, input_data)
with _log_info(logger, model.name):
worker = SamplingProcessor(
nmr_samples, thinning, burnin, method or 'AMWG',
model, input_data.mask, input_data.nifti_header, output_folder,
get_full_tmp_results_path(output_folder, tmp_dir), recalculate,
samples_storage_strategy=samples_storage_strategy,
post_sampling_cb=post_sampling_cb,
sampler_options=sampler_options)
processing_strategy = get_processing_strategy('sampling')
return processing_strategy.process(worker)
def compute_bootstrap(model,
input_data,
optimization_results,
output_folder,
bootstrap_method,
optimization_method,
nmr_samples,
tmp_dir,
recalculate=False,
keep_samples=True,
optimizer_options=None,
bootstrap_options=None):
"""Sample a composite model using residual bootstrapping
Args:
model (:class:`~mdt.models.base.EstimableModel`): a composite model to sample
input_data (:class:`~mdt.lib.input_data.MRIInputData`): The input data object with which the model
is initialized before running
optimization_results (dict or str): the optimization results, either a dictionary with results or the
path to a folder.
output_folder (string): The relative output path.
The resulting maps are placed in a subdirectory (named after the model name) in this output folder.
bootstrap_method (str): the bootstrap method to use, one of 'residual' or 'wild'.
optimization_method (str): The optimization routine to use.
nmr_samples (int): the number of samples we would like to return.
tmp_dir (str): the preferred temporary storage dir
recalculate (boolean): If we want to recalculate the results if they are already present.
keep_samples (boolean): determines if we keep any of the chains. If set to False, the chains will
be discarded after generating the mean and standard deviations.
optimizer_options (dict): the additional optimization options
bootstrap_options (dict): the bootstrap options
"""
from mdt.__version__ import __version__
logger = logging.getLogger(__name__)
logger.info('Using MDT version {}'.format(__version__))
logger.info('Preparing {} bootstrap for model {}'.format(
bootstrap_method, model.name))
output_folder = os.path.join(output_folder, model.name,
'{}_bootstrap'.format(bootstrap_method))
if not model.is_input_data_sufficient(input_data):
raise InsufficientProtocolError(
'The provided protocol is insufficient for this model. '
'The reported errors where: {}'.format(
model.get_input_data_problems(input_data)))
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
if recalculate:
shutil.rmtree(output_folder)
else:
if os.path.exists(os.path.join(output_folder, 'UsedMask.nii.gz')) \
or os.path.exists(os.path.join(output_folder, 'UsedMask.nii')):
logger.info('Not recalculating {} model'.format(model.name))
return load_samples(output_folder)
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
bootstrap_options = bootstrap_options or {}
with per_model_logging_context(output_folder, overwrite=recalculate):
with _log_info(logger, model.name):
if bootstrap_method == 'residual':
worker_class = ResidualBootstrappingProcessor
else:
worker_class = WildBootstrappingProcessor
worker = worker_class(optimization_method,
input_data,
optimization_results,
nmr_samples,
model,
input_data.mask,
input_data.nifti_header,
output_folder,
get_intermediate_results_path(
output_folder, tmp_dir),
recalculate,
keep_samples=keep_samples,
optimizer_options=optimizer_options,
**bootstrap_options)
processing_strategy = get_processing_strategy('sampling')
return processing_strategy.process(worker)
def sample_composite_model(model,
input_data,
output_folder,
nmr_samples,
thinning,
burnin,
tmp_dir,
recalculate=False,
store_samples=True,
sample_items_to_save=None,
initialization_data=None):
"""Sample a composite model.
Args:
model (:class:`~mdt.models.composite.DMRICompositeModel`): a composite model to sample
input_data (:class:`~mdt.utils.MRIInputData`): The input data object with which the model
is initialized before running
output_folder (string): The full path to the folder where to place the output
nmr_samples (int): the number of samples we would like to return.
burnin (int): the number of samples to burn-in, that is, to discard before returning the desired
number of samples
thinning (int): how many sample we wait before storing a new one. This will draw extra samples such that
the total number of samples generated is ``nmr_samples * (thinning)`` and the number of samples stored
is ``nmr_samples``. If set to one or lower we store every sample after the burn in.
tmp_dir (str): the preferred temporary storage dir
recalculate (boolean): If we want to recalculate the results if they are already present.
store_samples (boolean, sequence or :class:`mdt.processing_strategies.SamplesStorageStrategy`): if set to False
we will store none of the samples. If set to True we will save all samples. If set to a sequence we expect a
sequence of integer numbers with sample positions to store. Finally, you can also give a subclass instance
of :class:`~mdt.processing_strategies.SamplesStorageStrategy` (it is then typically set to
a :class:`mdt.processing_strategies.SaveThinnedSamples` instance).
sample_items_to_save (list): list of output names we want to store the samples of. If given, we only
store the items specified in this list. Valid items are the free parameter names of the model and the
items 'LogLikelihood' and 'LogPrior'.
initialization_data (:class:`~mdt.utils.InitializationData`): provides (extra) initialization data to use
during model fitting. If we are optimizing a cascade model this data only applies to the last model in the
cascade.
"""
samples_storage_strategy = SaveAllSamples()
if store_samples:
if sample_items_to_save:
samples_storage_strategy = SaveSpecificMaps(
included=sample_items_to_save)
else:
samples_storage_strategy = SaveNoSamples()
if not model.is_input_data_sufficient(input_data):
raise InsufficientProtocolError(
'The provided protocol is insufficient for this model. '
'The reported errors where: {}'.format(
model.get_input_data_problems(input_data)))
logger = logging.getLogger(__name__)
if not recalculate:
if os.path.exists(os.path.join(output_folder, 'UsedMask.nii.gz')) \
or os.path.exists(os.path.join(output_folder, 'UsedMask.nii')):
logger.info('Not recalculating {} model'.format(model.name))
return load_samples(output_folder)
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
model.set_input_data(input_data)
with per_model_logging_context(output_folder, overwrite=recalculate):
if initialization_data:
logger.info(
'Preparing the model with the user provided initialization data.'
)
initialization_data.apply_to_model(model, input_data)
with _log_info(logger, model.name):
worker = SamplingProcessor(
nmr_samples,
thinning,
burnin,
model,
input_data.mask,
input_data.nifti_header,
output_folder,
get_full_tmp_results_path(output_folder, tmp_dir),
recalculate,
samples_storage_strategy=samples_storage_strategy)
processing_strategy = get_processing_strategy('sampling')
return processing_strategy.process(worker)
def sample_model(model,
problem_data,
output_folder,
sampler=None,
recalculate=False,
cl_device_ind=None,
double_precision=False,
store_samples=True,
tmp_results_dir=True,
save_user_script_info=True,
initialization_maps=None):
"""Sample a composite model using the given cascading strategy.
Args:
model (:class:`~mdt.models.composite.DMRICompositeModel` or str): the model to sample
problem_data (:class:`~mdt.utils.DMRIProblemData`): the problem data object
output_folder (string): The path to the folder where to place the output, we will make a subdir with the
model name in it (for the optimization results) and then a subdir with the samples output.
sampler (:class:`mot.cl_routines.sampling.base.AbstractSampler`): the sampler to use
recalculate (boolean): If we want to recalculate the results if they are already present.
cl_device_ind (int): the index of the CL device to use. The index is from the list from the function
utils.get_cl_devices().
double_precision (boolean): if we would like to do the calculations in double precision
store_samples (boolean): if set to False we will store none of the samples. Use this
if you are only interested in the volume maps and not in the entire sample chain.
tmp_results_dir (str, True or None): The temporary dir for the calculations. Set to a string to use
that path directly, set to True to use the config value, set to None to disable.
save_user_script_info (boolean, str or SaveUserScriptInfo): The info we need to save about the script the
user is currently executing. If True (default) we use the stack to lookup the script the user is executing
and save that using a SaveFromScript saver. If a string is given we use that filename again for the
SaveFromScript saver. If False or None, we do not write any information. If a SaveUserScriptInfo is
given we use that directly.
initialization_maps (dict): 4d maps to initialize the sampling with. Per default this is None,
common practice is to use the maps from an optimization as starting point
Returns:
dict: the samples per parameter as a numpy memmap if store_samples is True
"""
import mdt.utils
from mot.load_balance_strategies import EvenDistribution
from mdt.model_sampling import sample_composite_model
from mdt.models.cascade import DMRICascadeModelInterface
import mot.configuration
if not mdt.utils.check_user_components():
init_user_settings(pass_if_exists=True)
if isinstance(model, string_types):
model = get_model(model)
if isinstance(model, DMRICascadeModelInterface):
raise ValueError(
'The function \'sample_model()\' does not accept cascade models.')
if not model.is_protocol_sufficient(problem_data.protocol):
raise InsufficientProtocolError(
'The given protocol is insufficient for this model. '
'The reported errors where: {}'.format(
model.get_protocol_problems(problem_data.protocol)))
if cl_device_ind is not None and not isinstance(cl_device_ind,
collections.Iterable):
cl_device_ind = [cl_device_ind]
if cl_device_ind is None:
cl_context_action = mot.configuration.VoidConfigurationAction()
else:
cl_context_action = mot.configuration.RuntimeConfigurationAction(
cl_environments=[get_cl_devices()[ind] for ind in cl_device_ind],
load_balancer=EvenDistribution())
with mot.configuration.config_context(cl_context_action):
if sampler is None:
sampler = configuration.get_sampler()
processing_strategy = get_processing_strategy('sampling',
model_names=model.name)
processing_strategy.set_tmp_dir(
get_temporary_results_dir(tmp_results_dir))
output_folder = os.path.join(output_folder, model.name, 'samples')
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
with per_model_logging_context(output_folder, overwrite=recalculate):
logger = logging.getLogger(__name__)
logger.info('Using MDT version {}'.format(__version__))
logger.info('Preparing for model {0}'.format(model.name))
if initialization_maps:
model.set_initial_parameters(
create_roi(initialization_maps, problem_data.mask))
model.double_precision = double_precision
results = sample_composite_model(model,
problem_data,
output_folder,
sampler,
processing_strategy,
recalculate=recalculate,
store_samples=store_samples)
easy_save_user_script_info(save_user_script_info,
output_folder + '/used_scripts.py',
stack()[1][0].f_globals.get('__file__'))
return results