def combine(self):
super().combine()
statistic_maps = {}
for name in self._sample_storage:
samples_path = os.path.join(self._output_dir,
name + '.samples.npy')
samples = open_memmap(samples_path, mode='r')
statistic_maps[name] = np.mean(samples, axis=1)
statistic_maps[name + '.std'] = np.std(samples, axis=1)
write_all_as_nifti(restore_volumes(statistic_maps, self._mask),
os.path.join(self._output_dir, 'univariate_normal'),
nifti_header=self._nifti_header,
gzip=self._write_volumes_gzipped)
write_all_as_nifti({'UsedMask': self._mask},
self._output_dir,
nifti_header=self._nifti_header,
gzip=self._write_volumes_gzipped)
if not self._keep_samples:
for ind, name in enumerate(self._model.get_free_param_names()):
os.remove(os.path.join(self._output_dir,
name + '.samples.npy'))
else:
return load_samples(self._output_dir)
def combine(self):
super().combine()
for subdir in self._subdirs:
self._combine_volumes(self._output_dir, self._tmp_storage_dir,
self._nifti_header, maps_subdir=subdir)
if self._samples_output_stored:
return load_samples(self._output_dir)
return SamplingProcessor.SampleChainNotStored()
def combine(self):
super(SamplingProcessingWorker, self).combine()
self._combine_volumes(self._output_dir, self._tmp_storage_dir,
self._problem_data.volume_header, maps_subdir='volume_maps')
if self._store_samples:
for samples in glob.glob(os.path.join(self._tmp_storage_dir, '*.samples.npy')):
shutil.move(samples, self._output_dir)
return load_samples(self._output_dir)
return SamplingProcessingWorker.SampleChainNotStored()
def view_result_samples(data, **kwargs):
"""View the samples from the given results set.
Args:
data (string or dict): The location of the maps to use the samples from, or the samples themselves.
kwargs (dict): see SampleVisualizer for all the supported keywords
"""
from mdt.visualization.samples import SampleVisualizer
if isinstance(data, string_types):
data = load_samples(data)
if kwargs.get('voxel_ind') is None:
kwargs.update({'voxel_ind': data[list(data.keys())[0]].shape[0] / 2})
SampleVisualizer(data).show(**kwargs)
def sample_composite_model(model, input_data, output_folder, nmr_samples, thinning, burnin, tmp_dir,
method=None, recalculate=False, store_samples=True, sample_items_to_save=None,
initialization_data=None, post_sampling_cb=None, sampler_options=None):
"""Sample a composite model.
Args:
model (:class:`~mdt.models.composite.DMRICompositeModel`): a composite model to sample
input_data (:class:`~mdt.utils.MRIInputData`): The input data object with which the model
is initialized before running
output_folder (string): The full path to the folder where to place the output
nmr_samples (int): the number of samples we would like to return.
burnin (int): the number of samples to burn-in, that is, to discard before returning the desired
number of samples
thinning (int): how many sample we wait before storing a new one. This will draw extra samples such that
the total number of samples generated is ``nmr_samples * (thinning)`` and the number of samples stored
is ``nmr_samples``. If set to one or lower we store every sample after the burn in.
tmp_dir (str): the preferred temporary storage dir
method (str): The sampling method to use, one of:
- 'AMWG', for the Adaptive Metropolis-Within-Gibbs method
- 'SCAM', for the Single Component Adaptive Metropolis
- 'FSL', for the sampling method used in the FSL toolbox
- 'MWG', for the Metropolis-Within-Gibbs (simple random walk metropolis without updates)
If not given, defaults to 'AMWG'.
recalculate (boolean): If we want to recalculate the results if they are already present.
store_samples (boolean, sequence or :class:`mdt.lib.processing_strategies.SamplesStorageStrategy`): if set to
False, we will store none of the samples. If set to True we will save all samples. If set to a sequence we
expect a sequence of integer numbers with sample positions to store. Finally, you can also give a subclass
instance of :class:`~mdt.lib.processing_strategies.SamplesStorageStrategy` (it is then typically set to
a :class:`mdt.lib.processing_strategies.SaveThinnedSamples` instance).
sample_items_to_save (list): list of output names we want to store the samples of. If given, we only
store the items specified in this list. Valid items are the free parameter names of the model and the
items 'LogLikelihood' and 'LogPrior'.
initialization_data (:class:`~mdt.utils.InitializationData`): provides (extra) initialization data to use
during model fitting. If we are optimizing a cascade model this data only applies to the last model in the
cascade.
post_sampling_cb (Callable[
[mot.sample.base.SamplingOutput, mdt.models.composite.DMRICompositeModel], Optional[Dict]]):
additional post-processing called after sampling. This function can optionally return a (nested)
dictionary with as keys dir-/file-names and as values maps to be stored in the results directory.
sampler_options (dict): specific options for the MCMC routine. These will be provided to the sampling routine
as additional keyword arguments to the constructor.
"""
samples_storage_strategy = SaveAllSamples()
if store_samples:
if sample_items_to_save:
samples_storage_strategy = SaveSpecificMaps(included=sample_items_to_save)
else:
samples_storage_strategy = SaveNoSamples()
if not model.is_input_data_sufficient(input_data):
raise InsufficientProtocolError(
'The provided protocol is insufficient for this model. '
'The reported errors where: {}'.format(model.get_input_data_problems(input_data)))
logger = logging.getLogger(__name__)
if not recalculate:
if os.path.exists(os.path.join(output_folder, 'UsedMask.nii.gz')) \
or os.path.exists(os.path.join(output_folder, 'UsedMask.nii')):
logger.info('Not recalculating {} model'.format(model.name))
return load_samples(output_folder)
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
model.set_input_data(input_data)
with per_model_logging_context(output_folder, overwrite=recalculate):
if initialization_data:
logger.info('Preparing the model with the user provided initialization data.')
initialization_data.apply_to_model(model, input_data)
with _log_info(logger, model.name):
worker = SamplingProcessor(
nmr_samples, thinning, burnin, method or 'AMWG',
model, input_data.mask, input_data.nifti_header, output_folder,
get_full_tmp_results_path(output_folder, tmp_dir), recalculate,
samples_storage_strategy=samples_storage_strategy,
post_sampling_cb=post_sampling_cb,
sampler_options=sampler_options)
processing_strategy = get_processing_strategy('sampling')
return processing_strategy.process(worker)
def compute_bootstrap(model,
input_data,
optimization_results,
output_folder,
bootstrap_method,
optimization_method,
nmr_samples,
tmp_dir,
recalculate=False,
keep_samples=True,
optimizer_options=None,
bootstrap_options=None):
"""Sample a composite model using residual bootstrapping
Args:
model (:class:`~mdt.models.base.EstimableModel`): a composite model to sample
input_data (:class:`~mdt.lib.input_data.MRIInputData`): The input data object with which the model
is initialized before running
optimization_results (dict or str): the optimization results, either a dictionary with results or the
path to a folder.
output_folder (string): The relative output path.
The resulting maps are placed in a subdirectory (named after the model name) in this output folder.
bootstrap_method (str): the bootstrap method to use, one of 'residual' or 'wild'.
optimization_method (str): The optimization routine to use.
nmr_samples (int): the number of samples we would like to return.
tmp_dir (str): the preferred temporary storage dir
recalculate (boolean): If we want to recalculate the results if they are already present.
keep_samples (boolean): determines if we keep any of the chains. If set to False, the chains will
be discarded after generating the mean and standard deviations.
optimizer_options (dict): the additional optimization options
bootstrap_options (dict): the bootstrap options
"""
from mdt.__version__ import __version__
logger = logging.getLogger(__name__)
logger.info('Using MDT version {}'.format(__version__))
logger.info('Preparing {} bootstrap for model {}'.format(
bootstrap_method, model.name))
output_folder = os.path.join(output_folder, model.name,
'{}_bootstrap'.format(bootstrap_method))
if not model.is_input_data_sufficient(input_data):
raise InsufficientProtocolError(
'The provided protocol is insufficient for this model. '
'The reported errors where: {}'.format(
model.get_input_data_problems(input_data)))
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
if recalculate:
shutil.rmtree(output_folder)
else:
if os.path.exists(os.path.join(output_folder, 'UsedMask.nii.gz')) \
or os.path.exists(os.path.join(output_folder, 'UsedMask.nii')):
logger.info('Not recalculating {} model'.format(model.name))
return load_samples(output_folder)
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
bootstrap_options = bootstrap_options or {}
with per_model_logging_context(output_folder, overwrite=recalculate):
with _log_info(logger, model.name):
if bootstrap_method == 'residual':
worker_class = ResidualBootstrappingProcessor
else:
worker_class = WildBootstrappingProcessor
worker = worker_class(optimization_method,
input_data,
optimization_results,
nmr_samples,
model,
input_data.mask,
input_data.nifti_header,
output_folder,
get_intermediate_results_path(
output_folder, tmp_dir),
recalculate,
keep_samples=keep_samples,
optimizer_options=optimizer_options,
**bootstrap_options)
processing_strategy = get_processing_strategy('sampling')
return processing_strategy.process(worker)
def sample_composite_model(model,
input_data,
output_folder,
nmr_samples,
thinning,
burnin,
tmp_dir,
recalculate=False,
store_samples=True,
sample_items_to_save=None,
initialization_data=None):
"""Sample a composite model.
Args:
model (:class:`~mdt.models.composite.DMRICompositeModel`): a composite model to sample
input_data (:class:`~mdt.utils.MRIInputData`): The input data object with which the model
is initialized before running
output_folder (string): The full path to the folder where to place the output
nmr_samples (int): the number of samples we would like to return.
burnin (int): the number of samples to burn-in, that is, to discard before returning the desired
number of samples
thinning (int): how many sample we wait before storing a new one. This will draw extra samples such that
the total number of samples generated is ``nmr_samples * (thinning)`` and the number of samples stored
is ``nmr_samples``. If set to one or lower we store every sample after the burn in.
tmp_dir (str): the preferred temporary storage dir
recalculate (boolean): If we want to recalculate the results if they are already present.
store_samples (boolean, sequence or :class:`mdt.processing_strategies.SamplesStorageStrategy`): if set to False
we will store none of the samples. If set to True we will save all samples. If set to a sequence we expect a
sequence of integer numbers with sample positions to store. Finally, you can also give a subclass instance
of :class:`~mdt.processing_strategies.SamplesStorageStrategy` (it is then typically set to
a :class:`mdt.processing_strategies.SaveThinnedSamples` instance).
sample_items_to_save (list): list of output names we want to store the samples of. If given, we only
store the items specified in this list. Valid items are the free parameter names of the model and the
items 'LogLikelihood' and 'LogPrior'.
initialization_data (:class:`~mdt.utils.InitializationData`): provides (extra) initialization data to use
during model fitting. If we are optimizing a cascade model this data only applies to the last model in the
cascade.
"""
samples_storage_strategy = SaveAllSamples()
if store_samples:
if sample_items_to_save:
samples_storage_strategy = SaveSpecificMaps(
included=sample_items_to_save)
else:
samples_storage_strategy = SaveNoSamples()
if not model.is_input_data_sufficient(input_data):
raise InsufficientProtocolError(
'The provided protocol is insufficient for this model. '
'The reported errors where: {}'.format(
model.get_input_data_problems(input_data)))
logger = logging.getLogger(__name__)
if not recalculate:
if os.path.exists(os.path.join(output_folder, 'UsedMask.nii.gz')) \
or os.path.exists(os.path.join(output_folder, 'UsedMask.nii')):
logger.info('Not recalculating {} model'.format(model.name))
return load_samples(output_folder)
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
model.set_input_data(input_data)
with per_model_logging_context(output_folder, overwrite=recalculate):
if initialization_data:
logger.info(
'Preparing the model with the user provided initialization data.'
)
initialization_data.apply_to_model(model, input_data)
with _log_info(logger, model.name):
worker = SamplingProcessor(
nmr_samples,
thinning,
burnin,
model,
input_data.mask,
input_data.nifti_header,
output_folder,
get_full_tmp_results_path(output_folder, tmp_dir),
recalculate,
samples_storage_strategy=samples_storage_strategy)
processing_strategy = get_processing_strategy('sampling')
return processing_strategy.process(worker)