def combine(self): super().combine() statistic_maps = {} for name in self._sample_storage: samples_path = os.path.join(self._output_dir, name + '.samples.npy') samples = open_memmap(samples_path, mode='r') statistic_maps[name] = np.mean(samples, axis=1) statistic_maps[name + '.std'] = np.std(samples, axis=1) write_all_as_nifti(restore_volumes(statistic_maps, self._mask), os.path.join(self._output_dir, 'univariate_normal'), nifti_header=self._nifti_header, gzip=self._write_volumes_gzipped) write_all_as_nifti({'UsedMask': self._mask}, self._output_dir, nifti_header=self._nifti_header, gzip=self._write_volumes_gzipped) if not self._keep_samples: for ind, name in enumerate(self._model.get_free_param_names()): os.remove(os.path.join(self._output_dir, name + '.samples.npy')) else: return load_samples(self._output_dir)
def combine(self): super().combine() for subdir in self._subdirs: self._combine_volumes(self._output_dir, self._tmp_storage_dir, self._nifti_header, maps_subdir=subdir) if self._samples_output_stored: return load_samples(self._output_dir) return SamplingProcessor.SampleChainNotStored()
def combine(self): super(SamplingProcessingWorker, self).combine() self._combine_volumes(self._output_dir, self._tmp_storage_dir, self._problem_data.volume_header, maps_subdir='volume_maps') if self._store_samples: for samples in glob.glob(os.path.join(self._tmp_storage_dir, '*.samples.npy')): shutil.move(samples, self._output_dir) return load_samples(self._output_dir) return SamplingProcessingWorker.SampleChainNotStored()
def view_result_samples(data, **kwargs): """View the samples from the given results set. Args: data (string or dict): The location of the maps to use the samples from, or the samples themselves. kwargs (dict): see SampleVisualizer for all the supported keywords """ from mdt.visualization.samples import SampleVisualizer if isinstance(data, string_types): data = load_samples(data) if kwargs.get('voxel_ind') is None: kwargs.update({'voxel_ind': data[list(data.keys())[0]].shape[0] / 2}) SampleVisualizer(data).show(**kwargs)
def sample_composite_model(model, input_data, output_folder, nmr_samples, thinning, burnin, tmp_dir, method=None, recalculate=False, store_samples=True, sample_items_to_save=None, initialization_data=None, post_sampling_cb=None, sampler_options=None): """Sample a composite model. Args: model (:class:`~mdt.models.composite.DMRICompositeModel`): a composite model to sample input_data (:class:`~mdt.utils.MRIInputData`): The input data object with which the model is initialized before running output_folder (string): The full path to the folder where to place the output nmr_samples (int): the number of samples we would like to return. burnin (int): the number of samples to burn-in, that is, to discard before returning the desired number of samples thinning (int): how many sample we wait before storing a new one. This will draw extra samples such that the total number of samples generated is ``nmr_samples * (thinning)`` and the number of samples stored is ``nmr_samples``. If set to one or lower we store every sample after the burn in. tmp_dir (str): the preferred temporary storage dir method (str): The sampling method to use, one of: - 'AMWG', for the Adaptive Metropolis-Within-Gibbs method - 'SCAM', for the Single Component Adaptive Metropolis - 'FSL', for the sampling method used in the FSL toolbox - 'MWG', for the Metropolis-Within-Gibbs (simple random walk metropolis without updates) If not given, defaults to 'AMWG'. recalculate (boolean): If we want to recalculate the results if they are already present. store_samples (boolean, sequence or :class:`mdt.lib.processing_strategies.SamplesStorageStrategy`): if set to False, we will store none of the samples. If set to True we will save all samples. If set to a sequence we expect a sequence of integer numbers with sample positions to store. Finally, you can also give a subclass instance of :class:`~mdt.lib.processing_strategies.SamplesStorageStrategy` (it is then typically set to a :class:`mdt.lib.processing_strategies.SaveThinnedSamples` instance). sample_items_to_save (list): list of output names we want to store the samples of. If given, we only store the items specified in this list. Valid items are the free parameter names of the model and the items 'LogLikelihood' and 'LogPrior'. initialization_data (:class:`~mdt.utils.InitializationData`): provides (extra) initialization data to use during model fitting. If we are optimizing a cascade model this data only applies to the last model in the cascade. post_sampling_cb (Callable[ [mot.sample.base.SamplingOutput, mdt.models.composite.DMRICompositeModel], Optional[Dict]]): additional post-processing called after sampling. This function can optionally return a (nested) dictionary with as keys dir-/file-names and as values maps to be stored in the results directory. sampler_options (dict): specific options for the MCMC routine. These will be provided to the sampling routine as additional keyword arguments to the constructor. """ samples_storage_strategy = SaveAllSamples() if store_samples: if sample_items_to_save: samples_storage_strategy = SaveSpecificMaps(included=sample_items_to_save) else: samples_storage_strategy = SaveNoSamples() if not model.is_input_data_sufficient(input_data): raise InsufficientProtocolError( 'The provided protocol is insufficient for this model. ' 'The reported errors where: {}'.format(model.get_input_data_problems(input_data))) logger = logging.getLogger(__name__) if not recalculate: if os.path.exists(os.path.join(output_folder, 'UsedMask.nii.gz')) \ or os.path.exists(os.path.join(output_folder, 'UsedMask.nii')): logger.info('Not recalculating {} model'.format(model.name)) return load_samples(output_folder) if not os.path.isdir(output_folder): os.makedirs(output_folder) model.set_input_data(input_data) with per_model_logging_context(output_folder, overwrite=recalculate): if initialization_data: logger.info('Preparing the model with the user provided initialization data.') initialization_data.apply_to_model(model, input_data) with _log_info(logger, model.name): worker = SamplingProcessor( nmr_samples, thinning, burnin, method or 'AMWG', model, input_data.mask, input_data.nifti_header, output_folder, get_full_tmp_results_path(output_folder, tmp_dir), recalculate, samples_storage_strategy=samples_storage_strategy, post_sampling_cb=post_sampling_cb, sampler_options=sampler_options) processing_strategy = get_processing_strategy('sampling') return processing_strategy.process(worker)
def compute_bootstrap(model, input_data, optimization_results, output_folder, bootstrap_method, optimization_method, nmr_samples, tmp_dir, recalculate=False, keep_samples=True, optimizer_options=None, bootstrap_options=None): """Sample a composite model using residual bootstrapping Args: model (:class:`~mdt.models.base.EstimableModel`): a composite model to sample input_data (:class:`~mdt.lib.input_data.MRIInputData`): The input data object with which the model is initialized before running optimization_results (dict or str): the optimization results, either a dictionary with results or the path to a folder. output_folder (string): The relative output path. The resulting maps are placed in a subdirectory (named after the model name) in this output folder. bootstrap_method (str): the bootstrap method to use, one of 'residual' or 'wild'. optimization_method (str): The optimization routine to use. nmr_samples (int): the number of samples we would like to return. tmp_dir (str): the preferred temporary storage dir recalculate (boolean): If we want to recalculate the results if they are already present. keep_samples (boolean): determines if we keep any of the chains. If set to False, the chains will be discarded after generating the mean and standard deviations. optimizer_options (dict): the additional optimization options bootstrap_options (dict): the bootstrap options """ from mdt.__version__ import __version__ logger = logging.getLogger(__name__) logger.info('Using MDT version {}'.format(__version__)) logger.info('Preparing {} bootstrap for model {}'.format( bootstrap_method, model.name)) output_folder = os.path.join(output_folder, model.name, '{}_bootstrap'.format(bootstrap_method)) if not model.is_input_data_sufficient(input_data): raise InsufficientProtocolError( 'The provided protocol is insufficient for this model. ' 'The reported errors where: {}'.format( model.get_input_data_problems(input_data))) if not os.path.isdir(output_folder): os.makedirs(output_folder) if recalculate: shutil.rmtree(output_folder) else: if os.path.exists(os.path.join(output_folder, 'UsedMask.nii.gz')) \ or os.path.exists(os.path.join(output_folder, 'UsedMask.nii')): logger.info('Not recalculating {} model'.format(model.name)) return load_samples(output_folder) if not os.path.isdir(output_folder): os.makedirs(output_folder) bootstrap_options = bootstrap_options or {} with per_model_logging_context(output_folder, overwrite=recalculate): with _log_info(logger, model.name): if bootstrap_method == 'residual': worker_class = ResidualBootstrappingProcessor else: worker_class = WildBootstrappingProcessor worker = worker_class(optimization_method, input_data, optimization_results, nmr_samples, model, input_data.mask, input_data.nifti_header, output_folder, get_intermediate_results_path( output_folder, tmp_dir), recalculate, keep_samples=keep_samples, optimizer_options=optimizer_options, **bootstrap_options) processing_strategy = get_processing_strategy('sampling') return processing_strategy.process(worker)
def sample_composite_model(model, input_data, output_folder, nmr_samples, thinning, burnin, tmp_dir, recalculate=False, store_samples=True, sample_items_to_save=None, initialization_data=None): """Sample a composite model. Args: model (:class:`~mdt.models.composite.DMRICompositeModel`): a composite model to sample input_data (:class:`~mdt.utils.MRIInputData`): The input data object with which the model is initialized before running output_folder (string): The full path to the folder where to place the output nmr_samples (int): the number of samples we would like to return. burnin (int): the number of samples to burn-in, that is, to discard before returning the desired number of samples thinning (int): how many sample we wait before storing a new one. This will draw extra samples such that the total number of samples generated is ``nmr_samples * (thinning)`` and the number of samples stored is ``nmr_samples``. If set to one or lower we store every sample after the burn in. tmp_dir (str): the preferred temporary storage dir recalculate (boolean): If we want to recalculate the results if they are already present. store_samples (boolean, sequence or :class:`mdt.processing_strategies.SamplesStorageStrategy`): if set to False we will store none of the samples. If set to True we will save all samples. If set to a sequence we expect a sequence of integer numbers with sample positions to store. Finally, you can also give a subclass instance of :class:`~mdt.processing_strategies.SamplesStorageStrategy` (it is then typically set to a :class:`mdt.processing_strategies.SaveThinnedSamples` instance). sample_items_to_save (list): list of output names we want to store the samples of. If given, we only store the items specified in this list. Valid items are the free parameter names of the model and the items 'LogLikelihood' and 'LogPrior'. initialization_data (:class:`~mdt.utils.InitializationData`): provides (extra) initialization data to use during model fitting. If we are optimizing a cascade model this data only applies to the last model in the cascade. """ samples_storage_strategy = SaveAllSamples() if store_samples: if sample_items_to_save: samples_storage_strategy = SaveSpecificMaps( included=sample_items_to_save) else: samples_storage_strategy = SaveNoSamples() if not model.is_input_data_sufficient(input_data): raise InsufficientProtocolError( 'The provided protocol is insufficient for this model. ' 'The reported errors where: {}'.format( model.get_input_data_problems(input_data))) logger = logging.getLogger(__name__) if not recalculate: if os.path.exists(os.path.join(output_folder, 'UsedMask.nii.gz')) \ or os.path.exists(os.path.join(output_folder, 'UsedMask.nii')): logger.info('Not recalculating {} model'.format(model.name)) return load_samples(output_folder) if not os.path.isdir(output_folder): os.makedirs(output_folder) model.set_input_data(input_data) with per_model_logging_context(output_folder, overwrite=recalculate): if initialization_data: logger.info( 'Preparing the model with the user provided initialization data.' ) initialization_data.apply_to_model(model, input_data) with _log_info(logger, model.name): worker = SamplingProcessor( nmr_samples, thinning, burnin, model, input_data.mask, input_data.nifti_header, output_folder, get_full_tmp_results_path(output_folder, tmp_dir), recalculate, samples_storage_strategy=samples_storage_strategy) processing_strategy = get_processing_strategy('sampling') return processing_strategy.process(worker)