def run(self, args, extra_args): dwi_name = os.path.splitext(os.path.realpath(args.dwi))[0] dwi_name = dwi_name.replace('.nii', '') if args.output_name: output_name = os.path.realpath(args.output_name) else: output_name = dwi_name + '_mask.nii.gz' if args.cl_device_ind: if isinstance(args.cl_device_ind, int): mot.configuration.set_cl_environments( [self.available_devices[args.cl_device_ind]]) else: mot.configuration.set_cl_environments([ self.available_devices[ind] for ind in args.cl_device_ind ]) mot.configuration.set_load_balancer(EvenDistribution()) mdt.create_median_otsu_brain_mask(os.path.realpath(args.dwi), os.path.realpath(args.protocol), output_name, median_radius=args.median_radius, numpass=args.numpass, dilate=args.dilate, mask_threshold=args.mask_threshold) logger = logging.getLogger(__name__) logger.info('Saved the mask to: {}'.format(output_name))
def _run_composite_model(self, model, recalculate, model_names): with mot.configuration.config_context( RuntimeConfigurationAction(cl_environments=self._cl_envs, load_balancer=self._load_balancer)): with per_model_logging_context( os.path.join(self._output_folder, model.name)): self._logger.info('Using MDT version {}'.format(__version__)) self._logger.info('Preparing for model {0}'.format(model.name)) self._logger.info('Current cascade: {0}'.format(model_names)) optimizer = self._optimizer or get_optimizer_for_model( model_names) if self._cl_device_indices is not None: all_devices = get_cl_devices() optimizer.cl_environments = [ all_devices[ind] for ind in self._cl_device_indices ] optimizer.load_balancer = EvenDistribution() processing_strategy = get_processing_strategy( 'optimization', model_names=model_names) processing_strategy.set_tmp_dir(self._tmp_results_dir) fitter = SingleModelFit(model, self._problem_data, self._output_folder, optimizer, processing_strategy, recalculate=recalculate) results = fitter.run() return results
def _update_settings(self): selection = [ind for ind in range(self.cldevicesSelection.count()) if self.cldevicesSelection.item(ind).isSelected()] mot.configuration.set_cl_environments([self.all_cl_devices[ind] for ind in selection]) mot.configuration.set_load_balancer(EvenDistribution()) update_gui_config({'runtime_settings': {'cl_device_ind': selection}})
def load(self, value): if 'cl_device_ind' in value: if value['cl_device_ind'] is not None: from mdt.utils import get_cl_devices all_devices = get_cl_devices() indices = value['cl_device_ind'] if not isinstance(indices, collections.Iterable): indices = [indices] devices = [ all_devices[ind] for ind in indices if ind < len(all_devices) ] if devices: mot.configuration.set_cl_environments(devices) mot.configuration.set_load_balancer(EvenDistribution())
def sample_model(model, input_data, output_folder, nmr_samples=None, burnin=None, thinning=None, recalculate=False, cl_device_ind=None, double_precision=False, store_samples=True, sample_items_to_save=None, tmp_results_dir=True, save_user_script_info=True, initialization_data=None, post_processing=None): """Sample a composite model using the Adaptive Metropolis-Within-Gibbs (AMWG) MCMC algorithm [1]. Args: model (:class:`~mdt.models.composite.DMRICompositeModel` or str): the model to sample input_data (:class:`~mdt.utils.MRIInputData`): the input data object containing all the info needed for the model fitting. output_folder (string): The path to the folder where to place the output, we will make a subdir with the model name in it (for the optimization results) and then a subdir with the samples output. nmr_samples (int): the number of samples we would like to return. burnin (int): the number of samples to burn-in, that is, to discard before returning the desired number of samples thinning (int): how many sample we wait before storing a new one. This will draw extra samples such that the total number of samples generated is ``nmr_samples * (thinning)`` and the number of samples stored is ``nmr_samples``. If set to one or lower we store every sample after the burn in. recalculate (boolean): If we want to recalculate the results if they are already present. cl_device_ind (int): the index of the CL device to use. The index is from the list from the function utils.get_cl_devices(). double_precision (boolean): if we would like to do the calculations in double precision store_samples (boolean): determines if we store any of the samples. If set to False we will store none of the samples. sample_items_to_save (list): list of output names we want to store the samples of. If given, we only store the items specified in this list. Valid items are the free parameter names of the model and the items 'LogLikelihood' and 'LogPrior'. tmp_results_dir (str, True or None): The temporary dir for the calculations. Set to a string to use that path directly, set to True to use the config value, set to None to disable. save_user_script_info (boolean, str or SaveUserScriptInfo): The info we need to save about the script the user is currently executing. If True (default) we use the stack to lookup the script the user is executing and save that using a SaveFromScript saver. If a string is given we use that filename again for the SaveFromScript saver. If False or None, we do not write any information. If a SaveUserScriptInfo is given we use that directly. initialization_data (:class:`~mdt.utils.InitializationData` or dict): provides (extra) initialization data to use during model fitting. If we are optimizing a cascade model this data only applies to the last model in the cascade. If a dictionary is given we will load the elements as arguments to the :class:`mdt.utils.SimpleInitializationData` class. For example:: initialization_data = {'fixes': {...}, 'inits': {...}} is transformed into:: initialization_data = SimpleInitializationData(fixes={...}, inits={...}) post_processing (dict): a dictionary with flags for post-processing options to enable or disable. For valid elements, please see the configuration file settings for ``sampling`` under ``post_processing``. Valid input for this parameter is for example: {'sample_statistics': True} to enable automatic calculation of the sampling statistics. Returns: dict: if store_samples is True then we return the samples per parameter as a numpy memmap. If store_samples is False we return None References: 1. Roberts GO, Rosenthal JS. Examples of adaptive MCMC. J Comput Graph Stat. 2009;18(2):349-367. doi:10.1198/jcgs.2009.06134. """ import mdt.utils from mot.load_balance_strategies import EvenDistribution from mdt.model_sampling import sample_composite_model from mdt.models.cascade import DMRICascadeModelInterface import mot.configuration settings = mdt.configuration.get_general_sampling_settings() if nmr_samples is None: nmr_samples = settings['nmr_samples'] if burnin is None: burnin = settings['burnin'] if thinning is None: thinning = settings['thinning'] if not isinstance(initialization_data, InitializationData) and initialization_data is not None: initialization_data = SimpleInitializationData(**initialization_data) if not mdt.utils.check_user_components(): init_user_settings(pass_if_exists=True) if isinstance(model, string_types): model = get_model(model)() if post_processing: model.update_active_post_processing('sampling', post_processing) if isinstance(model, DMRICascadeModelInterface): raise ValueError( 'The function \'sample_model()\' does not accept cascade models.') if cl_device_ind is not None and not isinstance(cl_device_ind, collections.Iterable): cl_device_ind = [cl_device_ind] if cl_device_ind is None: cl_context_action = mot.configuration.VoidConfigurationAction() else: cl_envs = [get_cl_devices()[ind] for ind in cl_device_ind] cl_context_action = mot.configuration.RuntimeConfigurationAction( cl_environments=cl_envs, load_balancer=EvenDistribution(), double_precision=double_precision) with mot.configuration.config_context(cl_context_action): base_dir = os.path.join(output_folder, model.name, 'samples') if not os.path.isdir(base_dir): os.makedirs(base_dir) if recalculate: shutil.rmtree(base_dir) logger = logging.getLogger(__name__) logger.info('Using MDT version {}'.format(__version__)) logger.info('Preparing for model {0}'.format(model.name)) logger.info('The parameters we will sample are: {0}'.format( model.get_free_param_names())) results = sample_composite_model( model, input_data, base_dir, nmr_samples, thinning, burnin, get_temporary_results_dir(tmp_results_dir), recalculate=recalculate, store_samples=store_samples, sample_items_to_save=sample_items_to_save, initialization_data=initialization_data) easy_save_user_script_info(save_user_script_info, os.path.join(base_dir, 'used_scripts.py'), stack()[1][0].f_globals.get('__file__')) return results
def __init__(self, model, problem_data, output_folder, optimizer=None, recalculate=False, only_recalculate_last=False, cascade_subdir=False, cl_device_ind=None, double_precision=False, tmp_results_dir=True): """Setup model fitting for the given input model and data. To actually fit the model call run(). Args: model (:class:`~mdt.models.composite.DMRICompositeModel` or :class:`~mdt.models.cascade.DMRICascadeModelInterface`): the model we want to optimize. problem_data (:class:`~mdt.utils.DMRIProblemData`): the problem data object which contains the dwi image, the dwi header, the brain_mask and the protocol to use. output_folder (string): The full path to the folder where to place the output optimizer (:class:`mot.cl_routines.optimizing.base.AbstractOptimizer`): The optimization routine to use. If None, we create one using the configuration files. recalculate (boolean): If we want to recalculate the results if they are already present. only_recalculate_last (boolean): If we want to recalculate all the models. This is only of importance when dealing with CascadeModels. If set to true we only recalculate the last element in the chain (if recalculate is set to True, that is). If set to false, we recalculate everything. This only holds for the first level of the cascade. cascade_subdir (boolean): if we want to create a subdirectory for the given model if it is a cascade model. Per default we output the maps of cascaded results in the same directory, this allows reusing cascaded results for other cascades (for example, if you cascade BallStick -> Noddi you can use the BallStick results also for BallStick -> Charmed). This flag disables that behaviour and instead outputs the results of a cascade model to a subdirectory for that cascade. This does not apply recursive. cl_device_ind (int): the index of the CL device to use. The index is from the list from the function get_cl_devices(). This can also be a list of device indices. double_precision (boolean): if we would like to do the calculations in double precision tmp_results_dir (str, True or None): The temporary dir for the calculations. Set to a string to use that path directly, set to True to use the config value, set to None to disable. """ if isinstance(model, string_types): model = get_model(model) model.double_precision = double_precision self._model = model self._problem_data = problem_data self._output_folder = output_folder if cascade_subdir and isinstance(self._model, DMRICascadeModelInterface): self._output_folder += '/{}'.format(self._model.name) self._optimizer = optimizer self._recalculate = recalculate self._only_recalculate_last = only_recalculate_last self._logger = logging.getLogger(__name__) self._cl_device_indices = cl_device_ind self._model_names_list = [] self._tmp_results_dir = get_temporary_results_dir(tmp_results_dir) if self._cl_device_indices is not None and not isinstance( self._cl_device_indices, collections.Iterable): self._cl_device_indices = [self._cl_device_indices] self._cl_envs = None self._load_balancer = None if self._cl_device_indices is not None: all_devices = get_cl_devices() self._cl_envs = [ all_devices[ind] for ind in self._cl_device_indices ] self._load_balancer = EvenDistribution() if not model.is_protocol_sufficient(self._problem_data.protocol): raise InsufficientProtocolError( 'The provided protocol is insufficient for this model. ' 'The reported errors where: {}'.format( self._model.get_protocol_problems( self._problem_data.protocol)))
def sample_model(model, problem_data, output_folder, sampler=None, recalculate=False, cl_device_ind=None, double_precision=False, store_samples=True, tmp_results_dir=True, save_user_script_info=True, initialization_maps=None): """Sample a composite model using the given cascading strategy. Args: model (:class:`~mdt.models.composite.DMRICompositeModel` or str): the model to sample problem_data (:class:`~mdt.utils.DMRIProblemData`): the problem data object output_folder (string): The path to the folder where to place the output, we will make a subdir with the model name in it (for the optimization results) and then a subdir with the samples output. sampler (:class:`mot.cl_routines.sampling.base.AbstractSampler`): the sampler to use recalculate (boolean): If we want to recalculate the results if they are already present. cl_device_ind (int): the index of the CL device to use. The index is from the list from the function utils.get_cl_devices(). double_precision (boolean): if we would like to do the calculations in double precision store_samples (boolean): if set to False we will store none of the samples. Use this if you are only interested in the volume maps and not in the entire sample chain. tmp_results_dir (str, True or None): The temporary dir for the calculations. Set to a string to use that path directly, set to True to use the config value, set to None to disable. save_user_script_info (boolean, str or SaveUserScriptInfo): The info we need to save about the script the user is currently executing. If True (default) we use the stack to lookup the script the user is executing and save that using a SaveFromScript saver. If a string is given we use that filename again for the SaveFromScript saver. If False or None, we do not write any information. If a SaveUserScriptInfo is given we use that directly. initialization_maps (dict): 4d maps to initialize the sampling with. Per default this is None, common practice is to use the maps from an optimization as starting point Returns: dict: the samples per parameter as a numpy memmap if store_samples is True """ import mdt.utils from mot.load_balance_strategies import EvenDistribution from mdt.model_sampling import sample_composite_model from mdt.models.cascade import DMRICascadeModelInterface import mot.configuration if not mdt.utils.check_user_components(): init_user_settings(pass_if_exists=True) if isinstance(model, string_types): model = get_model(model) if isinstance(model, DMRICascadeModelInterface): raise ValueError( 'The function \'sample_model()\' does not accept cascade models.') if not model.is_protocol_sufficient(problem_data.protocol): raise InsufficientProtocolError( 'The given protocol is insufficient for this model. ' 'The reported errors where: {}'.format( model.get_protocol_problems(problem_data.protocol))) if cl_device_ind is not None and not isinstance(cl_device_ind, collections.Iterable): cl_device_ind = [cl_device_ind] if cl_device_ind is None: cl_context_action = mot.configuration.VoidConfigurationAction() else: cl_context_action = mot.configuration.RuntimeConfigurationAction( cl_environments=[get_cl_devices()[ind] for ind in cl_device_ind], load_balancer=EvenDistribution()) with mot.configuration.config_context(cl_context_action): if sampler is None: sampler = configuration.get_sampler() processing_strategy = get_processing_strategy('sampling', model_names=model.name) processing_strategy.set_tmp_dir( get_temporary_results_dir(tmp_results_dir)) output_folder = os.path.join(output_folder, model.name, 'samples') if not os.path.isdir(output_folder): os.makedirs(output_folder) with per_model_logging_context(output_folder, overwrite=recalculate): logger = logging.getLogger(__name__) logger.info('Using MDT version {}'.format(__version__)) logger.info('Preparing for model {0}'.format(model.name)) if initialization_maps: model.set_initial_parameters( create_roi(initialization_maps, problem_data.mask)) model.double_precision = double_precision results = sample_composite_model(model, problem_data, output_folder, sampler, processing_strategy, recalculate=recalculate, store_samples=store_samples) easy_save_user_script_info(save_user_script_info, output_folder + '/used_scripts.py', stack()[1][0].f_globals.get('__file__')) return results
def __init__(self, model, input_data, output_folder, optimizer=None, recalculate=False, only_recalculate_last=False, cascade_subdir=False, cl_device_ind=None, double_precision=False, tmp_results_dir=True, initialization_data=None, post_processing=None): """Setup model fitting for the given input model and data. To actually fit the model call run(). Args: model (str or :class:`~mdt.models.composite.DMRICompositeModel` or :class:`~mdt.models.cascade.DMRICascadeModelInterface`): the model we want to optimize. input_data (:class:`~mdt.utils.MRIInputData`): the input data object containing all the info needed for the model fitting. output_folder (string): The full path to the folder where to place the output optimizer (:class:`mot.cl_routines.optimizing.base.AbstractOptimizer`): The optimization routine to use. If None, we create one using the configuration files. recalculate (boolean): If we want to recalculate the results if they are already present. only_recalculate_last (boolean): If we want to recalculate all the models. This is only of importance when dealing with CascadeModels. If set to true we only recalculate the last element in the chain (if recalculate is set to True, that is). If set to false, we recalculate everything. This only holds for the first level of the cascade. cascade_subdir (boolean): if we want to create a subdirectory for the given model if it is a cascade model. Per default we output the maps of cascaded results in the same directory, this allows reusing cascaded results for other cascades (for example, if you cascade BallStick -> Noddi you can use the BallStick results also for BallStick -> Charmed). This flag disables that behaviour and instead outputs the results of a cascade model to a subdirectory for that cascade. This does not apply recursive. cl_device_ind (int): the index of the CL device to use. The index is from the list from the function get_cl_devices(). This can also be a list of device indices. double_precision (boolean): if we would like to do the calculations in double precision tmp_results_dir (str, True or None): The temporary dir for the calculations. Set to a string to use that path directly, set to True to use the config value, set to None to disable. initialization_data (:class:`~mdt.utils.InitializationData`): extra initialization data to use during model fitting. If we are optimizing a cascade model this data only applies to the last model in the cascade. post_processing (dict): a dictionary with flags for post-processing options to enable or disable. For valid elements, please see the configuration file settings for ``optimization`` under ``post_processing``. Valid input for this parameter is for example: {'covariance': False} to disable automatic calculation of the covariance from the Hessian. """ if isinstance(model, string_types): model = get_model(model)() if post_processing: model.update_active_post_processing('optimization', post_processing) self._model = model self._input_data = input_data self._output_folder = output_folder if cascade_subdir and isinstance(self._model, DMRICascadeModelInterface): self._output_folder += '/{}'.format(self._model.name) self._optimizer = optimizer self._recalculate = recalculate self._only_recalculate_last = only_recalculate_last self._logger = logging.getLogger(__name__) self._model_names_list = [] self._tmp_results_dir = get_temporary_results_dir(tmp_results_dir) self._initialization_data = initialization_data or SimpleInitializationData( ) if cl_device_ind is not None and not isinstance( cl_device_ind, collections.Iterable): cl_device_ind = [cl_device_ind] cl_environments = None if cl_device_ind is not None: cl_environments = [get_cl_devices()[ind] for ind in cl_device_ind] self._cl_runtime_info = CLRuntimeInfo( cl_environments=cl_environments, load_balancer=EvenDistribution(), double_precision=double_precision) if not model.is_input_data_sufficient(self._input_data): raise InsufficientProtocolError( 'The provided protocol is insufficient for this model. ' 'The reported errors where: {}'.format( self._model.get_input_data_problems(self._input_data)))