def _build_results(self): """ build the results after a fit is performed :returns: :rtype: """ # set the median fit self.restore_median_fit() # Find maximum of the log posterior idx = self._log_probability_values.argmax() # Get parameter values at the maximum approximate_MAP_point = self._raw_samples[idx, :] # Sets the values of the parameters to their MAP values for i, parameter in enumerate(self._free_parameters): self._free_parameters[parameter].value = approximate_MAP_point[i] # Get the value of the posterior for each dataset at the MAP log_posteriors = collections.OrderedDict() log_prior = self._log_prior(approximate_MAP_point) # keep track of the total number of data points # and the total posterior total_n_data_points = 0 total_log_posterior = 0 for dataset in list(self._data_list.values()): log_posterior = dataset.get_log_like() + log_prior log_posteriors[dataset.name] = log_posterior total_n_data_points += dataset.get_number_of_data_points() total_log_posterior += log_posterior # compute the statistical measures statistical_measures = collections.OrderedDict() # compute the point estimates statistical_measures["AIC"] = aic(total_log_posterior, len(self._free_parameters), total_n_data_points) statistical_measures["BIC"] = bic(total_log_posterior, len(self._free_parameters), total_n_data_points) this_dic, pdic = dic(self) # compute the posterior estimates statistical_measures["DIC"] = this_dic statistical_measures["PDIC"] = pdic if self._marginal_likelihood is not None: statistical_measures["log(Z)"] = self._marginal_likelihood # TODO: add WAIC # Instance the result self._results = BayesianResults( self._likelihood_model, self._raw_samples, log_posteriors, statistical_measures=statistical_measures, log_probabilty=self._log_like_values)
def _build_results(self): # Find maximum of the log posterior idx = self._log_probability_values.argmax() # Get parameter values at the maximum approximate_MAP_point = self._raw_samples[idx, :] # Sets the values of the parameters to their MAP values for i, parameter in enumerate(self._free_parameters): self._free_parameters[parameter].value = approximate_MAP_point[i] # Get the value of the posterior for each dataset at the MAP log_posteriors = collections.OrderedDict() log_prior = self._log_prior(approximate_MAP_point) # keep track of the total number of data points # and the total posterior total_n_data_points = 0 total_log_posterior = 0 for dataset in self._data_list.values(): log_posterior = dataset.get_log_like() + log_prior log_posteriors[dataset.name] = log_posterior total_n_data_points += dataset.get_number_of_data_points() total_log_posterior += log_posterior # compute the statistical measures statistical_measures = collections.OrderedDict() # compute the point estimates statistical_measures['AIC'] = aic(total_log_posterior,len(self._free_parameters),total_n_data_points) statistical_measures['BIC'] = bic(total_log_posterior,len(self._free_parameters),total_n_data_points) this_dic, pdic = dic(self) # compute the posterior estimates statistical_measures['DIC'] = this_dic statistical_measures['PDIC'] = pdic if self._marginal_likelihood is not None: statistical_measures['log(Z)'] = self._marginal_likelihood #TODO: add WAIC # Instance the result self._results = BayesianResults(self._likelihood_model, self._raw_samples, log_posteriors,statistical_measures=statistical_measures)
class BayesianAnalysis(object): def __init__(self, likelihood_model, data_list, **kwargs): """ Bayesian analysis. :param likelihood_model: the likelihood model :param data_list: the list of datasets to use (normally an instance of DataList) :param kwargs: use 'verbose=True' for verbose operation :return: """ self._analysis_type = "bayesian" # Verify that all the free parameters have priors for parameter_name, parameter in likelihood_model.free_parameters.iteritems(): if not parameter.has_prior(): raise RuntimeError("You need to define priors for all free parameters before instancing a " "Bayesian analysis") # Process optional keyword parameters self.verbose = False for k, v in kwargs.iteritems(): if k.lower() == "verbose": self.verbose = bool(kwargs["verbose"]) self._likelihood_model = likelihood_model self._data_list = data_list for dataset in self._data_list.values(): dataset.set_model(self._likelihood_model) # Now get the nuisance parameters from the data and add them to the model # NOTE: it is important that this is *after* the setting of the model, as some # plugins might need to adjust the number of nuisance parameters depending on the # likelihood model for parameter_name, parameter in dataset.nuisance_parameters.items(): # Enforce that the nuisance parameter contains the instance name, because otherwise multiple instance # of the same plugin will overwrite each other's nuisance parameters assert dataset.name in parameter_name, "This is a bug of the plugin for %s: nuisance parameters " \ "must contain the instance name" % type(dataset) self._likelihood_model.add_external_parameter(parameter) # # Make sure that the current model is used in all data sets # # for dataset in self.data_list.values(): # dataset.set_model(self._likelihood_model) # Init the samples to None self._samples = None self._raw_samples = None self._sampler = None self._log_like_values = None self._results = None # Get the initial list of free parameters, useful for debugging purposes self._update_free_parameters() @property def results(self): return self._results @property def analysis_type(self): return self._analysis_type @property def log_like_values(self): """ Returns the value of the log_likelihood found by the bayesian sampler while sampling from the posterior. If you need to find the values of the parameters which generated a given value of the log. likelihood, remember that the samples accessible through the property .raw_samples are ordered in the same way as the vector returned by this method. :return: a vector of log. like values """ return self._log_like_values @property def log_probability_values(self): """ Returns the value of the log_probability (posterior) found by the bayesian sampler while sampling from the posterior. If you need to find the values of the parameters which generated a given value of the log. likelihood, remember that the samples accessible through the property .raw_samples are ordered in the same way as the vector returned by this method. :return: a vector of log probabilty values """ return self._log_probability_values @property def log_marginal_likelihood(self): """ Return the log marginal likelihood (evidence) if computed :return: """ return self._marginal_likelihood def sample(self, n_walkers, burn_in, n_samples, quiet=False, seed=None): """ Sample the posterior with the Goodman & Weare's Affine Invariant Markov chain Monte Carlo :param n_walkers: :param burn_in: :param n_samples: :param quiet: if False, do not print results :param seed: if provided, it is used to seed the random numbers generator before the MCMC :return: MCMC samples """ self._update_free_parameters() n_dim = len(self._free_parameters.keys()) # Get starting point p0 = self._get_starting_points(n_walkers) sampling_procedure = sample_with_progress # Deactivate memoization in astromodels, which is useless in this case since we will never use twice the # same set of parameters with use_astromodels_memoization(False): if threeML_config['parallel']['use-parallel']: c = ParallelClient() view = c[:] sampler = emcee.EnsembleSampler(n_walkers, n_dim, self.get_posterior, pool=view) # Sampling with progress in parallel is super-slow, so let's # use the non-interactive one sampling_procedure = sample_without_progress else: sampler = emcee.EnsembleSampler(n_walkers, n_dim, self.get_posterior) # If a seed is provided, set the random number seed if seed is not None: sampler._random.seed(seed) # Sample the burn-in pos, prob, state = sampling_procedure(title="Burn-in", p0=p0, sampler=sampler, n_samples=burn_in) # Reset sampler sampler.reset() # Run the true sampling _ = sampling_procedure(title="Sampling", p0=pos, sampler=sampler, n_samples=n_samples, rstate0=state) acc = np.mean(sampler.acceptance_fraction) print("\nMean acceptance fraction: %s\n" % acc) self._sampler = sampler self._raw_samples = sampler.flatchain # Compute the corresponding values of the likelihood # First we need the prior log_prior = map(lambda x: self._log_prior(x), self._raw_samples) # Now we get the log posterior and we remove the log prior self._log_like_values = sampler.flatlnprobability - log_prior # we also want to store the log probability self._log_probability_values = sampler.flatlnprobability self._marginal_likelihood = None self._build_samples_dictionary() self._build_results() # Display results if not quiet: self._results.display() return self.samples def sample_parallel_tempering(self, n_temps, n_walkers, burn_in, n_samples, quiet=False): """ Sample with parallel tempering :param: n_temps :param: n_walkers :param: burn_in :param: n_samples :return: MCMC samples """ free_parameters = self._likelihood_model.free_parameters n_dim = len(free_parameters.keys()) sampler = emcee.PTSampler(n_temps, n_walkers, n_dim, self._log_like, self._log_prior) # Get one starting point for each temperature p0 = np.empty((n_temps, n_walkers, n_dim)) for i in range(n_temps): p0[i, :, :] = self._get_starting_points(n_walkers) print("Running burn-in of %s samples...\n" % burn_in) p, lnprob, lnlike = sample_with_progress("Burn-in", p0, sampler, burn_in) # Reset sampler sampler.reset() print("\nSampling\n") _ = sample_with_progress("Sampling", p, sampler, n_samples, lnprob0=lnprob, lnlike0=lnlike) self._sampler = sampler # Now build the _samples dictionary self._raw_samples = sampler.flatchain.reshape(-1, sampler.flatchain.shape[-1]) self._log_probability_values = None self._log_like_values = None self._marginal_likelihood = None self._build_samples_dictionary() self._build_results() # Display results if not quiet: self._results.display() return self.samples def sample_multinest(self, n_live_points, chain_name="chains/fit-", quiet=False, **kwargs): """ Sample the posterior with MULTINEST nested sampling (Feroz & Hobson) :param: n_live_points: number of MULTINEST livepoints :param: chain_names: where to stor the multinest incremental output :param: quiet: Whether or not to should results :param: **kwargs (pyMULTINEST kwords) :return: MCMC samples """ assert has_pymultinest, "You don't have pymultinest installed, so you cannot run the Multinest sampler" self._update_free_parameters() n_dim = len(self._free_parameters.keys()) # MULTINEST has a convergence criteria and therefore, there is no way # to determine progress sampling_procedure = sample_without_progress # MULTINEST uses a different call signiture for # sampling so we construct callbakcs loglike, multinest_prior = self._construct_multinest_posterior() # We need to check if the MCMC # chains will have a place on # the disk to write and if not, # create one mcmc_chains_out_dir = "" tmp = chain_name.split('/') for s in tmp[:-1]: mcmc_chains_out_dir += s + '/' if using_mpi: # if we are running in parallel and this is not the # first engine, then we want to wait and let everything finish if rank != 0: # let these guys take a break time.sleep(1) else: # create mcmc chains directory only on first engine if not os.path.exists(mcmc_chains_out_dir): os.makedirs(mcmc_chains_out_dir) else: if not os.path.exists(mcmc_chains_out_dir): os.makedirs(mcmc_chains_out_dir) print("\nSampling\n") print("MULTINEST has its own convergence criteria... you will have to wait blindly for it to finish") print("If INS is enabled, one can monitor the likelihood in the terminal for completion information") # Multinest must be run parallel via an external method # see the demo in the examples folder!! if threeML_config['parallel']['use-parallel']: raise RuntimeError("If you want to run multinest in parallell you need to use an ad-hoc method") else: sampler = pymultinest.run(loglike, multinest_prior, n_dim, n_dim, outputfiles_basename=chain_name, n_live_points=n_live_points, **kwargs) # Use PyMULTINEST analyzer to gather parameter info process_fit = False if using_mpi: # if we are running in parallel and this is not the # first engine, then we want to wait and let everything finish if rank !=0: # let these guys take a break time.sleep(5) # these engines do not need to read process_fit = False else: # wait for a moment to allow it all to turn off time.sleep(5) process_fit = True else: process_fit = True if process_fit: multinest_analyzer = pymultinest.analyse.Analyzer(n_params=n_dim, outputfiles_basename=chain_name) # Get the log. likelihood values from the chain self._log_like_values = multinest_analyzer.get_equal_weighted_posterior()[:, -1] self._sampler = sampler self._raw_samples = multinest_analyzer.get_equal_weighted_posterior()[:, :-1] # now get the log probability self._log_probability_values = self._log_like_values + np.array([self._log_prior(samples) for samples in self._raw_samples]) self._build_samples_dictionary() self._marginal_likelihood = multinest_analyzer.get_stats()['global evidence'] / np.log(10.) self._build_results() # Display results if not quiet: self._results.display() # now get the marginal likelihood return self.samples def _build_samples_dictionary(self): """ Build the dictionary to access easily the samples by parameter :return: none """ self._samples = collections.OrderedDict() for i, (parameter_name, parameter) in enumerate(self._free_parameters.iteritems()): # Add the samples for this parameter for this source self._samples[parameter_name] = self._raw_samples[:, i] def _build_results(self): # Find maximum of the log posterior idx = self._log_probability_values.argmax() # Get parameter values at the maximum approximate_MAP_point = self._raw_samples[idx, :] # Sets the values of the parameters to their MAP values for i, parameter in enumerate(self._free_parameters): self._free_parameters[parameter].value = approximate_MAP_point[i] # Get the value of the posterior for each dataset at the MAP log_posteriors = collections.OrderedDict() log_prior = self._log_prior(approximate_MAP_point) # keep track of the total number of data points # and the total posterior total_n_data_points = 0 total_log_posterior = 0 for dataset in self._data_list.values(): log_posterior = dataset.get_log_like() + log_prior log_posteriors[dataset.name] = log_posterior total_n_data_points += dataset.get_number_of_data_points() total_log_posterior += log_posterior # compute the statistical measures statistical_measures = collections.OrderedDict() # compute the point estimates statistical_measures['AIC'] = aic(total_log_posterior,len(self._free_parameters),total_n_data_points) statistical_measures['BIC'] = bic(total_log_posterior,len(self._free_parameters),total_n_data_points) this_dic, pdic = dic(self) # compute the posterior estimates statistical_measures['DIC'] = this_dic statistical_measures['PDIC'] = pdic if self._marginal_likelihood is not None: statistical_measures['log(Z)'] = self._marginal_likelihood #TODO: add WAIC # Instance the result self._results = BayesianResults(self._likelihood_model, self._raw_samples, log_posteriors,statistical_measures=statistical_measures) @property def raw_samples(self): """ Access the samples from the posterior distribution generated by the selected sampler in raw form (i.e., in the format returned by the sampler) :return: the samples as returned by the sampler """ return self._raw_samples @property def samples(self): """ Access the samples from the posterior distribution generated by the selected sampler :return: a dictionary with the samples from the posterior distribution for each parameter """ return self._samples @property def sampler(self): """ Access the instance of the sampler used to sample the posterior distribution :return: an instance of the sampler """ return self._sampler def plot_chains(self, thin=None): """ Produce a plot of the series of samples for each parameter :parameter thin: use only one sample every 'thin' samples :return: a matplotlib.figure instance """ return self._results.plot_chains( thin ) @property def likelihood_model(self): """ :return: likelihood model (a Model instance) """ return self._likelihood_model @property def data_list(self): """ :return: data list for this analysis """ return self._data_list def convergence_plots(self, n_samples_in_each_subset, n_subsets): """ Compute the mean and variance for subsets of the samples, and plot them. They should all be around the same values if the MCMC has converged to the posterior distribution. The subsamples are taken with two different strategies: the first is to slide a fixed-size window, the second is to take random samples from the chain (bootstrap) :param n_samples_in_each_subset: number of samples in each subset :param n_subsets: number of subsets to take for each strategy :return: a matplotlib.figure instance """ return self._results.convergence_plots( n_samples_in_each_subset, n_subsets) def restore_median_fit(self): """ Sets the model parameters to the mean of the marginal distributions """ for i, (parameter_name, parameter) in enumerate(self._free_parameters.iteritems()): # Add the samples for this parameter for this source mean_par = np.median(self._samples[parameter_name]) parameter.value = mean_par def _update_free_parameters(self): """ Update the dictionary of the current free parameters :return: """ self._free_parameters = self._likelihood_model.free_parameters def get_posterior(self, trial_values): """Compute the posterior for the normal sampler""" # Assign this trial values to the parameters and # store the corresponding values for the priors # self._update_free_parameters() assert len(self._free_parameters) == len(trial_values), ("Something is wrong. Number of free parameters " "do not match the number of trial values.") log_prior = 0 # with use_ for i, (parameter_name, parameter) in enumerate(self._free_parameters.iteritems()): prior_value = parameter.prior(trial_values[i]) if prior_value == 0: # Outside allowed region of parameter space return -np.inf else: parameter.value = trial_values[i] log_prior += math.log10(prior_value) log_like = self._log_like(trial_values) # print("Log like is %s, log_prior is %s, for trial values %s" % (log_like, log_prior,trial_values)) return log_like + log_prior def _construct_multinest_posterior(self): """ pymultinest becomes confused with the self pointer. We therefore ceate callbacks that pymultinest can understand. Here, we construct the prior and log. likelihood for multinest on the unit cube """ # First update the free parameters (in case the user changed them after the construction of the class) self._update_free_parameters() def loglike(trial_values, ndim, params): # NOTE: the _log_like function DOES NOT assign trial_values to the parameters for i, parameter in enumerate(self._free_parameters.values()): parameter.value = trial_values[i] log_like = self._log_like(trial_values) if self.verbose: n_par = len(self._free_parameters) print( "Trial values %s gave a log_like of %s" % (map(lambda i: "%.2g" % trial_values[i], range(n_par)), log_like)) return log_like # Now construct the prior # MULTINEST priors are defined on the unit cube # and should return the value in the bounds... not the # probability. Therefore, we must make some transforms def prior(params, ndim, nparams): for i, (parameter_name, parameter) in enumerate(self._free_parameters.iteritems()): try: params[i] = parameter.prior.from_unit_cube(params[i]) except AttributeError: raise RuntimeError("The prior you are trying to use for parameter %s is " "not compatible with multinest" % parameter_name) # Give a test run to the prior to check that it is working. If it crashes while multinest is going # it will not stop multinest from running and generate thousands of exceptions (argh!) n_dim = len(self._free_parameters) _ = prior([0.5] * n_dim, n_dim, []) return loglike, prior def _get_starting_points(self, n_walkers, variance=0.1): # Generate the starting points for the walkers by getting random # values for the parameters close to the current value # Fractional variance for randomization # (0.1 means var = 0.1 * value ) p0 = [] for i in range(n_walkers): this_p0 = map(lambda x: x.get_randomized_value(variance), self._free_parameters.values()) p0.append(this_p0) return p0 def _log_prior(self, trial_values): """Compute the sum of log-priors, used in the parallel tempering sampling""" # Compute the sum of the log-priors log_prior = 0 for i, (parameter_name, parameter) in enumerate(self._free_parameters.iteritems()): prior_value = parameter.prior(trial_values[i]) if prior_value == 0: # Outside allowed region of parameter space return -np.inf else: parameter.value = trial_values[i] log_prior += math.log10(prior_value) return log_prior def _log_like(self, trial_values): """Compute the log-likelihood""" # Get the value of the log-likelihood for this parameters try: # Loop over each dataset and get the likelihood values for each set log_like_values = map(lambda dataset: dataset.get_log_like(), self._data_list.values()) except ModelAssertionViolation: # Fit engine or sampler outside of allowed zone return -np.inf except: # We don't want to catch more serious issues raise # Sum the values of the log-like log_like = np.sum(log_like_values) if not np.isfinite(log_like): # Issue warning custom_warnings.warn("Likelihood value is infinite for parameters %s" % trial_values, LikelihoodIsInfinite) return -np.inf return log_like @staticmethod def _calc_min_interval(x, alpha): """ Internal method to determine the minimum interval of a given width Assumes that x is sorted numpy array. :param a: a numpy array containing samples :param alpha: probability of type I error :returns: list containing min and max HDI """ n = len(x) cred_mass = 1.0 - alpha interval_idx_inc = int(np.floor(cred_mass * n)) n_intervals = n - interval_idx_inc interval_width = x[interval_idx_inc:] - x[:n_intervals] if len(interval_width) == 0: raise ValueError('Too few elements for interval calculation') min_idx = np.argmin(interval_width) hdi_min = x[min_idx] hdi_max = x[min_idx + interval_idx_inc] return hdi_min, hdi_max def _hpd(self, x, alpha=0.05): """Calculate highest posterior density (HPD) of array for given alpha. The HPD is the minimum width Bayesian credible interval (BCI). :param x: array containing MCMC samples :param alpha : Desired probability of type I error (defaults to 0.05) """ # Currently only 1D available. # future addition will fix this # Make a copy of trace # x = x.copy() # For multivariate node # if x.ndim > 1: # Transpose first, then sort # tx = np.transpose(x, list(range(x.ndim))[1:] + [0]) # dims = np.shape(tx) # Container list for intervals # intervals = np.resize(0.0, dims[:-1] + (2,)) # sx = np.sort(tx[index]) # Append to list # intervals[index] = self._calc_min_interval(sx, alpha) # Transpose back before returning # return np.array(intervals) # else: # Sort univariate node sx = np.sort(x) return np.array(self._calc_min_interval(sx, alpha))
class BayesianAnalysis(object): def __init__(self, likelihood_model, data_list, **kwargs): """ Bayesian analysis. :param likelihood_model: the likelihood model :param data_list: the list of datasets to use (normally an instance of DataList) :param kwargs: use 'verbose=True' for verbose operation :return: """ self._analysis_type = "bayesian" # Verify that all the free parameters have priors for parameter_name, parameter in likelihood_model.free_parameters.items( ): if not parameter.has_prior(): raise RuntimeError( "You need to define priors for all free parameters before instancing a " "Bayesian analysis") # Process optional keyword parameters self.verbose = False for k, v in kwargs.items(): if k.lower() == "verbose": self.verbose = bool(kwargs["verbose"]) self._likelihood_model = likelihood_model self._data_list = data_list for dataset in list(self._data_list.values()): dataset.set_model(self._likelihood_model) # Now get the nuisance parameters from the data and add them to the model # NOTE: it is important that this is *after* the setting of the model, as some # plugins might need to adjust the number of nuisance parameters depending on the # likelihood model for parameter_name, parameter in list( dataset.nuisance_parameters.items()): # Enforce that the nuisance parameter contains the instance name, because otherwise multiple instance # of the same plugin will overwrite each other's nuisance parameters assert dataset.name in parameter_name, "This is a bug of the plugin for %s: nuisance parameters " \ "must contain the instance name" % type(dataset) self._likelihood_model.add_external_parameter(parameter) # # Make sure that the current model is used in all data sets # # for dataset in self.data_list.values(): # dataset.set_model(self._likelihood_model) # Init the samples to None self._samples = None self._raw_samples = None self._sampler = None self._log_like_values = None self._results = None # Get the initial list of free parameters, useful for debugging purposes self._update_free_parameters() @property def results(self): return self._results @property def analysis_type(self): return self._analysis_type @property def log_like_values(self): """ Returns the value of the log_likelihood found by the bayesian sampler while sampling from the posterior. If you need to find the values of the parameters which generated a given value of the log. likelihood, remember that the samples accessible through the property .raw_samples are ordered in the same way as the vector returned by this method. :return: a vector of log. like values """ return self._log_like_values @property def log_probability_values(self): """ Returns the value of the log_probability (posterior) found by the bayesian sampler while sampling from the posterior. If you need to find the values of the parameters which generated a given value of the log. likelihood, remember that the samples accessible through the property .raw_samples are ordered in the same way as the vector returned by this method. :return: a vector of log probabilty values """ return self._log_probability_values @property def log_marginal_likelihood(self): """ Return the log marginal likelihood (evidence) if computed :return: """ return self._marginal_likelihood def sample(self, n_walkers, burn_in, n_samples, quiet=False, seed=None): """ Sample the posterior with the Goodman & Weare's Affine Invariant Markov chain Monte Carlo :param n_walkers: :param burn_in: :param n_samples: :param quiet: if False, do not print results :param seed: if provided, it is used to seed the random numbers generator before the MCMC :return: MCMC samples """ self._update_free_parameters() n_dim = len(list(self._free_parameters.keys())) # Get starting point p0 = self._get_starting_points(n_walkers) sampling_procedure = sample_with_progress # Deactivate memoization in astromodels, which is useless in this case since we will never use twice the # same set of parameters with use_astromodels_memoization(False): if threeML_config['parallel']['use-parallel']: c = ParallelClient() view = c[:] sampler = emcee.EnsembleSampler(n_walkers, n_dim, self.get_posterior, pool=view) # Sampling with progress in parallel is super-slow, so let's # use the non-interactive one sampling_procedure = sample_without_progress else: sampler = emcee.EnsembleSampler(n_walkers, n_dim, self.get_posterior) # If a seed is provided, set the random number seed if seed is not None: sampler._random.seed(seed) # Sample the burn-in pos, prob, state = sampling_procedure(title="Burn-in", p0=p0, sampler=sampler, n_samples=burn_in) # Reset sampler sampler.reset() # Run the true sampling _ = sampling_procedure(title="Sampling", p0=pos, sampler=sampler, n_samples=n_samples, rstate0=state) acc = np.mean(sampler.acceptance_fraction) print("\nMean acceptance fraction: %s\n" % acc) self._sampler = sampler self._raw_samples = sampler.get_chain(flat=True) # Compute the corresponding values of the likelihood # First we need the prior log_prior = [self._log_prior(x) for x in self._raw_samples] # Now we get the log posterior and we remove the log prior self._log_like_values = sampler.get_log_prob(flat=True) - log_prior # we also want to store the log probability self._log_probability_values = sampler.get_log_prob(flat=True) self._marginal_likelihood = None self._build_samples_dictionary() self._build_results() # Display results if not quiet: self._results.display() return self.samples def sample_parallel_tempering(self, n_temps, n_walkers, burn_in, n_samples, quiet=False): """ Sample with parallel tempering :param: n_temps :param: n_walkers :param: burn_in :param: n_samples :return: MCMC samples """ free_parameters = self._likelihood_model.free_parameters n_dim = len(list(free_parameters.keys())) sampler = emcee.PTSampler(n_temps, n_walkers, n_dim, self._log_like, self._log_prior) # Get one starting point for each temperature p0 = np.empty((n_temps, n_walkers, n_dim)) for i in range(n_temps): p0[i, :, :] = self._get_starting_points(n_walkers) print("Running burn-in of %s samples...\n" % burn_in) p, lnprob, lnlike = sample_with_progress("Burn-in", p0, sampler, burn_in) # Reset sampler sampler.reset() print("\nSampling\n") _ = sample_with_progress("Sampling", p, sampler, n_samples, lnprob0=lnprob, lnlike0=lnlike) self._sampler = sampler # Now build the _samples dictionary self._raw_samples = sampler.get_chain(flat=True).reshape( -1, sampler.get_chain(flat=True).shape[-1]) self._log_probability_values = None self._log_like_values = None self._marginal_likelihood = None self._build_samples_dictionary() self._build_results() # Display results if not quiet: self._results.display() return self.samples def sample_multinest(self, n_live_points, chain_name="chains/fit-", quiet=False, **kwargs): """ Sample the posterior with MULTINEST nested sampling (Feroz & Hobson) :param: n_live_points: number of MULTINEST livepoints :param: chain_names: where to stor the multinest incremental output :param: quiet: Whether or not to should results :param: **kwargs (pyMULTINEST kwords) :return: MCMC samples """ assert has_pymultinest, "You don't have pymultinest installed, so you cannot run the Multinest sampler" self._update_free_parameters() n_dim = len(list(self._free_parameters.keys())) # MULTINEST has a convergence criteria and therefore, there is no way # to determine progress sampling_procedure = sample_without_progress # MULTINEST uses a different call signiture for # sampling so we construct callbakcs loglike, multinest_prior = self._construct_multinest_posterior() # We need to check if the MCMC # chains will have a place on # the disk to write and if not, # create one mcmc_chains_out_dir = "" tmp = chain_name.split('/') for s in tmp[:-1]: mcmc_chains_out_dir += s + '/' if using_mpi: # if we are running in parallel and this is not the # first engine, then we want to wait and let everything finish if rank != 0: # let these guys take a break time.sleep(1) else: # create mcmc chains directory only on first engine if not os.path.exists(mcmc_chains_out_dir): os.makedirs(mcmc_chains_out_dir) else: if not os.path.exists(mcmc_chains_out_dir): os.makedirs(mcmc_chains_out_dir) print("\nSampling\n") print( "MULTINEST has its own convergence criteria... you will have to wait blindly for it to finish" ) print( "If INS is enabled, one can monitor the likelihood in the terminal for completion information" ) # Multinest must be run parallel via an external method # see the demo in the examples folder!! if threeML_config['parallel']['use-parallel']: raise RuntimeError( "If you want to run multinest in parallell you need to use an ad-hoc method" ) else: sampler = pymultinest.run(loglike, multinest_prior, n_dim, n_dim, outputfiles_basename=chain_name, n_live_points=n_live_points, **kwargs) # Use PyMULTINEST analyzer to gather parameter info process_fit = False if using_mpi: # if we are running in parallel and this is not the # first engine, then we want to wait and let everything finish if rank != 0: # let these guys take a break time.sleep(5) # these engines do not need to read process_fit = False else: # wait for a moment to allow it all to turn off time.sleep(5) process_fit = True else: process_fit = True if process_fit: multinest_analyzer = pymultinest.analyse.Analyzer( n_params=n_dim, outputfiles_basename=chain_name) # Get the log. likelihood values from the chain self._log_like_values = multinest_analyzer.get_equal_weighted_posterior( )[:, -1] self._sampler = sampler self._raw_samples = multinest_analyzer.get_equal_weighted_posterior( )[:, :-1] # now get the log probability self._log_probability_values = self._log_like_values + np.array( [self._log_prior(samples) for samples in self._raw_samples]) self._build_samples_dictionary() self._marginal_likelihood = old_div( multinest_analyzer.get_stats()['global evidence'], np.log(10.)) self._build_results() # Display results if not quiet: self._results.display() # now get the marginal likelihood return self.samples def _build_samples_dictionary(self): """ Build the dictionary to access easily the samples by parameter :return: none """ self._samples = collections.OrderedDict() for i, (parameter_name, parameter) in enumerate(self._free_parameters.items()): # Add the samples for this parameter for this source self._samples[parameter_name] = self._raw_samples[:, i] def _build_results(self): # Find maximum of the log posterior idx = self._log_probability_values.argmax() # Get parameter values at the maximum approximate_MAP_point = self._raw_samples[idx, :] # Sets the values of the parameters to their MAP values for i, parameter in enumerate(self._free_parameters): self._free_parameters[parameter].value = approximate_MAP_point[i] # Get the value of the posterior for each dataset at the MAP log_posteriors = collections.OrderedDict() log_prior = self._log_prior(approximate_MAP_point) # keep track of the total number of data points # and the total posterior total_n_data_points = 0 total_log_posterior = 0 for dataset in list(self._data_list.values()): log_posterior = dataset.get_log_like() + log_prior log_posteriors[dataset.name] = log_posterior total_n_data_points += dataset.get_number_of_data_points() total_log_posterior += log_posterior # compute the statistical measures statistical_measures = collections.OrderedDict() # compute the point estimates statistical_measures['AIC'] = aic(total_log_posterior, len(self._free_parameters), total_n_data_points) statistical_measures['BIC'] = bic(total_log_posterior, len(self._free_parameters), total_n_data_points) this_dic, pdic = dic(self) # compute the posterior estimates statistical_measures['DIC'] = this_dic statistical_measures['PDIC'] = pdic if self._marginal_likelihood is not None: statistical_measures['log(Z)'] = self._marginal_likelihood #TODO: add WAIC # Instance the result self._results = BayesianResults( self._likelihood_model, self._raw_samples, log_posteriors, statistical_measures=statistical_measures) @property def raw_samples(self): """ Access the samples from the posterior distribution generated by the selected sampler in raw form (i.e., in the format returned by the sampler) :return: the samples as returned by the sampler """ return self._raw_samples @property def samples(self): """ Access the samples from the posterior distribution generated by the selected sampler :return: a dictionary with the samples from the posterior distribution for each parameter """ return self._samples @property def sampler(self): """ Access the instance of the sampler used to sample the posterior distribution :return: an instance of the sampler """ return self._sampler def plot_chains(self, thin=None): """ Produce a plot of the series of samples for each parameter :parameter thin: use only one sample every 'thin' samples :return: a matplotlib.figure instance """ return self._results.plot_chains(thin) @property def likelihood_model(self): """ :return: likelihood model (a Model instance) """ return self._likelihood_model @property def data_list(self): """ :return: data list for this analysis """ return self._data_list def convergence_plots(self, n_samples_in_each_subset, n_subsets): """ Compute the mean and variance for subsets of the samples, and plot them. They should all be around the same values if the MCMC has converged to the posterior distribution. The subsamples are taken with two different strategies: the first is to slide a fixed-size window, the second is to take random samples from the chain (bootstrap) :param n_samples_in_each_subset: number of samples in each subset :param n_subsets: number of subsets to take for each strategy :return: a matplotlib.figure instance """ return self._results.convergence_plots(n_samples_in_each_subset, n_subsets) def restore_median_fit(self): """ Sets the model parameters to the mean of the marginal distributions """ for i, (parameter_name, parameter) in enumerate(self._free_parameters.items()): # Add the samples for this parameter for this source mean_par = np.median(self._samples[parameter_name]) parameter.value = mean_par def _update_free_parameters(self): """ Update the dictionary of the current free parameters :return: """ self._free_parameters = self._likelihood_model.free_parameters def get_posterior(self, trial_values): """Compute the posterior for the normal sampler""" # Assign this trial values to the parameters and # store the corresponding values for the priors # self._update_free_parameters() assert len(self._free_parameters) == len(trial_values), ( "Something is wrong. Number of free parameters " "do not match the number of trial values.") log_prior = 0 # with use_ for i, (parameter_name, parameter) in enumerate(self._free_parameters.items()): prior_value = parameter.prior(trial_values[i]) if prior_value == 0: # Outside allowed region of parameter space return -np.inf else: parameter.value = trial_values[i] log_prior += math.log10(prior_value) log_like = self._log_like(trial_values) # print("Log like is %s, log_prior is %s, for trial values %s" % (log_like, log_prior,trial_values)) return log_like + log_prior def _construct_multinest_posterior(self): """ pymultinest becomes confused with the self pointer. We therefore ceate callbacks that pymultinest can understand. Here, we construct the prior and log. likelihood for multinest on the unit cube """ # First update the free parameters (in case the user changed them after the construction of the class) self._update_free_parameters() def loglike(trial_values, ndim, params): # NOTE: the _log_like function DOES NOT assign trial_values to the parameters for i, parameter in enumerate(self._free_parameters.values()): parameter.value = trial_values[i] log_like = self._log_like(trial_values) if self.verbose: n_par = len(self._free_parameters) print("Trial values %s gave a log_like of %s" % (["%.2g" % trial_values[i] for i in range(n_par)], log_like)) return log_like # Now construct the prior # MULTINEST priors are defined on the unit cube # and should return the value in the bounds... not the # probability. Therefore, we must make some transforms def prior(params, ndim, nparams): for i, (parameter_name, parameter) in enumerate(self._free_parameters.items()): try: params[i] = parameter.prior.from_unit_cube(params[i]) except AttributeError: raise RuntimeError( "The prior you are trying to use for parameter %s is " "not compatible with multinest" % parameter_name) # Give a test run to the prior to check that it is working. If it crashes while multinest is going # it will not stop multinest from running and generate thousands of exceptions (argh!) n_dim = len(self._free_parameters) _ = prior([0.5] * n_dim, n_dim, []) return loglike, prior def _get_starting_points(self, n_walkers, variance=0.1): # Generate the starting points for the walkers by getting random # values for the parameters close to the current value # Fractional variance for randomization # (0.1 means var = 0.1 * value ) p0 = [] for i in range(n_walkers): this_p0 = [ x.get_randomized_value(variance) for x in list(self._free_parameters.values()) ] p0.append(this_p0) return p0 def _log_prior(self, trial_values): """Compute the sum of log-priors, used in the parallel tempering sampling""" # Compute the sum of the log-priors log_prior = 0 for i, (parameter_name, parameter) in enumerate(self._free_parameters.items()): prior_value = parameter.prior(trial_values[i]) if prior_value == 0: # Outside allowed region of parameter space return -np.inf else: parameter.value = trial_values[i] log_prior += math.log10(prior_value) return log_prior def _log_like(self, trial_values): """Compute the log-likelihood""" # Get the value of the log-likelihood for this parameters try: # Loop over each dataset and get the likelihood values for each set log_like_values = [ dataset.get_log_like() for dataset in list(self._data_list.values()) ] except ModelAssertionViolation: # Fit engine or sampler outside of allowed zone return -np.inf except: # We don't want to catch more serious issues raise # Sum the values of the log-like log_like = np.sum(log_like_values) if not np.isfinite(log_like): # Issue warning custom_warnings.warn( "Likelihood value is infinite for parameters %s" % trial_values, LikelihoodIsInfinite) return -np.inf return log_like @staticmethod def _calc_min_interval(x, alpha): """ Internal method to determine the minimum interval of a given width Assumes that x is sorted numpy array. :param a: a numpy array containing samples :param alpha: probability of type I error :returns: list containing min and max HDI """ n = len(x) cred_mass = 1.0 - alpha interval_idx_inc = int(np.floor(cred_mass * n)) n_intervals = n - interval_idx_inc interval_width = x[interval_idx_inc:] - x[:n_intervals] if len(interval_width) == 0: raise ValueError('Too few elements for interval calculation') min_idx = np.argmin(interval_width) hdi_min = x[min_idx] hdi_max = x[min_idx + interval_idx_inc] return hdi_min, hdi_max def _hpd(self, x, alpha=0.05): """Calculate highest posterior density (HPD) of array for given alpha. The HPD is the minimum width Bayesian credible interval (BCI). :param x: array containing MCMC samples :param alpha : Desired probability of type I error (defaults to 0.05) """ # Currently only 1D available. # future addition will fix this # Make a copy of trace # x = x.copy() # For multivariate node # if x.ndim > 1: # Transpose first, then sort # tx = np.transpose(x, list(range(x.ndim))[1:] + [0]) # dims = np.shape(tx) # Container list for intervals # intervals = np.resize(0.0, dims[:-1] + (2,)) # sx = np.sort(tx[index]) # Append to list # intervals[index] = self._calc_min_interval(sx, alpha) # Transpose back before returning # return np.array(intervals) # else: # Sort univariate node sx = np.sort(x) return np.array(self._calc_min_interval(sx, alpha))