def test_seed_returns_same_data(self): data = make_data() number_of_pixels_to_select = data.size // 2 seed = 243 subset1 = make_subset_data(data, pixels=number_of_pixels_to_select, seed=seed) subset2 = make_subset_data(data, pixels=number_of_pixels_to_select, seed=seed) self.assertTrue(np.all(subset1.values == subset2.values))
def fit(self, model, data): parameters = model._parameters par_names = model._parameter_names time_start = time.time() if self.npixels is not None and self.new_pixels is None: data = make_subset_data(data, pixels=self.npixels, seed=self.seed) if self.popsize is None: npars = len(parameters) self.popsize = int(2 + npars + np.sqrt(npars)) # cma default popsize is 4+3*ln(n). Ours is larger for npars > 5. if self.walker_initial_pos is None: self.walker_initial_pos = model.generate_guess(self.popsize, seed=self.seed) obj_func = LnpostWrapper(model, data, self.new_pixels, True) sampler = run_cma(obj_func.evaluate, parameters, self.walker_initial_pos, self.weights, self.tols, self.seed, self.parallel) xrecent = sampler.logger.data['xrecent'] samples = xr.DataArray( [xrecent[:, 5:]], dims=['walker', 'chain', 'parameter'], coords={'parameter': par_names}) lnprobs = xr.DataArray([-xrecent[:, 4]], dims=['walker', 'chain']) best_vals = sampler.best.get()[0] diffs = sampler.result.stds intervals = [UncertainValue(best_val, diff, name=par) for best_val, diff, par in zip(best_vals, diffs, par_names)] stop = dict(sampler.stop()) d_time = time.time() - time_start kwargs = {'lnprobs': lnprobs, 'samples': samples, 'intervals': intervals, 'stop_condition': stop, 'popsize': self.popsize} return FitResult(data, model, self, d_time, kwargs)
def fit(self, model, data): """ fit a model to some data Parameters ---------- model : :class:`~holopy.fitting.model.Model` object A model describing the scattering system which leads to your data and the parameters to vary to fit it to the data data : xarray.DataArray The data to fit Returns ------- result : :class:`FitResult` an object containing the best fit parameters and information about the fit """ time_start = time.time() parameters = model._parameters if len(parameters) == 0: raise MissingParameter('at least one parameter to fit') if self.npixels is not None: data = make_subset_data(data, pixels=self.npixels, seed=self.seed) guess_prior = model.lnprior( {par.name: par.guess for par in parameters}) def residual(par_vals): noise = model._find_noise(par_vals, data) residuals = model._residuals(par_vals, data, noise).flatten() prior = np.sqrt(guess_prior - model.lnprior(par_vals)) residuals = np.append(residuals, prior) return residuals fitted_pars, minimizer_info = self.minimize(parameters, residual) if minimizer_info.status == 5: setattr(minimizer_info, 'converged', False) warnings.warn("Minimizer Convergence Failed, your results \ may not be correct.") else: setattr(minimizer_info, 'converged', True) # Getting errors: errors_rescaled = minimizer_info.perror if errors_rescaled is None: errors_rescaled = [0] * len(parameters) errors = self.unscale_pars_from_minimizer(parameters, errors_rescaled) intervals = [ UncertainValue(fitted_pars[name], errors[name], name=name) for name in errors.keys() ] d_time = time.time() - time_start return FitResult(data, model, self, d_time, { 'intervals': intervals, 'mpfit_details': minimizer_info })
def sample(self, model, data, nsamples=None, walker_initial_pos=None): if nsamples is not None: # deprecated as of 3.3 from holopy.fitting import fit_warning fit_warning('EmceeStrategy(nsamples=X)', 'passing nsamples to EmceeStrategy.sample') self.nsamples = nsamples if walker_initial_pos is not None: # deprecated as of 3.3 from holopy.fitting import fit_warning fit_warning('EmceeStrategy(walker_initial_pos=X)', 'passing walker_initial_pos to EmceeStrategy.sample') self.walker_initial_pos = walker_initial_pos time_start = time.time() if self.npixels is not None: data = make_subset_data(data, pixels=self.npixels, seed=self.seed) if self.walker_initial_pos is None: self.walker_initial_pos = model.generate_guess(self.nwalkers, seed=self.seed) sampler = sample_emcee(model=model, data=data, nwalkers=self.nwalkers, walker_initial_pos=self.walker_initial_pos, nsamples=self.nsamples, parallel=self.parallel, seed=self.seed) samples = emcee_samples_DataArray(sampler, model._parameters) lnprobs = emcee_lnprobs_DataArray(sampler) d_time = time.time() - time_start kwargs = {'lnprobs': lnprobs, 'samples': samples} return SamplingResult(data, model, self, d_time, kwargs)
def lnposterior(self, par_vals, data, pixels=None): """ Compute the log-posterior probability of pars given data Parameters ----------- pars: dict(string, float) Dictionary containing values for each parameter data: xarray The data to compute posterior against pixels: int(optional) Specify to use a random subset of all pixels in data Returns -------- lnposterior: float """ lnprior = self.lnprior(par_vals) # prior is sometimes used to forbid thing like negative radius # which will fail if you attempt to compute a hologram of, so # don't try to compute likelihood where the prior already # forbids you to be if lnprior == -np.inf: return lnprior else: if pixels is not None: data = make_subset_data(data, pixels=pixels) return lnprior + self.lnlike(par_vals, data)
def test_returns_correct_z_coords(self): data = make_data() number_of_pixels_to_select = 3 subset = make_subset_data(data, pixels=number_of_pixels_to_select) subset_z_coords = subset.coords['z'].values data_z_coords = data.coords['z'].values self.assertTrue(np.all(subset_z_coords == data_z_coords))
def _lnposterior(self, pars, data, pixels=None): """ Internal function taking pars as a list only """ lnprior = self._lnprior(pars) # prior is sometimes used to forbid thing like negative radius # which will fail if you attempt to compute a hologram of, so # don't try to compute likelihood where the prior already # forbids you to be if lnprior == -np.inf: return lnprior else: if pixels is not None: data = make_subset_data(data, pixels=pixels) return lnprior + self._lnlike(pars, data)
def test_returns_correct_xy_coords(self): data = make_data() number_of_pixels_to_select = 3 subset = make_subset_data(data, pixels=number_of_pixels_to_select) data_xy_indices = [ np.nonzero(data.values == from_subset)[1:] # index[0] is z for from_subset in subset.values ] for subset_index, data_xy_index in enumerate(data_xy_indices): coords_from_subset = [ subset.coords[k].values[subset_index] for k in 'xy' ] coords_from_data = [ data.coords[k].values[data_xy_index[which].squeeze()] for which, k in enumerate('xy') ] self.assertEqual(coords_from_subset, coords_from_data)
def _mcmc(self, best_fit, data, mcmc_kws=None, npixels=100): if mcmc_kws is None: mcmc_kws = self.DEFAULT_MCMC_PARAMS.copy() subset_data = make_subset_data(data, pixels=npixels) noise = self._estimate_noise_from(data) params = best_fit.params params.add('__lnsigma', value=np.log(noise), min=np.log(noise / 10), max=np.log(noise * 10)) residuals_calculator = ResidualsCalculator(subset_data, theory=self.theory) minimizer = Minimizer(residuals_calculator.calc_residuals, params, nan_policy='omit', fcn_kws={'data': subset_data}) if not self.quiet: print("Sampling with emcee ({}, npixels: {})".format( mcmc_kws, npixels)) self._update_mcmc_kwargs_with_pos(mcmc_kws, params) with warnings.catch_warnings(): warnings.simplefilter("ignore", RuntimeWarning) mcmc_result = minimizer.minimize(params=params, method='emcee', float_behavior='chi2', is_weighted=False, **mcmc_kws) if not self.quiet: print(report_fit(mcmc_result.params)) # Then the whole point of the ResidualsCalculator is storing the # best-fit value: best_params = { 'parameters': residuals_calculator.best_params, 'chisq': residuals_calculator.best_chisq } result = { 'mcmc_result': mcmc_result, 'lmfit_result': best_fit, 'best_result': best_params } return result
def fit(self, model, data): """ fit a model to some data Parameters ---------- model : :class:`~holopy.fitting.model.Model` object A model describing the scattering system which leads to your data and the parameters to vary to fit it to the data data : xarray.DataArray The data to fit Returns ------- result : :class:`FitResult` Contains the best fit parameters and information about the fit """ # timing decorator... time_start = time.time() parameters = model._parameters if len(parameters) == 0: raise MissingParameter('at least one parameter to fit') if self.npixels is None: data = flat(data) else: data = make_subset_data(data, pixels=self.npixels) guess_lnprior = model.lnprior( {par.name:par.guess for par in parameters}) def residual(rescaled_values): unscaled_values = self.unscale_pars_from_minimizer( parameters, rescaled_values) noise = model._find_noise(unscaled_values, data) residuals = model._residuals(unscaled_values, data, noise) ln_prior = model.lnprior(unscaled_values) - guess_lnprior zscore_prior = np.sqrt(2 * -ln_prior) np.append(residuals, zscore_prior) return residuals # The only work here fitted_pars, minimizer_info = self.minimize(parameters, residual) if not minimizer_info.success: warnings.warn("Minimizer Convergence Failed, your results \ may not be correct.") unit_errors = self._calculate_unit_noise_errors_from_fit(minimizer_info) noise = model._find_noise(fitted_pars, data) errors_scaled = noise * unit_errors errors = self.unscale_pars_from_minimizer(parameters, errors_scaled) intervals = [ UncertainValue( fitted_pars[par.name], errors[par.name], name=par.name) for err, par in zip(errors, parameters)] # timing decorator... d_time = time.time() - time_start kwargs = {'intervals': intervals, 'minimizer_info': minimizer_info} return FitResult(data, model, self, d_time, kwargs)
def test_returns_elements_of_data(self): data = make_data() number_of_pixels_to_select = 3 subset = make_subset_data(data, pixels=number_of_pixels_to_select) for datum in subset.values: self.assertIn(datum, data.values.ravel())
def test_returns_correct_number_of_pixels_on_detector_points(self): points = make_points() number_of_pixels_to_select = 3 subset = make_subset_data(points, pixels=number_of_pixels_to_select) self.assertEqual(subset.size, number_of_pixels_to_select)
def test_returns_correct_number_of_pixels(self): data = make_data() number_of_pixels_to_select = 3 subset = make_subset_data(data, pixels=number_of_pixels_to_select) self.assertEqual(subset.size, number_of_pixels_to_select)
def test_returns_data_when_nothing_passed(self): data = make_data() subset = make_subset_data(data) self.assertTrue(data is subset)