def test_seed_returns_same_data(self):
data = make_data()
number_of_pixels_to_select = data.size // 2
seed = 243
subset1 = make_subset_data(data,
pixels=number_of_pixels_to_select,
seed=seed)
subset2 = make_subset_data(data,
pixels=number_of_pixels_to_select,
seed=seed)
self.assertTrue(np.all(subset1.values == subset2.values))
def fit(self, model, data):
parameters = model._parameters
par_names = model._parameter_names
time_start = time.time()
if self.npixels is not None and self.new_pixels is None:
data = make_subset_data(data, pixels=self.npixels, seed=self.seed)
if self.popsize is None:
npars = len(parameters)
self.popsize = int(2 + npars + np.sqrt(npars))
# cma default popsize is 4+3*ln(n). Ours is larger for npars > 5.
if self.walker_initial_pos is None:
self.walker_initial_pos = model.generate_guess(self.popsize,
seed=self.seed)
obj_func = LnpostWrapper(model, data, self.new_pixels, True)
sampler = run_cma(obj_func.evaluate, parameters,
self.walker_initial_pos, self.weights, self.tols,
self.seed, self.parallel)
xrecent = sampler.logger.data['xrecent']
samples = xr.DataArray(
[xrecent[:, 5:]], dims=['walker', 'chain', 'parameter'],
coords={'parameter': par_names})
lnprobs = xr.DataArray([-xrecent[:, 4]], dims=['walker', 'chain'])
best_vals = sampler.best.get()[0]
diffs = sampler.result.stds
intervals = [UncertainValue(best_val, diff, name=par) for
best_val, diff, par in zip(best_vals, diffs, par_names)]
stop = dict(sampler.stop())
d_time = time.time() - time_start
kwargs = {'lnprobs': lnprobs, 'samples': samples,
'intervals': intervals, 'stop_condition': stop,
'popsize': self.popsize}
return FitResult(data, model, self, d_time, kwargs)
def fit(self, model, data):
"""
fit a model to some data
Parameters
----------
model : :class:`~holopy.fitting.model.Model` object
A model describing the scattering system which leads to your
data and the parameters to vary to fit it to the data
data : xarray.DataArray
The data to fit
Returns
-------
result : :class:`FitResult`
an object containing the best fit parameters and information
about the fit
"""
time_start = time.time()
parameters = model._parameters
if len(parameters) == 0:
raise MissingParameter('at least one parameter to fit')
if self.npixels is not None:
data = make_subset_data(data, pixels=self.npixels, seed=self.seed)
guess_prior = model.lnprior(
{par.name: par.guess
for par in parameters})
def residual(par_vals):
noise = model._find_noise(par_vals, data)
residuals = model._residuals(par_vals, data, noise).flatten()
prior = np.sqrt(guess_prior - model.lnprior(par_vals))
residuals = np.append(residuals, prior)
return residuals
fitted_pars, minimizer_info = self.minimize(parameters, residual)
if minimizer_info.status == 5:
setattr(minimizer_info, 'converged', False)
warnings.warn("Minimizer Convergence Failed, your results \
may not be correct.")
else:
setattr(minimizer_info, 'converged', True)
# Getting errors:
errors_rescaled = minimizer_info.perror
if errors_rescaled is None:
errors_rescaled = [0] * len(parameters)
errors = self.unscale_pars_from_minimizer(parameters, errors_rescaled)
intervals = [
UncertainValue(fitted_pars[name], errors[name], name=name)
for name in errors.keys()
]
d_time = time.time() - time_start
return FitResult(data, model, self, d_time, {
'intervals': intervals,
'mpfit_details': minimizer_info
})
def sample(self, model, data, nsamples=None, walker_initial_pos=None):
if nsamples is not None:
# deprecated as of 3.3
from holopy.fitting import fit_warning
fit_warning('EmceeStrategy(nsamples=X)',
'passing nsamples to EmceeStrategy.sample')
self.nsamples = nsamples
if walker_initial_pos is not None:
# deprecated as of 3.3
from holopy.fitting import fit_warning
fit_warning('EmceeStrategy(walker_initial_pos=X)',
'passing walker_initial_pos to EmceeStrategy.sample')
self.walker_initial_pos = walker_initial_pos
time_start = time.time()
if self.npixels is not None:
data = make_subset_data(data, pixels=self.npixels, seed=self.seed)
if self.walker_initial_pos is None:
self.walker_initial_pos = model.generate_guess(self.nwalkers,
seed=self.seed)
sampler = sample_emcee(model=model, data=data, nwalkers=self.nwalkers,
walker_initial_pos=self.walker_initial_pos,
nsamples=self.nsamples, parallel=self.parallel,
seed=self.seed)
samples = emcee_samples_DataArray(sampler, model._parameters)
lnprobs = emcee_lnprobs_DataArray(sampler)
d_time = time.time() - time_start
kwargs = {'lnprobs': lnprobs, 'samples': samples}
return SamplingResult(data, model, self, d_time, kwargs)
def lnposterior(self, par_vals, data, pixels=None):
"""
Compute the log-posterior probability of pars given data
Parameters
-----------
pars: dict(string, float)
Dictionary containing values for each parameter
data: xarray
The data to compute posterior against
pixels: int(optional)
Specify to use a random subset of all pixels in data
Returns
--------
lnposterior: float
"""
lnprior = self.lnprior(par_vals)
# prior is sometimes used to forbid thing like negative radius
# which will fail if you attempt to compute a hologram of, so
# don't try to compute likelihood where the prior already
# forbids you to be
if lnprior == -np.inf:
return lnprior
else:
if pixels is not None:
data = make_subset_data(data, pixels=pixels)
return lnprior + self.lnlike(par_vals, data)
def test_returns_correct_z_coords(self):
data = make_data()
number_of_pixels_to_select = 3
subset = make_subset_data(data, pixels=number_of_pixels_to_select)
subset_z_coords = subset.coords['z'].values
data_z_coords = data.coords['z'].values
self.assertTrue(np.all(subset_z_coords == data_z_coords))
def _lnposterior(self, pars, data, pixels=None):
"""
Internal function taking pars as a list only
"""
lnprior = self._lnprior(pars)
# prior is sometimes used to forbid thing like negative radius
# which will fail if you attempt to compute a hologram of, so
# don't try to compute likelihood where the prior already
# forbids you to be
if lnprior == -np.inf:
return lnprior
else:
if pixels is not None:
data = make_subset_data(data, pixels=pixels)
return lnprior + self._lnlike(pars, data)
def test_returns_correct_xy_coords(self):
data = make_data()
number_of_pixels_to_select = 3
subset = make_subset_data(data, pixels=number_of_pixels_to_select)
data_xy_indices = [
np.nonzero(data.values == from_subset)[1:] # index[0] is z
for from_subset in subset.values
]
for subset_index, data_xy_index in enumerate(data_xy_indices):
coords_from_subset = [
subset.coords[k].values[subset_index] for k in 'xy'
]
coords_from_data = [
data.coords[k].values[data_xy_index[which].squeeze()]
for which, k in enumerate('xy')
]
self.assertEqual(coords_from_subset, coords_from_data)
def _mcmc(self, best_fit, data, mcmc_kws=None, npixels=100):
if mcmc_kws is None:
mcmc_kws = self.DEFAULT_MCMC_PARAMS.copy()
subset_data = make_subset_data(data, pixels=npixels)
noise = self._estimate_noise_from(data)
params = best_fit.params
params.add('__lnsigma',
value=np.log(noise),
min=np.log(noise / 10),
max=np.log(noise * 10))
residuals_calculator = ResidualsCalculator(subset_data,
theory=self.theory)
minimizer = Minimizer(residuals_calculator.calc_residuals,
params,
nan_policy='omit',
fcn_kws={'data': subset_data})
if not self.quiet:
print("Sampling with emcee ({}, npixels: {})".format(
mcmc_kws, npixels))
self._update_mcmc_kwargs_with_pos(mcmc_kws, params)
with warnings.catch_warnings():
warnings.simplefilter("ignore", RuntimeWarning)
mcmc_result = minimizer.minimize(params=params,
method='emcee',
float_behavior='chi2',
is_weighted=False,
**mcmc_kws)
if not self.quiet:
print(report_fit(mcmc_result.params))
# Then the whole point of the ResidualsCalculator is storing the
# best-fit value:
best_params = {
'parameters': residuals_calculator.best_params,
'chisq': residuals_calculator.best_chisq
}
result = {
'mcmc_result': mcmc_result,
'lmfit_result': best_fit,
'best_result': best_params
}
return result
def fit(self, model, data):
"""
fit a model to some data
Parameters
----------
model : :class:`~holopy.fitting.model.Model` object
A model describing the scattering system which leads to your
data and the parameters to vary to fit it to the data
data : xarray.DataArray
The data to fit
Returns
-------
result : :class:`FitResult`
Contains the best fit parameters and information about the fit
"""
# timing decorator...
time_start = time.time()
parameters = model._parameters
if len(parameters) == 0:
raise MissingParameter('at least one parameter to fit')
if self.npixels is None:
data = flat(data)
else:
data = make_subset_data(data, pixels=self.npixels)
guess_lnprior = model.lnprior(
{par.name:par.guess for par in parameters})
def residual(rescaled_values):
unscaled_values = self.unscale_pars_from_minimizer(
parameters, rescaled_values)
noise = model._find_noise(unscaled_values, data)
residuals = model._residuals(unscaled_values, data, noise)
ln_prior = model.lnprior(unscaled_values) - guess_lnprior
zscore_prior = np.sqrt(2 * -ln_prior)
np.append(residuals, zscore_prior)
return residuals
# The only work here
fitted_pars, minimizer_info = self.minimize(parameters, residual)
if not minimizer_info.success:
warnings.warn("Minimizer Convergence Failed, your results \
may not be correct.")
unit_errors = self._calculate_unit_noise_errors_from_fit(minimizer_info)
noise = model._find_noise(fitted_pars, data)
errors_scaled = noise * unit_errors
errors = self.unscale_pars_from_minimizer(parameters, errors_scaled)
intervals = [
UncertainValue(
fitted_pars[par.name], errors[par.name], name=par.name)
for err, par in zip(errors, parameters)]
# timing decorator...
d_time = time.time() - time_start
kwargs = {'intervals': intervals, 'minimizer_info': minimizer_info}
return FitResult(data, model, self, d_time, kwargs)
def test_returns_elements_of_data(self):
data = make_data()
number_of_pixels_to_select = 3
subset = make_subset_data(data, pixels=number_of_pixels_to_select)
for datum in subset.values:
self.assertIn(datum, data.values.ravel())
def test_returns_correct_number_of_pixels_on_detector_points(self):
points = make_points()
number_of_pixels_to_select = 3
subset = make_subset_data(points, pixels=number_of_pixels_to_select)
self.assertEqual(subset.size, number_of_pixels_to_select)
def test_returns_correct_number_of_pixels(self):
data = make_data()
number_of_pixels_to_select = 3
subset = make_subset_data(data, pixels=number_of_pixels_to_select)
self.assertEqual(subset.size, number_of_pixels_to_select)
def test_returns_data_when_nothing_passed(self):
data = make_data()
subset = make_subset_data(data)
self.assertTrue(data is subset)