def test_cached_gaussian_prior(self):
"""Check the cached_gaussian prior."""
settings = get_minimal_settings()
self.assertRaises(TypeError,
priors.GaussianCached,
prior_scale=10,
unexpected=0)
with warnings.catch_warnings(record=True) as war:
warnings.simplefilter("always")
settings.prior = priors.GaussianCached(prior_scale=10,
save_dict=True,
n_dim=settings.n_dim,
cache_dir=TEST_CACHE_DIR,
interp_density=10,
logx_min=-30)
self.assertEqual(len(war), 1)
# Test inside and outside cached regime (logx<-10).
# Need fairly low number of places
for logx in [-1, -11]:
self.assertAlmostEqual(settings.logx_given_logl(
settings.logl_given_logx(logx)),
logx,
places=3)
# Test array version of the function too
logx = np.asarray([-2])
self.assertAlmostEqual(settings.logx_given_logl(
settings.logl_given_logx(logx)[0]),
logx[0],
places=12)
settings.get_settings_dict()
# Generate NS run using get_run_data to check it checks the cache
# before submitting process to parallel apply
ns_run = ns.get_run_data(settings, 1, load=False, save=False)[0]
values = nestcheck.ns_run_utils.run_estimators(ns_run, ESTIMATOR_LIST)
self.assertFalse(np.any(np.isnan(values)))
# check the argument options and messages for interp_r_logx_dict
with warnings.catch_warnings(record=True) as war:
warnings.simplefilter("always")
self.assertRaises(
TypeError,
perfectns.cached_gaussian_prior.interp_r_logx_dict,
2000,
10,
logx_min=-100,
interp_density=1,
unexpected=0)
self.assertEqual(len(war), 1)
self.assertRaises(TypeError,
perfectns.cached_gaussian_prior.interp_r_logx_dict,
200,
10,
logx_min=-100,
interp_density=1,
unexpected=0)
def test_get_run_data_caching(self):
settings = get_minimal_settings()
settings.dynamic_goal = None
settings.n_samples_max = 100
ns.get_run_data(settings,
1,
save=True,
load=True,
check_loaded_settings=True,
cache_dir=TEST_CACHE_DIR)
# test loading and checking settings
ns.get_run_data(settings,
1,
save=True,
load=True,
check_loaded_settings=True,
cache_dir=TEST_CACHE_DIR)
# test loading and checking settings when settings are not the same
# this only works for changing a setting which dosnt affect the save
# name
settings.dynamic_goal = 0
ns.get_run_data(settings,
1,
save=True,
load=True,
check_loaded_settings=True,
cache_dir=TEST_CACHE_DIR)
settings.n_samples_max += 1
with warnings.catch_warnings(record=True) as war:
warnings.simplefilter("always")
ns.get_run_data(settings,
1,
save=True,
load=True,
check_loaded_settings=True,
cache_dir=TEST_CACHE_DIR)
self.assertEqual(len(war), 1)
def plot_dynamic_nlive(dynamic_goals, settings_in, **kwargs):
"""
Plot the allocations of live points as a function of logX for different
dynamic_goal settings.
Plots also include analytically calculated distributions of relative
posterior mass and relative posterior mass remaining.
Parameters
----------
dynamic_goals: list of ints or None
dynamic_goal setting values to plot.
settings_in: PerfectNSSettings object
tuned_dynamic_ps: list of bools, optional
tuned_dynamic_ps settings corresponding to each dynamic goal settings.
Defaults to False for all dynamic goals.
logx_min: float, optional
Lower limit of logx axis. If not specified this is set to the lowest
logx reached by any of the runs.
load: bool, optional
Should the nested sampling runs be loaded from cache if available?
save: bool, optional
Should the nested sampling runs be cached?
ymax: bool, optional
Maximum value for plot's nlive axis (yaxis).
n_run: int, optional
How many runs to plot for each dynamic goal.
npoints: int, optional
How many points to have in the logx array used to calculate and plot
analytical weights.
figsize: tuple, optional
Size of figure in inches.
Returns
-------
fig: matplotlib figure
"""
tuned_dynamic_ps = kwargs.pop('tuned_dynamic_ps',
[False] * len(dynamic_goals))
save = kwargs.pop('save', True)
load = kwargs.pop('load', True)
npoints = kwargs.pop('npoints', 100)
n_run = kwargs.pop('n_run', 10)
# Confine settings edits to within this function
settings = copy.deepcopy(settings_in)
run_dict = {}
# work out n_samples_max from first set of runs
n_sample_stats = np.zeros((len(dynamic_goals), 2))
method_names = [] # use list to store labels so order is preserved
for i, dg in enumerate(dynamic_goals):
print('dynamic_goal=' + str(dg))
# Make label
if dg is None:
label = 'standard'
else:
label = 'dynamic $G=' + str(dg) + '$'
if tuned_dynamic_ps[i] is True:
label = 'tuned ' + label
method_names.append(label)
settings.dynamic_goal = dg
settings.tuned_dynamic_p = tuned_dynamic_ps[i]
temp_runs = ns.get_run_data(settings, n_run, parallel=True,
load=load, save=save)
n_samples = np.asarray([run['logl'].shape[0] for run in temp_runs])
n_sample_stats[i, 0] = np.mean(n_samples)
n_sample_stats[i, 1] = np.std(n_samples, ddof=1)
if i == 0 and settings.n_samples_max is None:
settings.n_samples_max = int(n_sample_stats[0, 0] *
(settings.nlive_const - 1) /
settings.nlive_const)
run_dict[label] = temp_runs
print('mean samples per run:', n_sample_stats[i, 0],
'std:', n_sample_stats[i, 1])
fig = nestcheck.plots.plot_run_nlive(
method_names, run_dict, post_mass_norm='dynamic $G=1$',
npoints=npoints, logx_given_logl=settings.logx_given_logl,
logl_given_logx=settings.logl_given_logx,
cum_post_mass_norm='dynamic $G=0$', **kwargs)
# Plot the tuned posterior mass
if 'tuned dynamic $G=1$' in method_names:
ax = fig.axes[0]
logx = np.linspace(ax.get_xlim()[0], ax.get_xlim()[1], npoints)
# Get expected magnitude of parameter
# This is not defined for logx=0 so exclude final value of logx
param_exp = settings.r_given_logx(logx[:-1]) / np.sqrt(settings.n_dim)
# Tuned weight is the relative posterior mass times the expected
# magnitude of the paramer being considered
w_an = nestcheck.plots.rel_posterior_mass(
logx, settings.logl_given_logx(logx))
w_tuned = w_an[:-1] * param_exp
w_tuned /= np.trapz(w_tuned, x=logx[:-1])
# Get the normalising constant
integrals = np.zeros(len(run_dict['tuned dynamic $G=1$']))
for nr, run in enumerate(run_dict['tuned dynamic $G=1$']):
logx_run = settings.logx_given_logl(run['logl'])
logx[0] = 0 # to make lines extend all the way to the end
# for normalising analytic weight lines
integrals[nr] = -np.trapz(run['nlive_array'], x=logx_run)
w_tuned *= np.mean(integrals)
# Plot the tuned posterior mass
ax.plot(logx[:-1], w_tuned, linewidth=2, label='tuned importance',
linestyle='-.', dashes=(6, 1.5, 1, 1.5), color='k')
ax.legend(ncol=3)
return fig
def get_bootstrap_results(n_run, n_simulate, estimator_list, settings,
**kwargs):
"""
Generate data frame showing the standard deviations of the results of
repeated calculations and estimated sampling errors from bootstrap
resampling.
This function was used for Table 5 in 'Dynamic nested sampling: an improved
algorithm for nested sampling parameter estimation and evidence
calculation' (Higson et al., 2019). See the paper for more details.
Parameters
----------
n_run: int
how many runs to use
n_simulate: int
how many times to resample the nested sampling run in each bootstrap
standard deviation estimate.
estimator_list: list of estimator objects
settings: PerfectNSSettings object
load: bool, optional
should run data and results be loaded if available?
save: bool, optional
should run data and results be saved?
parallel: bool, optional
cache_dir: str, optional
Directory to use for caching.
add_sim_method: bool, optional
should we also calculate standard deviations using the simulated
weights method for comparison with bootstrap resampling? This method is
inaccurate for parameter estimation.
n_simulate_ci: int, optional
how many times to resample the nested sampling run in each bootstrap
credible interval estimate. These may require more simulations than the
standard deviation estimate.
run_random_seeds: list, optional
list of random seeds to use for generating runs.
n_run_ci: int, optional
how many runs to use for each credible interval estimate. You may want
to set this to lower than n_run if n_simulate_ci is large as otherwise
the credible interval estimate may take a long time.
cred_int: float, optional
one-tailed credible interval to calculate
max_workers: int or None, optional
Number of processes.
If max_workers is None then concurrent.futures.ProcessPoolExecutor
defaults to using the number of processors of the machine.
N.B. If max_workers=None and running on supercomputer clusters with
multiple nodes, this may default to the number of processors on a
single node and therefore there will be no speedup from multiple
nodes (must specify manually in this case).
Returns
-------
results: pandas data frame
results data frame.
Contains two columns for each estimator - the second column (with
'_unc' appended to the title) shows the numerical uncertainty in the
first column.
Contains rows:
true values: analytical values of estimators for this likelihood
and posterior if available
repeats mean: mean calculation result
repeats std: standard deviation of calculation results
bs std / repeats std: mean bootstrap standard deviation estimate as
a fraction of the standard deviation of repeated results.
bs estimate % variation: standard deviation of bootstrap estimates
as a percentage of the mean estimate.
[only if add sim method is True]:
sim std / repeats std: as for 'bs std / repeats std' but with
simulation method standard deviation estimates.
sim estimate % variation: as for 'bs estimate % variation' but
with simulation method standard deviation estimates.
bs [cred_int] CI: mean bootstrap credible interval estimate.
bs +-1std % coverage: % of calculation results falling within +- 1
mean bootstrap standard deviation estimate of the mean.
bs [cred_int] CI % coverage: % of calculation results which are
less than the mean bootstrap credible interval estimate.
"""
load = kwargs.pop('load', False)
save = kwargs.pop('save', False)
max_workers = kwargs.pop('max_workers', None)
ninit_sep = kwargs.pop('ninit_sep', True)
parallel = kwargs.pop('parallel', True)
cache_dir = kwargs.pop('cache_dir', 'cache')
add_sim_method = kwargs.pop('add_sim_method', False)
n_simulate_ci = kwargs.pop('n_simulate_ci', n_simulate)
n_run_ci = kwargs.pop('n_run_ci', n_run)
cred_int = kwargs.pop('cred_int', 0.95)
run_random_seeds = kwargs.pop('run_random_seeds', list(range(n_run)))
if kwargs:
raise TypeError('Unexpected **kwargs: {0}'.format(kwargs))
# make save_name
save_root = ('bootstrap_results_' + str(n_simulate) + 'nsim_' +
str(ninit_sep) + 'sep')
save_root += '_' + settings.save_name()
save_root += '_' + str(n_run) + 'reps'
save_file = cache_dir + '/' + save_root + '.pkl'
# try loading results
if load:
try:
return pd.read_pickle(save_file)
except OSError:
pass
# start function
est_names = [est.latex_name for est in estimator_list]
# generate runs
run_list = ns.get_run_data(settings, n_run, save=save, load=load,
random_seeds=run_random_seeds,
cache_dir=cache_dir,
max_workers=max_workers,
parallel=parallel)
# sort in order of random seeds. This makes credible intervals results
# reproducable even when only the first section of run_list is used.
run_list = sorted(run_list, key=lambda r: r['random_seed'])
rep_values = pu.parallel_apply(
nestcheck.ns_run_utils.run_estimators, run_list,
func_args=(estimator_list,), max_workers=max_workers,
parallel=parallel)
results = pf.summary_df_from_list(rep_values, est_names)
new_index = ['repeats ' +
results.index.get_level_values('calculation type'),
results.index.get_level_values('result type')]
results.set_index(new_index, inplace=True)
results.index.rename('calculation type', level=0, inplace=True)
# get bootstrap std estimate
bs_values = pu.parallel_apply(
nestcheck.error_analysis.run_std_bootstrap, run_list,
func_args=(estimator_list,),
func_kwargs={'n_simulate': n_simulate},
max_workers=max_workers,
parallel=parallel)
bs_df = pf.summary_df_from_list(bs_values, est_names)
# Get the mean bootstrap std estimate as a fraction of the std measured
# from repeated calculations.
results.loc[('bs std / repeats std', 'value'), :] = \
(bs_df.loc[('mean', 'value')] / results.loc[('repeats std', 'value')])
bs_std_ratio_unc = pf.array_ratio_std(
bs_df.loc[('mean', 'value')],
bs_df.loc[('mean', 'uncertainty')],
results.loc[('repeats std', 'value')],
results.loc[('repeats std', 'uncertainty')])
results.loc[('bs std / repeats std', 'uncertainty'), :] = \
bs_std_ratio_unc
# Get the fractional variation of std estimates
# multiply by 100 to express as a percentage
results.loc[('bs estimate % variation', 'value'), :] = \
100 * bs_df.loc[('std', 'value')] / bs_df.loc[('mean', 'value')]
results.loc[('bs estimate % variation', 'uncertainty'), :] = \
100 * bs_df.loc[('std', 'uncertainty')] / bs_df.loc[('mean', 'value')]
if add_sim_method:
# get std from simulation estimate
sim_values = pu.parallel_apply(
nestcheck.error_analysis.run_std_simulate, run_list,
func_args=(estimator_list,),
func_kwargs={'n_simulate': n_simulate},
max_workers=max_workers,
parallel=parallel)
sim_df = pf.summary_df_from_list(sim_values, est_names)
# Get the mean simulation std estimate as a fraction of the std
# measured from repeated calculations.
results.loc[('sim std / repeats std', 'value'), :] = \
(sim_df.loc[('mean', 'value')] /
results.loc[('repeats std', 'value')])
sim_std_ratio_unc = pf.array_ratio_std(
sim_df.loc[('mean', 'value')],
sim_df.loc[('mean', 'uncertainty')],
results.loc[('repeats std', 'value')],
results.loc[('repeats std', 'uncertainty')])
results.loc[('sim std / repeats std', 'uncertainty'), :] = \
sim_std_ratio_unc
# Get the fractional variation of std estimates
# Multiply by 100 to express as a percentage
results.loc[('sim estimate % variation', 'value'), :] = \
100 * sim_df.loc[('std', 'value')] / sim_df.loc[('mean', 'value')]
results.loc[('sim estimate % variation', 'uncertainty'), :] = \
(100 * sim_df.loc[('std', 'uncertainty')] /
sim_df.loc[('mean', 'value')])
# get bootstrap CI estimates
bs_cis = pu.parallel_apply(
nestcheck.error_analysis.run_ci_bootstrap, run_list[:n_run_ci],
func_args=(estimator_list,),
func_kwargs={'n_simulate': n_simulate_ci,
'cred_int': cred_int,
'random_seeds': range(n_simulate_ci)},
max_workers=max_workers, parallel=parallel)
bs_ci_df = pf.summary_df_from_list(bs_cis, est_names)
results.loc[('bs ' + str(cred_int) + ' CI', 'value'), :] = \
bs_ci_df.loc[('mean', 'value')]
results.loc[('bs ' + str(cred_int) + ' CI', 'uncertainty'), :] = \
bs_ci_df.loc[('mean', 'uncertainty')]
# add coverage for +- 1 bootstrap std estimate
max_value = (results.loc[('repeats mean', 'value')].values
+ bs_df.loc[('mean', 'value')].values)
min_value = (results.loc[('repeats mean', 'value')].values
- bs_df.loc[('mean', 'value')].values)
rep_values_array = np.stack(rep_values, axis=1)
assert rep_values_array.shape == (len(estimator_list), n_run)
coverage = np.zeros(rep_values_array.shape[0])
for i, _ in enumerate(coverage):
ind = np.where((rep_values_array[i, :] > min_value[i]) &
(rep_values_array[i, :] < max_value[i]))
coverage[i] = ind[0].shape[0] / rep_values_array.shape[1]
# multiply by 100 to express as a percentage
results.loc[('bs +-1std % coverage', 'value'), :] = coverage * 100
# add credible interval coverage
max_value = results.loc[('bs ' + str(cred_int) + ' CI', 'value')].values
ci_coverage = np.zeros(len(estimator_list))
for i, _ in enumerate(coverage):
ind = np.where(rep_values_array[i, :] < max_value[i])
ci_coverage[i] = ind[0].shape[0] / rep_values_array.shape[1]
# multiply by 100 to express as a percentage
results.loc[('bs ' + str(cred_int) + ' CI % coverage', 'value'), :] = \
(ci_coverage * 100)
if save:
# save the results data frame
print('get_bootstrap_results: results saved to\n' + save_file)
results.to_pickle(save_file)
return results
def get_dynamic_results(n_run, dynamic_goals_in, estimator_list_in,
settings_in, **kwargs):
"""
Generate data frame showing the standard deviations of the results of
repeated calculations and efficiency gains (ratios of variances of results
calculations) from different dynamic goals. To make the comparison fair,
for dynamic nested sampling settings.n_samples_max is set to slightly below
the mean number of samples used by standard nested sampling.
This function was used for Tables 1, 2, 3 and 4, as well as to generate the
results shown in figures 6 and 7 of 'Dynamic nested sampling: an improved
algorithm for nested sampling parameter estimation and evidence
calculation' (Higson et al., 2019). See the paper for a more detailed
description.
Parameters
----------
n_run: int
how many runs to use
dynamic_goals_in: list of floats
which dynamic goals to test
estimator_list_in: list of estimator objects
settings_in: PerfectNSSettings object
load: bool, optional
should run data and results be loaded if available?
save: bool, optional
should run data and results be saved?
overwrite_existing: bool, optional
if a file exists already but we generate new run data, should we
overwrite the existing file when saved?
run_random_seeds: list, optional
list of random seeds to use for generating runs.
parallel: bool, optional
cache_dir: str, optional
Directory to use for caching.
tuned_dynamic_ps: list of bools, same length as dynamic_goals_in, optional
max_workers: int or None, optional
Number of processes.
If max_workers is None then concurrent.futures.ProcessPoolExecutor
defaults to using the number of processors of the machine.
N.B. If max_workers=None and running on supercomputer clusters with
multiple nodes, this may default to the number of processors on a
single node and therefore there will be no speedup from multiple
nodes (must specify manually in this case).
Returns
-------
results: pandas data frame
results data frame.
Contains rows:
mean [dynamic goal]: mean calculation result for standard nested
sampling and dynamic nested sampling with each input dynamic
goal.
std [dynamic goal]: standard deviation of results for standard
nested sampling and dynamic nested sampling with each input
dynamic goal.
gain [dynamic goal]: the efficiency gain (computational speedup)
from dynamic nested sampling compared to standard nested
sampling. This equals (variance of standard results) /
(variance of dynamic results); see the dynamic nested
sampling paper for more details.
"""
load = kwargs.pop('load', False)
save = kwargs.pop('save', False)
max_workers = kwargs.pop('max_workers', None)
parallel = kwargs.pop('parallel', True)
cache_dir = kwargs.pop('cache_dir', 'cache')
overwrite_existing = kwargs.pop('overwrite_existing', True)
run_random_seeds = kwargs.pop('run_random_seeds', list(range(n_run)))
tuned_dynamic_ps = kwargs.pop('tuned_dynamic_ps',
[False] * len(dynamic_goals_in))
assert len(tuned_dynamic_ps) == len(dynamic_goals_in)
for goal in dynamic_goals_in:
assert goal is not None, \
'Goals should be dynamic - standard NS already included'
# Add a standard nested sampling run for comparison:
dynamic_goals = [None] + dynamic_goals_in
tuned_dynamic_ps = [False] + tuned_dynamic_ps
if kwargs:
raise TypeError('Unexpected **kwargs: {0}'.format(kwargs))
# Make a copy of the input settings to stop us editing them
settings = copy.deepcopy(settings_in)
# make save_name
save_root = 'dynamic_test'
for dg in dynamic_goals_in:
save_root += '_' + str(dg).replace('.', '_')
save_root += '_' + settings.save_name(include_dg=False)
save_root += '_' + str(n_run) + 'reps'
save_file = cache_dir + '/' + save_root + '.pkl'
# try loading results
if load:
try:
return pd.read_pickle(save_file)
except OSError:
print('Could not load file: ' + save_file)
# start function
# --------------
# get info on the number of samples taken in each run as well
estimator_list = [e.CountSamples()] + estimator_list_in
est_names = [est.latex_name for est in estimator_list]
method_names = []
method_values = []
assert dynamic_goals[0] is None, (
'Need to start with standard ns to calculate efficiency gains')
for i, dynamic_goal in enumerate(dynamic_goals):
# set up settings
settings.dynamic_goal = dynamic_goal
settings.tuned_dynamic_p = tuned_dynamic_ps[i]
# if we have already done the standard calculation, set n_samples_max
# for dynamic calculations so it is slightly smaller than the number
# of samples the standard calculation used to ensure a fair comparison
# of performance. Otherwise dynamic nested sampling will end up using
# more samples than standard nested sampling as it does not terminate
# until after the number of samples is greater than n_samples_max.
if i != 0 and settings.dynamic_goal is not None:
assert dynamic_goals[0] is None
assert isinstance(estimator_list[0], e.CountSamples)
n_samples_max = np.mean(np.asarray([val[0] for val in
method_values[0]]))
# This factor is a function of the dynamic goal as typically
# evidence calculations have longer additional threads than
# parameter estimation calculations.
reduce_factor = 1 - ((1.5 - 0.5 * settings.dynamic_goal) *
(settings.nbatch / settings.nlive_const))
settings.n_samples_max = int(n_samples_max * reduce_factor)
print('dynamic_goal=' + str(settings.dynamic_goal),
'n_samples_max=' + str(settings.n_samples_max))
# get a name for this calculation method
if dynamic_goal is None:
method_names.append('standard')
else:
method_names.append('dynamic $G=' +
str(settings.dynamic_goal) + '$')
if settings.tuned_dynamic_p is True:
method_names[-1] += ' tuned'
# generate runs and get results
run_list = ns.get_run_data(settings, n_run, parallel=parallel,
random_seeds=run_random_seeds,
load=load, save=save,
max_workers=max_workers,
cache_dir=cache_dir,
overwrite_existing=overwrite_existing)
method_values.append(pu.parallel_apply(
nestcheck.ns_run_utils.run_estimators, run_list,
func_args=(estimator_list,), max_workers=max_workers,
parallel=parallel))
results = pf.efficiency_gain_df(method_names, method_values, est_names)
if save:
# save the results data frame
print('get_dynamic_results: saving results to\n' + save_file)
results.to_pickle(save_file)
return results