def get_samples_snl(exp_desc, seed):
"""
Generates MCMC samples for a given SNL experiment.
"""
assert isinstance(exp_desc.inf, ed.SNL_Descriptor)
res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'mmd')
if os.path.exists(res_file + '.pkl'):
all_samples = util.io.load(res_file)
else:
exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0')
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(exp_desc)
_, _, all_nets = util.io.load(os.path.join(exp_dir, 'results'))
all_samples = []
for net in all_nets:
net.reset_theano_functions()
log_posterior = lambda t: net.eval([t, obs_xs]) + prior.eval(t)
sampler = mcmc.SliceSampler(true_ps, log_posterior, thin=thin)
sampler.gen(burnin, rng=rng) # burn in
samples = sampler.gen(n_mcmc_samples, rng=rng)
all_samples.append(samples)
util.io.save(all_samples, res_file)
return all_samples
def get_mmd_snpe(exp_desc, seed):
"""
Calculates the MMD for a given SNPE experiment.
"""
assert isinstance(exp_desc.inf, ed.SNPE_MDN_Descriptor)
res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'mmd')
if os.path.exists(res_file + '.pkl'):
all_mmds = util.io.load(res_file)
else:
exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0')
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(exp_desc)
true_samples = get_true_samples(seed)
scale = util.math.median_distance(true_samples)
all_posteriors, _, _, _ = util.io.load(os.path.join(exp_dir, 'results'))
all_mmds = []
for posterior in all_posteriors[1:]:
samples = posterior.gen(n_mcmc_samples, rng=rng)
err = two_sample.sq_maximum_mean_discrepancy(samples, true_samples, scale=scale)
all_mmds.append(err)
util.io.save(all_mmds, res_file)
return all_mmds
def get_samples_nde(exp_desc, seed):
"""
Generates MCMC samples for a given NDE experiment.
"""
assert isinstance(exp_desc.inf, ed.NDE_Descriptor)
res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'mmd')
if os.path.exists(res_file + '.pkl'):
samples = util.io.load(res_file)
else:
exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0')
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(exp_desc)
net = util.io.load(os.path.join(exp_dir, 'model'))
log_posterior = lambda t: net.eval([t, obs_xs]) + prior.eval(t)
sampler = mcmc.SliceSampler(true_ps, log_posterior, thin=thin)
sampler.gen(burnin, rng=rng) # burn in
samples = sampler.gen(n_mcmc_samples, rng=rng)
util.io.save(samples, res_file)
return samples
def get_mmd_sl(exp_desc, seed):
"""
Calculates the MMD for a given synth likelihood experiment.
"""
assert isinstance(exp_desc.inf, ed.SynthLik_Descriptor)
res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'mmd')
if os.path.exists(res_file + '.pkl'):
err, n_sims = util.io.load(res_file)
else:
exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0')
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(exp_desc)
samples, n_sims = util.io.load(os.path.join(exp_dir, 'results'))
true_samples = get_true_samples(seed)
scale = util.math.median_distance(true_samples)
err = two_sample.sq_maximum_mean_discrepancy(samples, true_samples, scale=scale)
util.io.save((err, n_sims), res_file)
return err, n_sims
def get_mmd_postprop(exp_desc, seed):
"""
Calculates the MMD for a given Post Prop experiment.
"""
assert isinstance(exp_desc.inf, ed.PostProp_Descriptor)
res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'mmd')
if os.path.exists(res_file + '.pkl'):
all_prop_mmds, post_err = util.io.load(res_file)
else:
exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0')
print(exp_dir)
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(exp_desc)
true_samples = get_true_samples(seed)
scale = util.math.median_distance(true_samples)
all_proposals, posterior, _, _ = util.io.load(os.path.join(exp_dir, 'results'))
all_prop_mmds = []
for i, proposal in enumerate(all_proposals[1:]):
samples = proposal.gen(n_mcmc_samples, rng=rng)
prop_err = two_sample.sq_maximum_mean_discrepancy(samples, true_samples, scale=scale)
all_prop_mmds.append(prop_err)
samples = posterior.gen(n_mcmc_samples, rng=rng)
post_err = two_sample.sq_maximum_mean_discrepancy(samples, true_samples, scale=scale)
util.io.save((all_prop_mmds, post_err), res_file)
return all_prop_mmds, post_err
def get_err_snpe(exp_desc, sim, seed):
"""
Calculates the error for a given SNPE experiment.
"""
assert isinstance(exp_desc.inf, ed.SNPE_MDN_Descriptor)
res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'err')
if os.path.exists(res_file + '.pkl'):
all_errs = util.io.load(res_file)
else:
exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0')
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(exp_desc)
true_ps, _ = sim.get_ground_truth()
all_posteriors, _, _, _ = util.io.load(os.path.join(exp_dir, 'results'))
all_errs = []
for posterior in all_posteriors[1:]:
samples = posterior.gen(n_mcmc_samples, rng=rng)
err = calc_err(true_ps, samples)
all_errs.append(err)
util.io.save(all_errs, res_file)
return all_errs
def get_mmd_smc(exp_desc, seed):
"""
Calculates the MMD for a given SMC ABC experiment.
"""
assert isinstance(exp_desc.inf, ed.SMC_ABC_Descriptor)
res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'mmd')
if os.path.exists(res_file + '.pkl'):
all_mmds, all_n_sims = util.io.load(res_file)
else:
exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0')
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(exp_desc)
true_samples = get_true_samples(seed)
scale = util.math.median_distance(true_samples)
all_samples, all_log_weights, _, _, all_n_sims = util.io.load(os.path.join(exp_dir, 'results'))
all_mmds = []
for samples, log_weights in zip(all_samples, all_log_weights):
weights = np.exp(log_weights)
err = two_sample.sq_maximum_mean_discrepancy(xs=samples, ys=true_samples, wxs=weights, scale=scale)
all_mmds.append(err)
util.io.save((all_mmds, all_n_sims), res_file)
return all_mmds, all_n_sims
def get_err_sl(exp_desc, sim, seed):
"""
Calculates the error for a given synth likelihood experiment.
"""
assert isinstance(exp_desc.inf, ed.SynthLik_Descriptor)
res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'err')
if os.path.exists(res_file + '.pkl'):
err, n_sims = util.io.load(res_file)
else:
exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0')
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(exp_desc)
samples, n_sims = util.io.load(os.path.join(exp_dir, 'results'))
true_ps, _ = sim.get_ground_truth()
err = calc_err(true_ps, samples)
util.io.save((err, n_sims), res_file)
return err, n_sims
def get_err_postprop(exp_desc, sim, seed):
"""
Calculates the error for a given Post Prop experiment.
"""
assert isinstance(exp_desc.inf, ed.PostProp_Descriptor)
res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'err')
if os.path.exists(res_file + '.pkl'):
all_prop_errs, post_err = util.io.load(res_file)
else:
exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0')
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(exp_desc)
true_ps, _ = sim.get_ground_truth()
all_proposals, posterior, _, _ = util.io.load(os.path.join(exp_dir, 'results'))
all_prop_errs = []
for i, proposal in enumerate(all_proposals[1:]):
samples = proposal.gen(n_mcmc_samples, rng=rng)
prop_err = calc_err(true_ps, samples)
all_prop_errs.append(prop_err)
samples = posterior.gen(n_mcmc_samples, rng=rng)
post_err = calc_err(true_ps, samples)
util.io.save((all_prop_errs, post_err), res_file)
return all_prop_errs, post_err
def get_err_snl(exp_desc):
"""
Calculates the error for a given SNL experiment.
"""
assert isinstance(exp_desc.inf, ed.SNL_Descriptor)
res_file = os.path.join(root, 'results', exp_desc.get_dir(), 'lik_mmd')
if os.path.exists(res_file + '.pkl'):
all_errs = util.io.load(res_file)
else:
exp_dir = os.path.join(root, 'experiments', exp_desc.get_dir(), '0')
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(exp_desc)
_, _, all_nets = util.io.load(os.path.join(exp_dir, 'results'))
all_errs = []
for net in all_nets:
net.reset_theano_functions()
all_errs.append(calc_mmd_cond(net))
util.io.save(all_errs, res_file)
return all_errs
def get_err_smc(exp_desc, sim, seed):
"""
Calculates the error for a given SMC ABC experiment.
"""
assert isinstance(exp_desc.inf, ed.SMC_ABC_Descriptor)
res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'err')
if os.path.exists(res_file + '.pkl'):
all_errs, all_n_sims = util.io.load(res_file)
else:
exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0')
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(exp_desc)
true_ps, _ = sim.get_ground_truth()
all_samples, all_log_weights, _, _, all_n_sims = util.io.load(os.path.join(exp_dir, 'results'))
all_errs = []
for samples, log_weights in zip(all_samples, all_log_weights):
weights = np.exp(log_weights)
err = calc_err(true_ps, samples, weights)
all_errs.append(err)
util.io.save((all_errs, all_n_sims), res_file)
return all_errs, all_n_sims
def get_samples_snl(exp_desc, trial, sim, prior_kind):
"""
Generates MCMC samples for a given SNL experiment.
"""
assert isinstance(exp_desc.inf, ed.SNL_Descriptor)
if prior_kind == 'original':
folder = exp_desc.get_dir()
prior = sim.Prior()
elif prior_kind == 'near_truth':
folder = os.path.join(exp_desc.sim, 'other', 'prior_near_truth',
exp_desc.inf.get_dir())
prior = sim.PriorNearTruth()
else:
raise ValueError('unknown prior: {0}'.format(prior_kind))
res_file = os.path.join(root, 'results', folder, str(trial), 'samples')
if os.path.exists(res_file + '.pkl'):
samples = util.io.load(res_file)
else:
exp_dir = os.path.join(root, 'experiments', folder, str(trial))
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(exp_desc)
net = util.io.load(os.path.join(exp_dir, 'model'))
true_ps, obs_xs = util.io.load(os.path.join(exp_dir, 'gt'))
if sim is simulators.lotka_volterra:
fs_net = get_failed_sims_model_lv()
log_posterior = lambda t: net.eval([t, obs_xs]) + prior.eval(
t) + np.log(fs_net.eval(t)[0])
else:
log_posterior = lambda t: net.eval([t, obs_xs]) + prior.eval(t)
print 'sampling trial {0}'.format(trial)
sampler = mcmc.SliceSampler(true_ps, log_posterior, thin=thin)
sampler.gen(burnin, rng=rng)
samples = sampler.gen(n_samples, rng=rng)
util.io.save(samples, res_file)
return samples
def print_log(self, trial=0):
"""
Prints the log of the experiment.
"""
print '\n' + '-' * 80
print 'PRINTING LOG:\n'
print self.exp_desc.pprint()
exp_dir = os.path.join(self.exp_dir, str(trial))
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(self.exp_desc)
assert util.io.load_txt(os.path.join(exp_dir, 'info.txt')) == self.exp_desc.pprint()
print util.io.load_txt(os.path.join(exp_dir, 'out.log'))
def view_results(self, trial=0, block=False):
"""
Shows the results of the experiment.
:param block: whether to block execution after showing results
"""
print '\n' + '-' * 80
print 'VIEWING RESULTS:\n'
print self.exp_desc.pprint()
exp_dir = os.path.join(self.exp_dir, str(trial))
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(self.exp_desc)
assert util.io.load_txt(os.path.join(exp_dir, 'info.txt')) == self.exp_desc.pprint()
if isinstance(self.exp_desc.inf, ed.ABC_Descriptor):
self._view_abc(exp_dir)
elif isinstance(self.exp_desc.inf, ed.SynthLik_Descriptor):
self._view_synth_lik(exp_dir)
elif isinstance(self.exp_desc.inf, ed.NDE_Descriptor):
self._view_nde(exp_dir)
elif isinstance(self.exp_desc.inf, ed.PostProp_Descriptor):
self._view_post_prop(exp_dir)
elif isinstance(self.exp_desc.inf, ed.SNPE_MDN_Descriptor):
self._view_snpe_mdn(exp_dir)
elif isinstance(self.exp_desc.inf, ed.SNL_Descriptor):
self._view_snl(exp_dir)
else:
raise TypeError('unknown inference descriptor')
plt.show(block=block)
def get_err_nde(exp_desc):
"""
Calculates the error for a given NDE experiment.
"""
assert isinstance(exp_desc.inf, ed.NDE_Descriptor)
res_file = os.path.join(root, 'results', exp_desc.get_dir(), 'lik_mmd')
if os.path.exists(res_file + '.pkl'):
err = util.io.load(res_file)
else:
exp_dir = os.path.join(root, 'experiments', exp_desc.get_dir(), '0')
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(exp_desc)
net = util.io.load(os.path.join(exp_dir, 'model'))
err = calc_mmd_cond(net)
util.io.save(err, res_file)
return err
def get_order_snl(exp_desc, n_trials, n_bins, sim, prior):
"""
Calculates the order statistic for a given SNL experiment.
"""
assert isinstance(exp_desc.inf, ed.SNL_Descriptor)
if prior == 'original':
folder = exp_desc.get_dir()
elif prior == 'near_truth':
folder = os.path.join(exp_desc.sim, 'other', 'prior_near_truth',
exp_desc.inf.get_dir())
else:
raise ValueError('unknown prior: {0}'.format(prior))
n_dims = sim.Prior().n_dims
order = np.empty([n_trials, n_dims])
for i in xrange(n_trials):
exp_dir = os.path.join(root, 'experiments', folder, str(i + 1))
if not os.path.exists(exp_dir):
raise misc.NonExistentExperiment(exp_desc)
true_ps, _ = util.io.load(os.path.join(exp_dir, 'gt'))
samples = get_samples_snl(exp_desc, i + 1, sim, prior)
samples = samples[:n_bins - 1]
if samples.shape[0] < n_bins - 1:
raise RuntimeError(
'not enough samples for {0} bins'.format(n_bins))
for j in xrange(n_dims):
order[i, j] = sum(true_ps[j] > samples[:, j])
return order
