def get_samples_snl(exp_desc, seed): """ Generates MCMC samples for a given SNL experiment. """ assert isinstance(exp_desc.inf, ed.SNL_Descriptor) res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'mmd') if os.path.exists(res_file + '.pkl'): all_samples = util.io.load(res_file) else: exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0') if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(exp_desc) _, _, all_nets = util.io.load(os.path.join(exp_dir, 'results')) all_samples = [] for net in all_nets: net.reset_theano_functions() log_posterior = lambda t: net.eval([t, obs_xs]) + prior.eval(t) sampler = mcmc.SliceSampler(true_ps, log_posterior, thin=thin) sampler.gen(burnin, rng=rng) # burn in samples = sampler.gen(n_mcmc_samples, rng=rng) all_samples.append(samples) util.io.save(all_samples, res_file) return all_samples
def get_mmd_snpe(exp_desc, seed): """ Calculates the MMD for a given SNPE experiment. """ assert isinstance(exp_desc.inf, ed.SNPE_MDN_Descriptor) res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'mmd') if os.path.exists(res_file + '.pkl'): all_mmds = util.io.load(res_file) else: exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0') if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(exp_desc) true_samples = get_true_samples(seed) scale = util.math.median_distance(true_samples) all_posteriors, _, _, _ = util.io.load(os.path.join(exp_dir, 'results')) all_mmds = [] for posterior in all_posteriors[1:]: samples = posterior.gen(n_mcmc_samples, rng=rng) err = two_sample.sq_maximum_mean_discrepancy(samples, true_samples, scale=scale) all_mmds.append(err) util.io.save(all_mmds, res_file) return all_mmds
def get_samples_nde(exp_desc, seed): """ Generates MCMC samples for a given NDE experiment. """ assert isinstance(exp_desc.inf, ed.NDE_Descriptor) res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'mmd') if os.path.exists(res_file + '.pkl'): samples = util.io.load(res_file) else: exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0') if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(exp_desc) net = util.io.load(os.path.join(exp_dir, 'model')) log_posterior = lambda t: net.eval([t, obs_xs]) + prior.eval(t) sampler = mcmc.SliceSampler(true_ps, log_posterior, thin=thin) sampler.gen(burnin, rng=rng) # burn in samples = sampler.gen(n_mcmc_samples, rng=rng) util.io.save(samples, res_file) return samples
def get_mmd_sl(exp_desc, seed): """ Calculates the MMD for a given synth likelihood experiment. """ assert isinstance(exp_desc.inf, ed.SynthLik_Descriptor) res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'mmd') if os.path.exists(res_file + '.pkl'): err, n_sims = util.io.load(res_file) else: exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0') if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(exp_desc) samples, n_sims = util.io.load(os.path.join(exp_dir, 'results')) true_samples = get_true_samples(seed) scale = util.math.median_distance(true_samples) err = two_sample.sq_maximum_mean_discrepancy(samples, true_samples, scale=scale) util.io.save((err, n_sims), res_file) return err, n_sims
def get_mmd_postprop(exp_desc, seed): """ Calculates the MMD for a given Post Prop experiment. """ assert isinstance(exp_desc.inf, ed.PostProp_Descriptor) res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'mmd') if os.path.exists(res_file + '.pkl'): all_prop_mmds, post_err = util.io.load(res_file) else: exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0') print(exp_dir) if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(exp_desc) true_samples = get_true_samples(seed) scale = util.math.median_distance(true_samples) all_proposals, posterior, _, _ = util.io.load(os.path.join(exp_dir, 'results')) all_prop_mmds = [] for i, proposal in enumerate(all_proposals[1:]): samples = proposal.gen(n_mcmc_samples, rng=rng) prop_err = two_sample.sq_maximum_mean_discrepancy(samples, true_samples, scale=scale) all_prop_mmds.append(prop_err) samples = posterior.gen(n_mcmc_samples, rng=rng) post_err = two_sample.sq_maximum_mean_discrepancy(samples, true_samples, scale=scale) util.io.save((all_prop_mmds, post_err), res_file) return all_prop_mmds, post_err
def get_err_snpe(exp_desc, sim, seed): """ Calculates the error for a given SNPE experiment. """ assert isinstance(exp_desc.inf, ed.SNPE_MDN_Descriptor) res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'err') if os.path.exists(res_file + '.pkl'): all_errs = util.io.load(res_file) else: exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0') if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(exp_desc) true_ps, _ = sim.get_ground_truth() all_posteriors, _, _, _ = util.io.load(os.path.join(exp_dir, 'results')) all_errs = [] for posterior in all_posteriors[1:]: samples = posterior.gen(n_mcmc_samples, rng=rng) err = calc_err(true_ps, samples) all_errs.append(err) util.io.save(all_errs, res_file) return all_errs
def get_mmd_smc(exp_desc, seed): """ Calculates the MMD for a given SMC ABC experiment. """ assert isinstance(exp_desc.inf, ed.SMC_ABC_Descriptor) res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'mmd') if os.path.exists(res_file + '.pkl'): all_mmds, all_n_sims = util.io.load(res_file) else: exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0') if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(exp_desc) true_samples = get_true_samples(seed) scale = util.math.median_distance(true_samples) all_samples, all_log_weights, _, _, all_n_sims = util.io.load(os.path.join(exp_dir, 'results')) all_mmds = [] for samples, log_weights in zip(all_samples, all_log_weights): weights = np.exp(log_weights) err = two_sample.sq_maximum_mean_discrepancy(xs=samples, ys=true_samples, wxs=weights, scale=scale) all_mmds.append(err) util.io.save((all_mmds, all_n_sims), res_file) return all_mmds, all_n_sims
def get_err_sl(exp_desc, sim, seed): """ Calculates the error for a given synth likelihood experiment. """ assert isinstance(exp_desc.inf, ed.SynthLik_Descriptor) res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'err') if os.path.exists(res_file + '.pkl'): err, n_sims = util.io.load(res_file) else: exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0') if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(exp_desc) samples, n_sims = util.io.load(os.path.join(exp_dir, 'results')) true_ps, _ = sim.get_ground_truth() err = calc_err(true_ps, samples) util.io.save((err, n_sims), res_file) return err, n_sims
def get_err_postprop(exp_desc, sim, seed): """ Calculates the error for a given Post Prop experiment. """ assert isinstance(exp_desc.inf, ed.PostProp_Descriptor) res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'err') if os.path.exists(res_file + '.pkl'): all_prop_errs, post_err = util.io.load(res_file) else: exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0') if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(exp_desc) true_ps, _ = sim.get_ground_truth() all_proposals, posterior, _, _ = util.io.load(os.path.join(exp_dir, 'results')) all_prop_errs = [] for i, proposal in enumerate(all_proposals[1:]): samples = proposal.gen(n_mcmc_samples, rng=rng) prop_err = calc_err(true_ps, samples) all_prop_errs.append(prop_err) samples = posterior.gen(n_mcmc_samples, rng=rng) post_err = calc_err(true_ps, samples) util.io.save((all_prop_errs, post_err), res_file) return all_prop_errs, post_err
def get_err_snl(exp_desc): """ Calculates the error for a given SNL experiment. """ assert isinstance(exp_desc.inf, ed.SNL_Descriptor) res_file = os.path.join(root, 'results', exp_desc.get_dir(), 'lik_mmd') if os.path.exists(res_file + '.pkl'): all_errs = util.io.load(res_file) else: exp_dir = os.path.join(root, 'experiments', exp_desc.get_dir(), '0') if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(exp_desc) _, _, all_nets = util.io.load(os.path.join(exp_dir, 'results')) all_errs = [] for net in all_nets: net.reset_theano_functions() all_errs.append(calc_mmd_cond(net)) util.io.save(all_errs, res_file) return all_errs
def get_err_smc(exp_desc, sim, seed): """ Calculates the error for a given SMC ABC experiment. """ assert isinstance(exp_desc.inf, ed.SMC_ABC_Descriptor) res_file = os.path.join(root, 'results/seed_'+str(seed), exp_desc.get_dir(), 'err') if os.path.exists(res_file + '.pkl'): all_errs, all_n_sims = util.io.load(res_file) else: exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0') if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(exp_desc) true_ps, _ = sim.get_ground_truth() all_samples, all_log_weights, _, _, all_n_sims = util.io.load(os.path.join(exp_dir, 'results')) all_errs = [] for samples, log_weights in zip(all_samples, all_log_weights): weights = np.exp(log_weights) err = calc_err(true_ps, samples, weights) all_errs.append(err) util.io.save((all_errs, all_n_sims), res_file) return all_errs, all_n_sims
def get_samples_snl(exp_desc, trial, sim, prior_kind): """ Generates MCMC samples for a given SNL experiment. """ assert isinstance(exp_desc.inf, ed.SNL_Descriptor) if prior_kind == 'original': folder = exp_desc.get_dir() prior = sim.Prior() elif prior_kind == 'near_truth': folder = os.path.join(exp_desc.sim, 'other', 'prior_near_truth', exp_desc.inf.get_dir()) prior = sim.PriorNearTruth() else: raise ValueError('unknown prior: {0}'.format(prior_kind)) res_file = os.path.join(root, 'results', folder, str(trial), 'samples') if os.path.exists(res_file + '.pkl'): samples = util.io.load(res_file) else: exp_dir = os.path.join(root, 'experiments', folder, str(trial)) if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(exp_desc) net = util.io.load(os.path.join(exp_dir, 'model')) true_ps, obs_xs = util.io.load(os.path.join(exp_dir, 'gt')) if sim is simulators.lotka_volterra: fs_net = get_failed_sims_model_lv() log_posterior = lambda t: net.eval([t, obs_xs]) + prior.eval( t) + np.log(fs_net.eval(t)[0]) else: log_posterior = lambda t: net.eval([t, obs_xs]) + prior.eval(t) print 'sampling trial {0}'.format(trial) sampler = mcmc.SliceSampler(true_ps, log_posterior, thin=thin) sampler.gen(burnin, rng=rng) samples = sampler.gen(n_samples, rng=rng) util.io.save(samples, res_file) return samples
def print_log(self, trial=0): """ Prints the log of the experiment. """ print '\n' + '-' * 80 print 'PRINTING LOG:\n' print self.exp_desc.pprint() exp_dir = os.path.join(self.exp_dir, str(trial)) if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(self.exp_desc) assert util.io.load_txt(os.path.join(exp_dir, 'info.txt')) == self.exp_desc.pprint() print util.io.load_txt(os.path.join(exp_dir, 'out.log'))
def view_results(self, trial=0, block=False): """ Shows the results of the experiment. :param block: whether to block execution after showing results """ print '\n' + '-' * 80 print 'VIEWING RESULTS:\n' print self.exp_desc.pprint() exp_dir = os.path.join(self.exp_dir, str(trial)) if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(self.exp_desc) assert util.io.load_txt(os.path.join(exp_dir, 'info.txt')) == self.exp_desc.pprint() if isinstance(self.exp_desc.inf, ed.ABC_Descriptor): self._view_abc(exp_dir) elif isinstance(self.exp_desc.inf, ed.SynthLik_Descriptor): self._view_synth_lik(exp_dir) elif isinstance(self.exp_desc.inf, ed.NDE_Descriptor): self._view_nde(exp_dir) elif isinstance(self.exp_desc.inf, ed.PostProp_Descriptor): self._view_post_prop(exp_dir) elif isinstance(self.exp_desc.inf, ed.SNPE_MDN_Descriptor): self._view_snpe_mdn(exp_dir) elif isinstance(self.exp_desc.inf, ed.SNL_Descriptor): self._view_snl(exp_dir) else: raise TypeError('unknown inference descriptor') plt.show(block=block)
def get_err_nde(exp_desc): """ Calculates the error for a given NDE experiment. """ assert isinstance(exp_desc.inf, ed.NDE_Descriptor) res_file = os.path.join(root, 'results', exp_desc.get_dir(), 'lik_mmd') if os.path.exists(res_file + '.pkl'): err = util.io.load(res_file) else: exp_dir = os.path.join(root, 'experiments', exp_desc.get_dir(), '0') if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(exp_desc) net = util.io.load(os.path.join(exp_dir, 'model')) err = calc_mmd_cond(net) util.io.save(err, res_file) return err
def get_order_snl(exp_desc, n_trials, n_bins, sim, prior): """ Calculates the order statistic for a given SNL experiment. """ assert isinstance(exp_desc.inf, ed.SNL_Descriptor) if prior == 'original': folder = exp_desc.get_dir() elif prior == 'near_truth': folder = os.path.join(exp_desc.sim, 'other', 'prior_near_truth', exp_desc.inf.get_dir()) else: raise ValueError('unknown prior: {0}'.format(prior)) n_dims = sim.Prior().n_dims order = np.empty([n_trials, n_dims]) for i in xrange(n_trials): exp_dir = os.path.join(root, 'experiments', folder, str(i + 1)) if not os.path.exists(exp_dir): raise misc.NonExistentExperiment(exp_desc) true_ps, _ = util.io.load(os.path.join(exp_dir, 'gt')) samples = get_samples_snl(exp_desc, i + 1, sim, prior) samples = samples[:n_bins - 1] if samples.shape[0] < n_bins - 1: raise RuntimeError( 'not enough samples for {0} bins'.format(n_bins)) for j in xrange(n_dims): order[i, j] = sum(true_ps[j] > samples[:, j]) return order