def plot_results(sim_name):
"""
Plots all results for a given simulator and kind of error.
"""
global sim
sim = misc.get_simulator(translate_sim_name(sim_name))
# gaussian
err_gauss = get_err_gaussian(sim_name)
err_gauss = max(err_gauss, 0.0)
# NDE
all_err_nde = []
all_n_sims_nde = []
for exp_desc in ed.parse(
util.io.load_txt('exps/{0}_nl.txt'.format(sim_name))):
all_err_nde.append(get_err_nde(exp_desc))
all_n_sims_nde.append(exp_desc.inf.n_samples)
all_err_snl = None
all_n_sims_snl = None
for exp_desc in ed.parse(
util.io.load_txt('exps/{0}_seq.txt'.format(sim_name))):
# SNL
if isinstance(exp_desc.inf, ed.SNL_Descriptor):
all_err_snl = get_err_snl(exp_desc)
all_n_sims_snl = [(i + 1) * exp_desc.inf.n_samples
for i in xrange(exp_desc.inf.n_rounds)]
matplotlib.rc('text', usetex=True)
matplotlib.rc('font', size=16)
all_n_sims = np.concatenate([all_n_sims_nde, all_n_sims_snl])
min_n_sims = np.min(all_n_sims)
max_n_sims = np.max(all_n_sims)
fig, ax = plt.subplots(1, 1)
ax.semilogx(all_n_sims_nde,
np.sqrt(all_err_nde),
's:',
color='b',
label='NL')
ax.semilogx(all_n_sims_snl,
np.sqrt(all_err_snl),
'o:',
color='r',
label='SNL')
ax.axhline(np.sqrt(err_gauss), linestyle='--', color='k', label='Gaussian')
ax.set_xlabel('Number of simulations (log scale)')
ax.set_ylabel('MMD')
ax.set_xlim([min_n_sims * 10**(-0.2), max_n_sims * 10**0.2])
ax.set_ylim([-0.1 if sim_name == 'gauss' else 0.0, ax.get_ylim()[1]])
ax.legend(fontsize=14,
loc='upper right' if sim_name == 'gauss' else 'lower right')
plt.show()
def run_trials(args):
"""
Runs experiments for multiple trials with random ground truth.
"""
from experiment_runner import ExperimentRunner
if args.start < 1:
raise ValueError('trial # must be a positive integer')
if args.end < args.start:
raise ValueError('end trial can''t be less than start trial')
exp_descs = sum([ed.parse(util.io.load_txt(f)) for f in args.files], [])
for exp_desc in exp_descs:
runner = ExperimentRunner(exp_desc)
for trial in xrange(args.start, args.end + 1):
try:
runner.run(trial=trial, sample_gt=True, rng=np.random)
except misc.AlreadyExistingExperiment:
print 'EXPERIMENT ALREADY EXISTS'
print 'ALL DONE'
def view_samples_snl(sim_name, which=None, use_lims=True):
"""
Plots likelihood samples for all SNL models.
"""
true_ps, obs_xs = sim.get_ground_truth()
lims = get_disp_lims() if use_lims else None
for exp_desc in ed.parse(
util.io.load_txt('exps/{0}_seq.txt'.format(sim_name))):
if isinstance(exp_desc.inf, ed.SNL_Descriptor):
exp_dir = os.path.join(root, 'experiments', exp_desc.get_dir(),
'0')
_, _, all_nets = util.io.load(os.path.join(exp_dir, 'results'))
for i, net in enumerate(all_nets):
if which is not None and i != which - 1:
continue
net.reset_theano_functions()
samples = net.gen(true_ps, n_samples)
fig = util.plot.plot_hist_marginals(samples,
lims=lims,
gt=obs_xs)
fig.suptitle('SNL, round = {0}'.format(i + 1))
plt.plot()
def plot_results(sim_name, run_name, average):
"""
Plots all results for a given simulator.
"""
fig, ax = plt.subplots(1, 1)
seeds = np.arange(42, 52)
for seed in seeds:
all_dist_ppr = None
all_dist_snp = None
all_dist_snl = None
for exp_desc in ed.parse(util.io.load_txt('exps/{0}_seq.txt'.format(sim_name))):
# Post Prop
if isinstance(exp_desc.inf, ed.PostProp_Descriptor):
all_dist_ppr = get_dist(exp_desc, average, seed)
# SNPE
if isinstance(exp_desc.inf, ed.SNPE_MDN_Descriptor):
all_dist_snp = get_dist(exp_desc, average, seed)
# SNL
if isinstance(exp_desc.inf, ed.SNL_Descriptor):
all_dist_snl = get_dist(exp_desc, average, seed)
# SNPE-C
matplotlib.rc('text', usetex=True)
matplotlib.rc('font', size=16)
ax.plot(np.arange(len(all_dist_ppr)) + 1, all_dist_ppr, '>:', color='c', label='SNPE-A')
ax.plot(np.arange(len(all_dist_snp)) + 1, all_dist_snp, 'p:', color='g', label='SNPE-B')
ax.plot(np.arange(len(all_dist_snl)) + 1, all_dist_snl, 'o:', color='r', label='SNL')
try:
all_dist_snpc = np.load('../lfi_experiments/snpec/results/'+sim_name+'_'+run_name+'/seed'+str(seed)+'/avg_dist.npy')
ax.plot(np.arange(len(all_dist_snpc)) + 1, all_dist_snpc, 'd-', color='k', label='SNPE-C')
except:
print ' could not load SNPE-C results, seed ' + str(seed)
ax.set_xlabel('Round')
ax.set_ylabel('{0} distance'.format(average[0].upper() + average[1:]))
ax.set_ylim([0.0, ax.get_ylim()[1]])
if seed == seeds[0]:
ax.legend(fontsize=14)
for seed in seeds:
try:
all_dist_snpc = np.load('../lfi_experiments/snpec/results/'+sim_name+'_'+run_name+'/seed'+str(seed)+'/avg_dist.npy')
ax.plot(np.arange(len(all_dist_snpc)) + 1, all_dist_snpc, 'd-', color='k', label='SNPE-C')
except:
print ' could not load SNPE-C results, seed ' + str(seed)
plt.show()
def plot_results(sim_name, prior):
"""
Plots all results for a given simulator.
"""
n_trials = 200
n_bins = 10
l_quant = get_hist_quantile(0.005, n_trials, n_bins)
u_quant = get_hist_quantile(0.995, n_trials, n_bins)
centre = 1.0 / n_bins
sim = get_sim(sim_name)
matplotlib.rc('text', usetex=True)
matplotlib.rc('font', size=15 if sim_name == 'lv' else 14)
# SNL
txt = util.io.load_txt('exps/{0}_calib.txt'.format(sim_name))
for exp_desc in ed.parse(txt):
order = get_order_snl(exp_desc, n_trials, n_bins, sim, prior)
for j in xrange(order.shape[1]):
fig, ax = plt.subplots(1, 1)
ax.hist(order[:, j],
bins=np.arange(n_bins + 1) - 0.5,
normed=True,
color='r')
ax.axhspan(l_quant, u_quant, facecolor='0.5', alpha=0.5)
ax.axhline(centre, color='k', lw=2)
ax.set_xlim([-0.5, n_bins - 0.5])
if sim_name == 'lv' and j != 1:
ax.set_ylim([0.0, ax.get_ylim()[1]])
else:
ax.set_ylim([0.0, u_quant * 1.1])
ax.tick_params(axis='x',
which='both',
top=False,
bottom=False,
labelbottom=False)
ax.tick_params(axis='y',
which='both',
left=False,
right=False,
labelleft=False)
if sim_name == 'lv':
ax.set_title(r'$\theta_{' + str(j + 1) + r'}$, ' +
('oscillating regime' if prior ==
'near_truth' else 'broad prior'))
else:
ax.set_title(r'$\theta_{' + str(j + 1) + r'}$')
plt.show()
def view_samples_nde(seed):
"""
Plots MCMC samples for all NDE experiments.
"""
for exp_desc in ed.parse(util.io.load_txt('exps/gauss_nl.txt')):
samples = get_samples_nde(exp_desc, seed)
fig = util.plot.plot_hist_marginals(samples, lims=sim.get_disp_lims(), gt=true_ps)
fig.suptitle('NDE, sims = {0}'.format(exp_desc.inf.n_samples))
plt.plot()
def view_samples_sl():
"""
Plots MCMC samples for all synth likelihood experiments.
"""
for exp_desc in ed.parse(util.io.load_txt('exps/gauss_sl.txt')):
exp_dir = os.path.join(root, 'experiments', exp_desc.get_dir(), '0')
samples, _ = util.io.load(os.path.join(exp_dir, 'results'))
fig = util.plot.plot_hist_marginals(samples, lims=sim.get_disp_lims(), gt=true_ps)
fig.suptitle('Synth Lik, sims = {0}'.format(exp_desc.inf.n_sims))
plt.plot()
def plot_results(sim_name, average):
"""
Plots all results for a given simulator.
"""
all_dist_ppr = None
all_dist_snp = None
all_dist_snl = None
for exp_desc in ed.parse(
util.io.load_txt('exps/{0}_seq.txt'.format(sim_name))):
# Post Prop
if isinstance(exp_desc.inf, ed.PostProp_Descriptor):
all_dist_ppr = get_dist(exp_desc, average)
# SNPE
if isinstance(exp_desc.inf, ed.SNPE_MDN_Descriptor):
all_dist_snp = get_dist(exp_desc, average)
# SNL
if isinstance(exp_desc.inf, ed.SNL_Descriptor):
all_dist_snl = get_dist(exp_desc, average)
matplotlib.rc('text', usetex=True)
matplotlib.rc('font', size=16)
fig, ax = plt.subplots(1, 1)
ax.plot(np.arange(len(all_dist_ppr)) + 1,
all_dist_ppr,
'>:',
color='c',
label='SNPE-A')
ax.plot(np.arange(len(all_dist_snp)) + 1,
all_dist_snp,
'p:',
color='g',
label='SNPE-B')
ax.plot(np.arange(len(all_dist_snl)) + 1,
all_dist_snl,
'o:',
color='r',
label='SNL')
ax.set_xlabel('Round')
ax.set_ylabel('{0} distance'.format(average[0].upper() + average[1:]))
ax.set_ylim([0.0, ax.get_ylim()[1]])
ax.legend(fontsize=14)
plt.show()
def view_samples_snl(seed):
"""
Plots MCMC samples for all SNL experiments.
"""
for exp_desc in ed.parse(util.io.load_txt('exps/gauss_seq.txt')):
if isinstance(exp_desc.inf, ed.SNL_Descriptor):
all_samples = get_samples_snl(exp_desc, seed)
for i, samples in enumerate(all_samples):
fig = util.plot.plot_hist_marginals(samples, lims=sim.get_disp_lims(), gt=true_ps)
fig.suptitle('SNL, round = {0}'.format(i + 1))
plt.plot()
def print_log(args):
"""
Prints experiment logs.
"""
from experiment_viewer import ExperimentViewer
exp_descs = sum([ed.parse(util.io.load_txt(f)) for f in args.files], [])
for exp_desc in exp_descs:
try:
ExperimentViewer(exp_desc).print_log()
except misc.NonExistentExperiment:
print 'EXPERIMENT DOES NOT EXIST'
def view_samples_nde(sim_name, seed):
"""
Plots MCMC samples for all NDE experiments.
"""
sim = get_sim(sim_name)
true_ps, _ = sim.get_ground_truth()
for exp_desc in ed.parse(util.io.load_txt('exps/{0}_nl.txt'.format(sim_name))):
samples = get_samples_nde(exp_desc, sim, seed)
fig = util.plot.plot_hist_marginals(samples, lims=sim.get_disp_lims(), gt=true_ps)
fig.suptitle('NDE, sims = {0}'.format(exp_desc.inf.n_samples))
plt.plot()
def view_results(args):
"""
Views experiments.
"""
from experiment_viewer import ExperimentViewer, plt
exp_descs = sum([ed.parse(util.io.load_txt(f)) for f in args.files], [])
for exp_desc in exp_descs:
try:
ExperimentViewer(exp_desc).view_results(trial=args.trial, block=args.block)
except misc.NonExistentExperiment:
print 'EXPERIMENT DOES NOT EXIST'
plt.show()
def run_experiment(args):
"""
Runs experiments.
"""
from experiment_runner import ExperimentRunner
exp_descs = sum([ed.parse(util.io.load_txt(f)) for f in args.files], [])
for exp_desc in exp_descs:
try:
ExperimentRunner(exp_desc).run(trial=0, sample_gt=False, rng=np.random.RandomState(42))
except misc.AlreadyExistingExperiment:
print 'EXPERIMENT ALREADY EXISTS'
print 'ALL DONE'
def view_samples_sl(sim_name, seed):
"""
Plots samples from all the synth likelihood experiments.
"""
sim = get_sim(sim_name)
true_ps, _ = sim.get_ground_truth()
for exp_desc in ed.parse(util.io.load_txt('exps/{0}_sl.txt'.format(sim_name))):
exp_dir = os.path.join(root, 'experiments/seed_'+str(seed), exp_desc.get_dir(), '0')
samples, _ = util.io.load(os.path.join(exp_dir, 'results'))
fig = util.plot.plot_hist_marginals(samples, lims=sim.get_disp_lims(), gt=true_ps)
fig.suptitle('Synth Lik, sims = {0}'.format(exp_desc.inf.n_sims))
plt.plot()
def run_experiment(args):
"""
Runs experiments.
"""
from experiment_runner import ExperimentRunner
exp_descs = sum([ed.parse(util.io.load_txt(f[:-2])) for f in args.files], [])
seed_descs = [int(float(f[-2:])) for f in args.files]
for exp_desc, seed in zip(exp_descs, seed_descs):
print '\n seed: '
print seed
exp_desc.get_dir = lambda: os.path.join(exp_desc.sim+str(seed), exp_desc.inf.get_dir())
print exp_desc.get_dir()
try:
er = ExperimentRunner(exp_desc)
er.run(trial=0, sample_gt=False, rng=np.random.RandomState(seed))
except misc.AlreadyExistingExperiment:
print 'EXPERIMENT ALREADY EXISTS'
print 'ALL DONE'
def view_samples_nde(sim_name, which=None, use_lims=True):
"""
Plots likelihood samples for all NDE models.
"""
true_ps, obs_xs = sim.get_ground_truth()
lims = get_disp_lims() if use_lims else None
for exp_desc in ed.parse(
util.io.load_txt('exps/{0}_nl.txt'.format(sim_name))):
if which is not None and exp_desc.inf.n_samples != which:
continue
exp_dir = os.path.join(root, 'experiments', exp_desc.get_dir(), '0')
net = util.io.load(os.path.join(exp_dir, 'model'))
samples = net.gen(true_ps, n_samples)
fig = util.plot.plot_hist_marginals(samples, lims=lims, gt=obs_xs)
fig.suptitle('NDE, sims = {0}'.format(exp_desc.inf.n_samples))
plt.plot()
def plot_results(run_name=''):
"""
# SMC
exp_desc = ed.parse(util.io.load_txt('exps/gauss_smc.txt'))[0]
all_mmd_smc, all_n_sims_smc = get_mmd_smc(exp_desc)
# SL
all_mmd_slk = []
all_n_sims_slk = []
for exp_desc in ed.parse(util.io.load_txt('exps/gauss_sl.txt')):
mmd, n_sims = get_mmd_sl(exp_desc)
all_mmd_slk.append(mmd)
all_n_sims_slk.append(n_sims)
# NDE
all_mmd_nde = []
all_n_sims_nde = []
for exp_desc in ed.parse(util.io.load_txt('exps/gauss_nl.txt')):
all_mmd_nde.append(get_mmd_nde(exp_desc))
all_n_sims_nde.append(exp_desc.inf.n_samples)
"""
#print 'seeds: ' + str(seeds)
if run_name == '.':
run_name = ''
fig, ax = plt.subplots(1, 1)
seeds = np.arange(42, 52)
for seed in seeds:
all_mmd_ppr = None
all_n_sims_ppr = None
all_mmd_snp = None
all_n_sims_snp = None
all_mmd_snl = None
all_n_sims_snl = None
for exp_desc in ed.parse(util.io.load_txt('exps/gauss_seq.txt')):
# Post Prop
if isinstance(exp_desc.inf, ed.PostProp_Descriptor):
all_prop_mmd, post_mmd = get_mmd_postprop(exp_desc, seed)
all_mmd_ppr = all_prop_mmd + [post_mmd]
all_n_sims_ppr = [(i + 1) * exp_desc.inf.n_samples_p for i in xrange(len(all_prop_mmd))]
all_n_sims_ppr.append(all_n_sims_ppr[-1] + exp_desc.inf.n_samples_f)
# SNPE
if isinstance(exp_desc.inf, ed.SNPE_MDN_Descriptor):
all_mmd_snp = get_mmd_snpe(exp_desc, seed)
all_n_sims_snp = [(i + 1) * exp_desc.inf.n_samples for i in xrange(exp_desc.inf.n_rounds)]
# SNL
if isinstance(exp_desc.inf, ed.SNL_Descriptor):
all_mmd_snl = get_mmd_snl(exp_desc, seed)
all_n_sims_snl = [(i + 1) * exp_desc.inf.n_samples for i in xrange(exp_desc.inf.n_rounds)]
matplotlib.rc('text', usetex=True)
matplotlib.rc('font', size=16)
all_n_sims = np.concatenate([all_n_sims_ppr, all_n_sims_snp, all_n_sims_snl])
min_n_sims = np.min(all_n_sims)
max_n_sims = np.max(all_n_sims)
all_mmd_snl = np.array(all_mmd_snl)
#print all_mmd_snl
print np.array(all_mmd_snl).shape
#ax.semilogx(all_n_sims_smc, np.sqrt(all_mmd_smc), 'v:', color='y', label='SMC ABC')
#ax.semilogx(all_n_sims_slk, np.sqrt(all_mmd_slk), 'D:', color='maroon', label='SL')
ax.semilogx(all_n_sims_ppr, np.sqrt(all_mmd_ppr), '>:', color='c', label='SNPE-A')
ax.semilogx(all_n_sims_snp, np.sqrt(all_mmd_snp), 'p:', color='g', label='SNPE-B')
#ax.semilogx(all_n_sims_nde, np.sqrt(all_mmd_nde), 's:', color='b', label='NL')
ax.semilogx(all_n_sims_snl, np.sqrt(all_mmd_snl), 'o:', color='r', label='SNL')
# SNPE-C
try:
all_mmd_snpc = np.load('../lfi_experiments/snpec/results/gauss'+run_name + '/seed'+str(seed)+'/all_mmds_N5000.npy')
all_n_sims_snpc = [(i + 1) * exp_desc.inf.n_samples for i in xrange(all_mmd_snpc.size)]
ax.semilogx(all_n_sims_snpc, np.sqrt(all_mmd_snpc), 'd-', color='k', label='SNPE-C')
except:
print ' could not load SNPE-C results, seed ' + str(seed)
ax.set_xlabel('Number of simulations (log scale)')
ax.set_ylabel('Maximum Mean Discrepancy')
ax.set_xlim([min_n_sims * 10 ** (-0.2), max_n_sims * 10 ** 0.2])
ax.set_ylim([0.0, ax.get_ylim()[1]])
if seed == seeds[0]:
ax.legend(fontsize=14)
#samples = get_true_samples(seed)
#np.save('/home/mackelab/Desktop/Projects/Biophysicality/code/lfi_experiments/snpec/results/gauss/seed'+str(seed)+'/samples', samples)
plt.show()
def plot_results(sim_name, run_name=''):
"""
Plots all results for a given simulator.
"""
sim = get_sim(sim_name)
# SMC
#exp_desc = ed.parse(util.io.load_txt('exps/{0}_smc.txt'.format(sim_name)))[0]
#all_err_smc, all_n_sims_smc = get_err_smc(exp_desc, sim)
# SL
#all_err_slk = []
#all_n_sims_slk = []
#for exp_desc in ed.parse(util.io.load_txt('exps/{0}_sl.txt'.format(sim_name))):
# err, n_sims = get_err_sl(exp_desc, sim)
# all_err_slk.append(err)
# all_n_sims_slk.append(n_sims)
# NDE
#all_err_nde = []
#all_n_sims_nde = []
#for exp_desc in ed.parse(util.io.load_txt('exps/{0}_nl.txt'.format(sim_name))):
# all_err_nde.append(get_err_nde(exp_desc, sim))
# all_n_sims_nde.append(exp_desc.inf.n_samples)
print 'sim: ' + str(sim_name)
#print 'seeds: ' + str(seeds)
if run_name == '.':
run_name = ''
fig, ax = plt.subplots(1, 1)
seeds = np.arange(42, 52)
for i in range(len(seeds)):
seed = seeds[i]
all_err_ppr = None
all_n_sims_ppr = None
all_err_snp = None
all_n_sims_snp = None
all_err_snl = None
all_n_sims_snl = None
# SMC
exp_desc = ed.parse(util.io.load_txt('exps/{0}_smc.txt'.format(sim_name)))[0]
if i==1:
all_err_smc = np.nan*np.ones_like(all_err_smc)
all_n_sims_smc = all_n_sims_smc
else:
all_err_smc, all_n_sims_smc = get_err_smc(exp_desc, sim, seed)
for exp_desc in ed.parse(util.io.load_txt('exps/{0}_seq.txt'.format(sim_name))):
# Post Prop
if isinstance(exp_desc.inf, ed.PostProp_Descriptor):
all_prop_err, post_err = get_err_postprop(exp_desc, sim, seed)
all_err_ppr = all_prop_err + [post_err]
all_n_sims_ppr = [(i + 1) * exp_desc.inf.n_samples_p for i in xrange(len(all_prop_err))]
all_n_sims_ppr.append(all_n_sims_ppr[-1] + exp_desc.inf.n_samples_f)
# SNPE
if isinstance(exp_desc.inf, ed.SNPE_MDN_Descriptor):
all_err_snp = get_err_snpe(exp_desc, sim, seed)
all_n_sims_snp = [(i + 1) * exp_desc.inf.n_samples for i in xrange(exp_desc.inf.n_rounds)]
# SNL
if isinstance(exp_desc.inf, ed.SNL_Descriptor):
all_err_snl = get_err_snl(exp_desc, sim, seed)
all_n_sims_snl = [(i + 1) * exp_desc.inf.n_samples for i in xrange(exp_desc.inf.n_rounds)]
matplotlib.rc('text', usetex=True)
matplotlib.rc('font', size=16)
all_n_sims = np.concatenate([all_n_sims_ppr, all_n_sims_snp, all_n_sims_snl])
min_n_sims = np.min(all_n_sims)
max_n_sims = np.max(all_n_sims)
ax.semilogx(all_n_sims_smc, all_err_smc, 'v:', color='y', label='SMC ABC')
#ax.semilogx(all_n_sims_slk, all_err_slk, 'D:', color='maroon', label='SL')
ax.semilogx(all_n_sims_ppr, all_err_ppr, '>:', color='c', label='SNPE-A')
ax.semilogx(all_n_sims_snp, all_err_snp, 'p:', color='g', label='SNPE-B')
#ax.semilogx(all_n_sims_nde, all_err_nde, 's:', color='b', label='NL')
ax.semilogx(all_n_sims_snl, all_err_snl, 'o:', color='r', label='SNL')
# SNPE-C
try:
all_err_snpc = np.load('../lfi_experiments/snpec/results/'+sim_name+run_name+'/seed'+str(seed)+'/all_prop_errs_N5000.npy')
all_n_sims_snpc = [(i + 1) * exp_desc.inf.n_samples for i in xrange(all_err_snpc.size)]
ax.semilogx(all_n_sims_snpc, all_err_snpc, 'd-', color='k', label='SNPE-C')
except:
print ' could not load SNPE-C results, seed ' + str(seed)
ax.set_xlabel('Number of simulations (log scale)')
ax.set_ylabel('$-$log probability of true parameters')
ax.set_xlim([min_n_sims * 10 ** (-0.2), max_n_sims * 10 ** 0.2])
if seed == seeds[0]:
ax.legend(fontsize=14)
for seed in seeds:
# SNPE-C
try:
all_err_snpc = np.load('../lfi_experiments/snpec/results/'+sim_name+run_name+'/seed'+str(seed)+'/all_prop_errs_N5000.npy')
all_n_sims_snpc = [(i + 1) * exp_desc.inf.n_samples for i in xrange(all_err_snpc.size)]
ax.semilogx(all_n_sims_snpc, all_err_snpc, 'd-', color='k', label='SNPE-C')
except:
print ' could not load SNPE-C results, seed ' + str(seed)
plt.show()
def plot_results(run_name=''):
"""
# SMC
exp_desc = ed.parse(util.io.load_txt('exps/gauss_smc.txt'))[0]
all_mmd_smc, all_n_sims_smc = get_mmd_smc(exp_desc)
# SL
all_mmd_slk = []
all_n_sims_slk = []
for exp_desc in ed.parse(util.io.load_txt('exps/gauss_sl.txt')):
mmd, n_sims = get_mmd_sl(exp_desc)
all_mmd_slk.append(mmd)
all_n_sims_slk.append(n_sims)
# NDE
all_mmd_nde = []
all_n_sims_nde = []
for exp_desc in ed.parse(util.io.load_txt('exps/gauss_nl.txt')):
all_mmd_nde.append(get_mmd_nde(exp_desc))
all_n_sims_nde.append(exp_desc.inf.n_samples)
"""
#print 'seeds: ' + str(seeds)
if run_name == '.':
run_name = ''
fig, ax = plt.subplots(1, 1)
seeds = np.arange(42, 52)
all_mmds_ppr, all_mmds_snp, all_mmds_snl, all_mmds_snpc = [],[],[],[]
ct = 0
for i in range(len(seeds)):
seed = seeds[i]
all_mmd_ppr = None
all_n_sims_ppr = None
all_mmd_snp = None
all_n_sims_snp = None
all_mmd_snl = None
all_n_sims_snl = None
# SMC
exp_desc = ed.parse(util.io.load_txt('exps/gauss_smc.txt'))[0]
if i==1:
all_mmd_smc = np.nan*np.ones_like(all_mmd_smc)
all_n_sims_smc = all_n_sims_smc
else:
all_mmd_smc, all_n_sims_smc = get_mmd_smc(exp_desc, seed)
#ct += 1
if i == 0 :
ax.semilogx(all_n_sims_smc, np.sqrt(all_mmd_smc), 'v:', color='y', linewidth=2.5, label='SMC ABC')
else:
ax.semilogx(all_n_sims_smc, np.sqrt(all_mmd_smc), 'v:', color='y', linewidth=2.5)
for exp_desc in ed.parse(util.io.load_txt('exps/gauss_seq.txt')):
# Post Prop
if isinstance(exp_desc.inf, ed.PostProp_Descriptor):
all_prop_mmd, post_mmd = get_mmd_postprop(exp_desc, seed)
all_mmd_ppr = all_prop_mmd + [post_mmd]
all_n_sims_ppr = [(i + 1) * exp_desc.inf.n_samples_p for i in xrange(len(all_prop_mmd))]
all_n_sims_ppr.append(all_n_sims_ppr[-1] + exp_desc.inf.n_samples_f)
all_mmds_ppr.append(all_mmd_ppr)
# SNPE
if isinstance(exp_desc.inf, ed.SNPE_MDN_Descriptor):
all_mmd_snp = get_mmd_snpe(exp_desc, seed)
all_n_sims_snp = [(i + 1) * exp_desc.inf.n_samples for i in xrange(exp_desc.inf.n_rounds)]
all_mmds_snp.append(all_mmd_snp)
# SNL
if isinstance(exp_desc.inf, ed.SNL_Descriptor):
all_mmd_snl = get_mmd_snl(exp_desc, seed)
all_n_sims_snl = [(i + 1) * exp_desc.inf.n_samples for i in xrange(exp_desc.inf.n_rounds)]
all_mmds_snl.append(all_mmd_snl)
all_n_sims = np.concatenate([all_n_sims_ppr, all_n_sims_snp, all_n_sims_snl])
min_n_sims = np.min(all_n_sims)
max_n_sims = np.max(all_n_sims)
all_mmd_snl = np.array(all_mmd_snl)
# SNPE-C
try:
all_mmd_snpc = np.load('../lfi-experiments/snpec/notebooks_apt/results/gauss'+run_name + '/seed'+str(seed)+'/all_mmds_N5000.npy')
all_mmd_snpc = all_mmd_snpc[:len(all_mmd_snl)]
assert len(all_mmd_snpc) >= 40
all_n_sims_snpc = [(i + 1) * exp_desc.inf.n_samples for i in xrange(all_mmd_snpc.size)]
all_mmds_snpc.append(np.asarray(all_mmd_snpc))
ct += 1
#ax.semilogx(all_n_sims_snpc, np.sqrt(all_mmd_snpc), 'd-', color='k', label='SNPE-C')
except:
print ' could not load SNPE-C results, seed ' + str(seed)
print('ct', ct)
#print([np.sqrt(all_mmd_snpc).shape for all_mmd_snpc in all_mmds_snpc])
mean_mmd_snp = np.mean(np.vstack( np.sqrt(all_mmds_snp)), axis=0)
mean_mmd_snl = np.mean(np.vstack( np.sqrt(all_mmds_snl)), axis=0)
mean_mmd_snpc = np.mean(np.vstack( [np.sqrt(all_mmd_snpc) for all_mmd_snpc in all_mmds_snpc]), axis=0)
sd_mmd_snp = np.std(np.vstack( np.sqrt(all_mmds_snp)), axis=0)
sd_mmd_snl = np.std(np.vstack( np.sqrt(all_mmds_snl)), axis=0)
sd_mmd_snpc = np.std(np.vstack( np.sqrt(all_mmds_snpc)), axis=0)
all_mmds_ppr = [np.pad(all_mmds_ppr[i],
pad_width=(0, np.max( [mean_mmd_snl.size - len(all_mmds_ppr[i]),0])),
mode='constant', constant_values=np.nan) for i in range(len(all_mmds_ppr))]
mean_mmd_ppr = np.nanmean(np.sqrt(np.vstack(all_mmds_ppr)), axis=0)
sd_mmd_ppr = np.nanstd(np.sqrt(np.vstack(all_mmds_ppr)), axis=0)
#print(np.sqrt(np.vstack(all_mmds_ppr))[:, :10])
#print('nan-counts', np.count_nonzero(~np.isnan(all_mmds_snp), axis=0))
#print('nan-counts', np.count_nonzero(~np.isnan(all_mmds_snl), axis=0))
#print('nan-counts', np.count_nonzero(~np.isnan(all_mmds_snpc), axis=0))
#print('nan-counts', np.count_nonzero(~np.isnan(np.vstack(all_mmds_ppr)), axis=0))
sd_mmd_snp /= np.sqrt(np.count_nonzero(~np.isnan(all_mmds_snp), axis=0))
sd_mmd_snl /= np.sqrt(np.count_nonzero(~np.isnan(all_mmds_snl), axis=0))
sd_mmd_snpc /= np.sqrt(np.count_nonzero(~np.isnan(all_mmds_snpc), axis=0))
sd_mmd_ppr /= np.sqrt(np.count_nonzero(~np.isnan(np.vstack(all_mmds_ppr)), axis=0))
matplotlib.rc('text', usetex=True)
matplotlib.rc('font', size=16)
all_n_sims_ppr = all_n_sims_snl if len(all_n_sims_ppr) < len(all_n_sims_snl) else all_n_sims_ppr
ax.semilogx(all_n_sims_ppr, mean_mmd_ppr, '>:', color='c', label='SNPE-A')
ax.semilogx(all_n_sims_snp, mean_mmd_snp, 'p:', color='g', label='SNPE-B')
#ax.semilogx(all_n_sims_nde, all_mmd_nde, 's:', color='b', label='NL')
ax.semilogx(all_n_sims_snl, mean_mmd_snl, 'o:', color='r', label='SNL')
ax.semilogx(all_n_sims_snpc, mean_mmd_snpc, 'd-', color='k', label='APT')
ax.fill_between(all_n_sims_ppr, mean_mmd_ppr-sd_mmd_ppr, mean_mmd_ppr+sd_mmd_ppr, color='c', alpha=0.3)
ax.fill_between(all_n_sims_snp, mean_mmd_snp-sd_mmd_snp, mean_mmd_snp+sd_mmd_snp, color='g', alpha=0.3)
ax.fill_between(all_n_sims_snl, mean_mmd_snl-sd_mmd_snl, mean_mmd_snl+sd_mmd_snl, color='r', alpha=0.3)
ax.fill_between(all_n_sims_snpc, mean_mmd_snpc-sd_mmd_snpc, mean_mmd_snpc+sd_mmd_snpc, color='k', alpha=0.3)
ax.set_xlabel('Number of simulations (log scale)')
ax.set_ylabel('Maximum Mean Discrepancy')
ax.set_xlim([min_n_sims * 10 ** (-0.2), max_n_sims * 10 ** 0.2])
ax.set_ylim([0.0, ax.get_ylim()[1]])
ax.legend(fontsize=14)
#samples = get_true_samples(seed)
#np.save('/home/mackelab/Desktop/Projects/Biophysicality/code/lfi_experiments/snpec/results/gauss/seed'+str(seed)+'/samples', samples)
plt.show()
def plot_results():
# SMC
exp_desc = ed.parse(util.io.load_txt('exps/gauss_smc.txt'))[0]
all_mmd_smc, all_n_sims_smc = get_mmd_smc(exp_desc)
# SL
all_mmd_slk = []
all_n_sims_slk = []
for exp_desc in ed.parse(util.io.load_txt('exps/gauss_sl.txt')):
mmd, n_sims = get_mmd_sl(exp_desc)
all_mmd_slk.append(mmd)
all_n_sims_slk.append(n_sims)
# NDE
all_mmd_nde = []
all_n_sims_nde = []
for exp_desc in ed.parse(util.io.load_txt('exps/gauss_nl.txt')):
all_mmd_nde.append(get_mmd_nde(exp_desc))
all_n_sims_nde.append(exp_desc.inf.n_samples)
all_mmd_ppr = None
all_n_sims_ppr = None
all_mmd_snp = None
all_n_sims_snp = None
all_mmd_snl = None
all_n_sims_snl = None
for exp_desc in ed.parse(util.io.load_txt('exps/gauss_seq.txt')):
# Post Prop
if isinstance(exp_desc.inf, ed.PostProp_Descriptor):
all_prop_mmd, post_mmd = get_mmd_postprop(exp_desc)
all_mmd_ppr = all_prop_mmd + [post_mmd]
all_n_sims_ppr = [(i + 1) * exp_desc.inf.n_samples_p
for i in xrange(len(all_prop_mmd))]
all_n_sims_ppr.append(all_n_sims_ppr[-1] +
exp_desc.inf.n_samples_f)
# SNPE
if isinstance(exp_desc.inf, ed.SNPE_MDN_Descriptor):
all_mmd_snp = get_mmd_snpe(exp_desc)
all_n_sims_snp = [(i + 1) * exp_desc.inf.n_samples
for i in xrange(exp_desc.inf.n_rounds)]
# SNL
if isinstance(exp_desc.inf, ed.SNL_Descriptor):
all_mmd_snl = get_mmd_snl(exp_desc)
all_n_sims_snl = [(i + 1) * exp_desc.inf.n_samples
for i in xrange(exp_desc.inf.n_rounds)]
matplotlib.rc('text', usetex=True)
matplotlib.rc('font', size=16)
all_n_sims = np.concatenate([
all_n_sims_slk, all_n_sims_smc, all_n_sims_ppr, all_n_sims_snp,
all_n_sims_nde, all_n_sims_snl
])
min_n_sims = np.min(all_n_sims)
max_n_sims = np.max(all_n_sims)
fig, ax = plt.subplots(1, 1)
ax.semilogx(all_n_sims_smc,
np.sqrt(all_mmd_smc),
'v:',
color='y',
label='SMC ABC')
ax.semilogx(all_n_sims_slk,
np.sqrt(all_mmd_slk),
'D:',
color='maroon',
label='SL')
ax.semilogx(all_n_sims_ppr,
np.sqrt(all_mmd_ppr),
'>:',
color='c',
label='SNPE-A')
ax.semilogx(all_n_sims_snp,
np.sqrt(all_mmd_snp),
'p:',
color='g',
label='SNPE-B')
ax.semilogx(all_n_sims_nde,
np.sqrt(all_mmd_nde),
's:',
color='b',
label='NL')
ax.semilogx(all_n_sims_snl,
np.sqrt(all_mmd_snl),
'o:',
color='r',
label='SNL')
ax.set_xlabel('Number of simulations (log scale)')
ax.set_ylabel('Maximum Mean Discrepancy')
ax.set_xlim([min_n_sims * 10**(-0.2), max_n_sims * 10**0.2])
ax.set_ylim([0.0, ax.get_ylim()[1]])
ax.legend(fontsize=14)
plt.show()
def plot_results(sim_name, run_name, average):
"""
Plots all results for a given simulator.
"""
fig, ax = plt.subplots(1, 1)
seeds = np.arange(42, 52)
all_dists_ppr, all_dists_snp, all_dists_snl, all_dists_snpc = [],[],[],[]
for i in range(len(seeds)):
seed = seeds[i]
all_dist_ppr = None
all_dist_snp = None
all_dist_snl = None
for exp_desc in ed.parse(
util.io.load_txt('exps/{0}_seq.txt'.format(sim_name))):
# Post Prop
if isinstance(exp_desc.inf, ed.PostProp_Descriptor):
all_dists_ppr.append(get_dist(exp_desc, average, seed))
# SNPE
if isinstance(exp_desc.inf, ed.SNPE_MDN_Descriptor):
all_dists_snp.append(get_dist(exp_desc, average, seed))
# SNL
if isinstance(exp_desc.inf, ed.SNL_Descriptor):
all_dists_snl.append(get_dist(exp_desc, average, seed))
# SNPE-C
try:
all_dists_snpc.append(
np.load('../lfi-experiments/snpec/notebooks_apt/results/' +
sim_name + run_name + '/seed' + str(seed) +
'/avg_dist.npy'))
except:
print ' could not load SNPE-C results, seed ' + str(seed)
all_dists_snpc = [
all_dists_snpc[i][:len(all_dists_snl[i])]
for i in range(len(all_dists_snpc))
]
mean_dist_snp = np.mean(np.vstack(all_dists_snp), axis=0)
mean_dist_snl = np.mean(np.vstack(all_dists_snl), axis=0)
mean_dist_snpc = np.mean(np.vstack(all_dists_snpc), axis=0)
sd_dist_snp = np.std(np.vstack(all_dists_snp), axis=0)
sd_dist_snl = np.std(np.vstack(all_dists_snl), axis=0)
sd_dist_snpc = np.std(np.vstack(all_dists_snpc), axis=0)
all_dists_ppr = [
np.pad(all_dists_ppr[i],
pad_width=(0,
np.max([
mean_dist_snl.size + 1 - len(all_dists_ppr[i]), 0
])),
mode='constant',
constant_values=np.nan) for i in range(len(all_dists_ppr))
]
mean_dist_ppr = np.nanmean(np.vstack(all_dists_ppr), axis=0)
sd_dist_ppr = np.nanstd(np.vstack(all_dists_ppr), axis=0)
matplotlib.rc('text', usetex=True)
matplotlib.rc('font', size=16)
ax.plot(np.arange(len(all_dists_ppr[-1])) + 1,
mean_dist_ppr,
'>:',
color='c',
label='SNPE-A')
ax.plot(np.arange(len(all_dists_snp[-1])) + 1,
mean_dist_snp,
'p:',
color='g',
label='SNPE-B')
ax.plot(np.arange(len(all_dists_snl[-1])) + 1,
mean_dist_snl,
'o:',
color='r',
label='SNL')
ax.plot(np.arange(len(all_dists_snpc[-1])) + 1,
mean_dist_snpc,
'd-',
color='k',
label='APT')
ax.fill_between(np.arange(len(all_dists_ppr[-1])) + 1,
mean_dist_ppr - sd_dist_ppr,
mean_dist_ppr + sd_dist_ppr,
color='c',
alpha=0.3)
ax.fill_between(np.arange(len(all_dists_snp[-1])) + 1,
mean_dist_snp - sd_dist_snp,
mean_dist_snp + sd_dist_snp,
color='g',
alpha=0.3)
ax.fill_between(np.arange(len(all_dists_snl[-1])) + 1,
mean_dist_snl - sd_dist_snl,
mean_dist_snl + sd_dist_snl,
color='r',
alpha=0.3)
ax.fill_between(np.arange(len(all_dists_snpc[-1])) + 1,
mean_dist_snpc - sd_dist_snpc,
mean_dist_snpc + sd_dist_snpc,
color='k',
alpha=0.3)
ax.set_xlabel('Round')
ax.set_ylabel('{0} distance'.format(average[0].upper() + average[1:]))
ax.set_ylim([0.0, ax.get_ylim()[1]])
ax.legend(fontsize=14)
plt.show()
def plot_results(sim_name):
"""
Plots all results for a given simulator.
"""
sim = get_sim(sim_name)
# SMC
exp_desc = ed.parse(util.io.load_txt(
'exps/{0}_smc.txt'.format(sim_name)))[0]
all_err_smc, all_n_sims_smc = get_err_smc(exp_desc, sim)
# SL
all_err_slk = []
all_n_sims_slk = []
for exp_desc in ed.parse(
util.io.load_txt('exps/{0}_sl.txt'.format(sim_name))):
err, n_sims = get_err_sl(exp_desc, sim)
all_err_slk.append(err)
all_n_sims_slk.append(n_sims)
# NDE
all_err_nde = []
all_n_sims_nde = []
for exp_desc in ed.parse(
util.io.load_txt('exps/{0}_nl.txt'.format(sim_name))):
all_err_nde.append(get_err_nde(exp_desc, sim))
all_n_sims_nde.append(exp_desc.inf.n_samples)
all_err_ppr = None
all_n_sims_ppr = None
all_err_snp = None
all_n_sims_snp = None
all_err_snl = None
all_n_sims_snl = None
for exp_desc in ed.parse(
util.io.load_txt('exps/{0}_seq.txt'.format(sim_name))):
# Post Prop
if isinstance(exp_desc.inf, ed.PostProp_Descriptor):
all_prop_err, post_err = get_err_postprop(exp_desc, sim)
all_err_ppr = all_prop_err + [post_err]
all_n_sims_ppr = [(i + 1) * exp_desc.inf.n_samples_p
for i in xrange(len(all_prop_err))]
all_n_sims_ppr.append(all_n_sims_ppr[-1] +
exp_desc.inf.n_samples_f)
# SNPE
if isinstance(exp_desc.inf, ed.SNPE_MDN_Descriptor):
all_err_snp = get_err_snpe(exp_desc, sim)
all_n_sims_snp = [(i + 1) * exp_desc.inf.n_samples
for i in xrange(exp_desc.inf.n_rounds)]
# SNL
if isinstance(exp_desc.inf, ed.SNL_Descriptor):
all_err_snl = get_err_snl(exp_desc, sim)
all_n_sims_snl = [(i + 1) * exp_desc.inf.n_samples
for i in xrange(exp_desc.inf.n_rounds)]
matplotlib.rc('text', usetex=True)
matplotlib.rc('font', size=16)
all_n_sims = np.concatenate([
all_n_sims_slk, all_n_sims_smc, all_n_sims_ppr, all_n_sims_snp,
all_n_sims_nde, all_n_sims_snl
])
min_n_sims = np.min(all_n_sims)
max_n_sims = np.max(all_n_sims)
fig, ax = plt.subplots(1, 1)
ax.semilogx(all_n_sims_smc, all_err_smc, 'v:', color='y', label='SMC ABC')
ax.semilogx(all_n_sims_slk, all_err_slk, 'D:', color='maroon', label='SL')
ax.semilogx(all_n_sims_ppr, all_err_ppr, '>:', color='c', label='SNPE-A')
ax.semilogx(all_n_sims_snp, all_err_snp, 'p:', color='g', label='SNPE-B')
ax.semilogx(all_n_sims_nde, all_err_nde, 's:', color='b', label='NL')
ax.semilogx(all_n_sims_snl, all_err_snl, 'o:', color='r', label='SNL')
ax.set_xlabel('Number of simulations (log scale)')
ax.set_ylabel('$-$log probability of true parameters')
ax.set_xlim([min_n_sims * 10**(-0.2), max_n_sims * 10**0.2])
ax.legend(fontsize=14)
plt.show()
def plot_results(sim_name, run_name=''):
"""
Plots all results for a given simulator.
"""
sim = get_sim(sim_name)
# SMC
#exp_desc = ed.parse(util.io.load_txt('exps/{0}_smc.txt'.format(sim_name)))[0]
#all_err_smc, all_n_sims_smc = get_err_smc(exp_desc, sim)
# SL
#all_err_slk = []
#all_n_sims_slk = []
#for exp_desc in ed.parse(util.io.load_txt('exps/{0}_sl.txt'.format(sim_name))):
# err, n_sims = get_err_sl(exp_desc, sim)
# all_err_slk.append(err)
# all_n_sims_slk.append(n_sims)
# NDE
#all_err_nde = []
#all_n_sims_nde = []
#for exp_desc in ed.parse(util.io.load_txt('exps/{0}_nl.txt'.format(sim_name))):
# all_err_nde.append(get_err_nde(exp_desc, sim))
# all_n_sims_nde.append(exp_desc.inf.n_samples)
print 'sim: ' + str(sim_name)
#print 'seeds: ' + str(seeds)
if run_name == '.':
run_name = ''
fig, ax = plt.subplots(1, 1)
seeds = np.arange(42, 52)
all_errs_ppr, all_errs_snp, all_errs_snl, all_errs_snpc = [],[],[],[]
ct = 0
for i in range(len(seeds)):
seed = seeds[i]
all_err_ppr = None
all_n_sims_ppr = None
all_err_snp = None
all_n_sims_snp = None
all_err_snl = None
all_n_sims_snl = None
"""
# SMC
exp_desc = ed.parse(util.io.load_txt('exps/{0}_smc.txt'.format(sim_name)))[0]
if i==1:
all_err_smc = np.nan*np.ones_like(all_err_smc)
all_n_sims_smc = all_n_sims_smc
else:
all_err_smc, all_n_sims_smc = get_err_smc(exp_desc, sim, seed)
ct += 1
if i == 0:
ax.semilogx(all_n_sims_smc, all_err_smc, 'v:', color='y', label='SMC ABC')
"""
for exp_desc in ed.parse(util.io.load_txt('exps/{0}_seq.txt'.format(sim_name))):
# Post Prop
if isinstance(exp_desc.inf, ed.PostProp_Descriptor):
all_prop_err, post_err = get_err_postprop(exp_desc, sim, seed)
all_err_ppr = all_prop_err + [post_err]
all_n_sims_ppr = [(i + 1) * exp_desc.inf.n_samples_p for i in xrange(len(all_prop_err))]
all_n_sims_ppr.append(all_n_sims_ppr[-1] + exp_desc.inf.n_samples_f)
all_errs_ppr.append(all_err_ppr)
# SNPE
if isinstance(exp_desc.inf, ed.SNPE_MDN_Descriptor):
all_err_snp = get_err_snpe(exp_desc, sim, seed)
all_n_sims_snp = [(i + 1) * exp_desc.inf.n_samples for i in xrange(exp_desc.inf.n_rounds)]
all_errs_snp.append(all_err_snp)
# SNL
if isinstance(exp_desc.inf, ed.SNL_Descriptor):
all_err_snl = get_err_snl(exp_desc, sim, seed)
all_n_sims_snl = [(i + 1) * exp_desc.inf.n_samples for i in xrange(exp_desc.inf.n_rounds)]
all_errs_snl.append(all_err_snl)
all_n_sims = np.concatenate([all_n_sims_ppr, all_n_sims_snp, all_n_sims_snl])
min_n_sims = np.min(all_n_sims)
max_n_sims = np.max(all_n_sims)
# SNPE-C
try:
all_err_snpc = np.load('../lfi-experiments/snpec/notebooks_apt/results/'+sim_name+run_name+'/seed'+str(seed)+'/all_prop_errs_N5000.npy')
all_n_sims_snpc = [(i + 1) * exp_desc.inf.n_samples for i in xrange(all_err_snpc.size)]
all_errs_snpc.append(all_err_snpc)
except:
print ' could not load SNPE-C results, seed ' + str(seed)
mean_err_snp = np.mean(np.vstack(all_errs_snp), axis=0)
mean_err_snl = np.mean(np.vstack(all_errs_snl), axis=0)
mean_err_snpc = np.nanmean(np.vstack(all_errs_snpc), axis=0)
sd_err_snp = np.std(np.vstack(all_errs_snp), axis=0)
sd_err_snl = np.std(np.vstack(all_errs_snl), axis=0)
sd_err_snpc = np.nanstd(np.vstack(all_errs_snpc), axis=0)
all_errs_ppr = [np.pad(all_errs_ppr[i],
pad_width=(0, np.max( [mean_err_snl.size +1 - len(all_errs_ppr[i]),0])),
mode='constant', constant_values=np.nan) for i in range(len(all_errs_ppr))]
mean_err_ppr = np.nanmean(np.vstack(all_errs_ppr), axis=0)
sd_err_ppr = np.nanstd(np.vstack(all_errs_ppr), axis=0)
#print('nan-counts', np.count_nonzero(~np.isnan(all_errs_snp), axis=0))
#print('nan-counts', np.count_nonzero(~np.isnan(all_errs_snl), axis=0))
#print('nan-counts', np.count_nonzero(~np.isnan(all_errs_snpc), axis=0))
#print('nan-counts', np.count_nonzero(~np.isnan(np.vstack(all_errs_ppr)), axis=0))
sd_err_snp /= np.sqrt(np.count_nonzero(~np.isnan(all_errs_snp), axis=0))
sd_err_snl /= np.sqrt(np.count_nonzero(~np.isnan(all_errs_snl), axis=0))
sd_err_snpc /= np.sqrt(np.count_nonzero(~np.isnan(all_errs_snpc), axis=0))
sd_err_ppr /= np.sqrt(np.count_nonzero(~np.isnan(np.vstack(all_errs_ppr)), axis=0))
matplotlib.rc('text', usetex=True)
matplotlib.rc('font', size=16)
if sim_name=='lv':
all_n_sims_ppr = np.arange(1,12) * 1000
elif sim_name=='gauss':
all_n_sims_ppr = np.arange(1,42) * 1000
else:
all_n_sims_ppr = all_n_sims_snl if len(all_n_sims_ppr) < len(all_n_sims_snl) else all_n_sims_ppr
print('mean_err_ppr', mean_err_ppr)
#ax.semilogx(all_n_sims_smc, all_err_smc, 'v:', color='y', label='SMC ABC')
#ax.semilogx(all_n_sims_slk, all_err_slk, 'D:', color='maroon', label='SL')
ax.semilogx(all_n_sims_ppr, mean_err_ppr, '>:', color='c', label='SNPE-A')
ax.semilogx(all_n_sims_snp, mean_err_snp, 'p:', color='g', label='SNPE-B')
#ax.semilogx(all_n_sims_nde, all_err_nde, 's:', color='b', label='NL')
ax.semilogx(all_n_sims_snl, mean_err_snl, 'o:', color='r', label='SNL')
ax.semilogx(all_n_sims_snpc, mean_err_snpc, 'd-', color='k', label='APT')
ax.fill_between(all_n_sims_ppr, mean_err_ppr-sd_err_ppr, mean_err_ppr+sd_err_ppr, color='c', alpha=0.3)
ax.fill_between(all_n_sims_snp, mean_err_snp-sd_err_snp, mean_err_snp+sd_err_snp, color='g', alpha=0.3)
ax.fill_between(all_n_sims_snl, mean_err_snl-sd_err_snl, mean_err_snl+sd_err_snl, color='r', alpha=0.3)
ax.fill_between(all_n_sims_snpc, mean_err_snpc-sd_err_snpc, mean_err_snpc+sd_err_snpc, color='k', alpha=0.3)
ax.set_xlabel('Number of simulations (log scale)')
#ax.set_ticks([100, 1000], fontsize=14)
ax.set_ylabel('$-$log probability of true parameters')
ax.set_xlim([min_n_sims * 10 ** (-0.2), max_n_sims * 10 ** 0.2])
#ax.legend(fontsize=14)
plt.xticks([1000, 10000], fontsize=16)
plt.show()
print('ct', ct)
