def abc_hodsim(Mr=21, b_normal=0.25): prior_min, prior_max = PriorRange('first_try') prior_range = np.zeros((len(prior_min),2)) prior_range[:,0] = prior_min prior_range[:,1] = prior_max abc_hod = ABC_HODsim() data_hod_dict = Data.data_hod_param(Mr=21) data_hod = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) print Data.data_nbar(Mr=Mr, b_normal=b_normal), Data.data_gmf(Mr=Mr, b_normal=b_normal) print abc_hod(data_hod, prior_range=prior_range, observables=['nbar', 'gmf'])
def abc_hodsim(Mr=21, b_normal=0.25): prior_min, prior_max = PriorRange('first_try') prior_range = np.zeros((len(prior_min), 2)) prior_range[:, 0] = prior_min prior_range[:, 1] = prior_max abc_hod = ABC_HODsim() data_hod_dict = Data.data_hod_param(Mr=21) data_hod = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) print Data.data_nbar(Mr=Mr, b_normal=b_normal), Data.data_gmf(Mr=Mr, b_normal=b_normal) print abc_hod(data_hod, prior_range=prior_range, observables=['nbar', 'gmf'])
def ABC_Coner(obvs, weighted=False): ''' Pretty corner plot ''' if obvs == 'nbargmf': result_dir = ''.join([ ut.dat_dir(), 'paper/ABC', obvs, '/run1/', ]) theta_file = lambda tt: ''.join( [result_dir, 'nbar_gmf_theta_t', str(tt), '.ABCnbargmf.dat']) w_file = lambda tt: ''.join( [result_dir, 'nbar_gmf_w_t', str(tt), '.ABCnbargmf.dat']) tf = 8 elif obvs == 'nbarxi': result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/']) theta_file = lambda tt: ''.join( [result_dir, 'nbar_xi_theta_t', str(tt), '.abc.dat']) w_file = lambda tt: ''.join( [result_dir, 'nbar_xi_w_t', str(tt), '.abc.dat']) tf = 9 else: raise ValueError theta = np.loadtxt(theta_file(tf)) if weighted: weights = np.loadtxt(w_file(tf)) else: weights = None true_dict = Data.data_hod_param(Mr=21) true_theta = [ true_dict['logM0'], np.log(true_dict['sigma_logM']), true_dict['logMmin'], true_dict['alpha'], true_dict['logM1'] ] prior_min, prior_max = PriorRange('first_try') prior_range = np.zeros((len(prior_min), 2)) prior_range[:, 0] = np.array(prior_min) prior_range[:, 1] = np.array(prior_max) fig = corner.corner(theta, weights=weights, truths=true_theta, truth_color='k', labels=[ r'$\log\;\mathcal{M}_{0}}$', r'$\log\;\sigma_\mathtt{log\;M}}$', r'$\log\;\mathcal{M}_\mathtt{min}}$', r'$\alpha$', r'$\log\;\mathcal{M}_{1}}$' ], label_kwargs={'fontsize': 25}, range=prior_range, quantiles=[0.16, 0.5, 0.84], show_titles=True, title_args={"fontsize": 12}, plot_datapoints=True, fill_contours=True, levels=[0.68, 0.95], color='#ee6a50', bins=20, smooth=1.0) fig_name = ''.join([ut.fig_dir(), 'paper.ABCcorner', '.', obvs, '.pdf']) fig.savefig(fig_name, bbox_inches='tight', dpi=150) plt.close() return None
def plot_mcmc(Nwalkers, Niter=1000, Nchains_burn=200, Mr=21, truths=None, observables=['nbar', 'xi'], plot_range=None): ''' Plot MCMC chains ''' if truths is None: data_hod_dict = Data.data_hod_param(Mr=Mr) truths = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) if plot_range is None: prior_min, prior_max = PriorRange(None) plot_range = np.zeros((len(prior_min),2)) plot_range[:,0] = prior_min plot_range[:,1] = prior_max # chain files chain_file = ''.join([util.dat_dir(), util.observable_id_flag(observables), '_Mr', str(Mr), '.mcmc_chain.dat']) #f = h5py.File(chain_file, 'r') #sample = f['positions'][:] sample = np.loadtxt(chain_file) # Posterior Likelihood Corner Plot fig = corner.corner( sample[Nchains_burn*Nwalkers:], truths=truths, truth_color='#ee6a50', labels=[ r'$\mathtt{\log\;M_{0}}$', r'$\mathtt{\log\;\sigma_{\logM}}$', r'$\mathtt{\log\;M_{min}}$', r'$\mathtt{\alpha}$', r'$\mathtt{\log\;M_{1}}$' ], label_kwargs={'fontsize': 25}, range=plot_range, quantiles=[0.16,0.5,0.84], show_titles=True, title_args={"fontsize": 12}, plot_datapoints=True, fill_contours=True, levels=[0.68, 0.95], color='b', bins=16, smooth=1.0) fig_file = ''.join([util.fig_dir(), util.observable_id_flag(observables), '_Mr', str(Mr), '.Niter', str(Niter), '.Nburn', str(Nchains_burn), '.mcmc_samples.test.png']) #print fig_file plt.savefig(fig_file) plt.close() # MCMC Chain plot Ndim = len(sample[0]) Nchain = len(sample)/Nwalkers chain_ensemble = sample.reshape(Nchain, Nwalkers, Ndim) fig , axes = plt.subplots(5, 1 , sharex=True, figsize=(10, 12)) labels=[ r'$\mathtt{\log\;M_{0}}$', r'$\mathtt{\log\;\sigma_{\logM}}$', r'$\mathtt{\log\;M_{min}}$', r'$\mathtt{\alpha}$', r'$\mathtt{\log\;M_{1}}$' ] for i in xrange(5): axes[i].plot(chain_ensemble[:, :, i], color="k", alpha=0.4) axes[i].yaxis.set_major_locator(MaxNLocator(5)) axes[i].axhline(truths[i], color="#888888", lw=2) axes[i].vlines(Nchains_burn, plot_range[i,0], plot_range[i,1], colors='#ee6a50', linewidth=4, alpha=1) axes[i].set_ylim([plot_range[i,0], plot_range[i,1]]) axes[i].set_xlim(0, 6000) axes[i].set_ylabel(labels[i], fontsize=25) axes[4].set_xlabel("Step Number", fontsize=25) fig.tight_layout(h_pad=0.0) fig_file = ''.join([util.fig_dir(), util.observable_id_flag(observables), '_Mr', str(Mr), '.Niter', str(Niter), '.Nburn', str(Nchains_burn), '.mcmc_time.test.png']) plt.savefig(fig_file) plt.close()
def mcmc_mpi(Nwalkers, Nchains, observables=['nbar', 'xi'], data_dict={ 'Mr': 21, 'b_normal': 0.25 }, prior_name='first_try', mcmcrun=None): ''' Standard MCMC implementaion Parameters ----------- - Nwalker : Number of walkers - Nchains : Number of MCMC chains - observables : list of observables. Options are: ['nbar','xi'],['nbar','gmf'],['xi'] - data_dict : dictionary that specifies the observation keywords ''' #Initializing the vector of observables and inverse covariance matrix if observables == ['xi']: fake_obs = Data.data_xi(**data_dict) #fake_obs_icov = Data.data_inv_cov('xi', **data_dict) fake_obs_icov = Data.data_cov(inference='mcmc', **data_dict)[1:16, 1:16] if observables == ['nbar', 'xi']: fake_obs = np.hstack( [Data.data_nbar(**data_dict), Data.data_xi(**data_dict)]) fake_obs_icov = Data.data_cov(inference='mcmc', **data_dict)[:16, :16] if observables == ['nbar', 'gmf']: ##### FIRST BIN OF GMF DROPPED ############### # CAUTION: hardcoded fake_obs = np.hstack( [Data.data_nbar(**data_dict), Data.data_gmf(**data_dict)[1:]]) fake_obs_icov = np.zeros((10, 10)) #print Data.data_cov(**data_dict)[17: , 17:].shape # Covariance matrix being adjusted accordingly fake_obs_icov[1:, 1:] = Data.data_cov(inference='mcmc', **data_dict)[17:, 17:] fake_obs_icov[0, 1:] = Data.data_cov(inference='mcmc', **data_dict)[0, 17:] fake_obs_icov[1:, 0] = Data.data_cov(inference='mcmc', **data_dict)[17:, 0] fake_obs_icov[0, 0] = Data.data_cov(inference='mcmc', **data_dict)[0, 0] # True HOD parameters data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr']) data_hod = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) Ndim = len(data_hod) # Priors prior_min, prior_max = PriorRange(prior_name) prior_range = np.zeros((len(prior_min), 2)) prior_range[:, 0] = prior_min prior_range[:, 1] = prior_max # mcmc chain output file chain_file = ''.join([ util.mcmc_dir(), util.observable_id_flag(observables), '.', mcmcrun, '.mcmc_chain.dat' ]) #print chain_file if os.path.isfile(chain_file) and continue_chain: print 'Continuing previous MCMC chain!' sample = np.loadtxt(chain_file) Nchain = Niter - (len(sample) / Nwalkers ) # Number of chains left to finish if Nchain > 0: pass else: raise ValueError print Nchain, ' iterations left to finish' # Initializing Walkers from the end of the chain pos0 = sample[-Nwalkers:] else: # new chain f = open(chain_file, 'w') f.close() Nchain = Niter # Initializing Walkers random_guess = data_hod pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + \ 5.e-2 * np.random.randn(Ndim * Nwalkers).reshape(Nwalkers, Ndim) #print pos0.shape # Initializing MPIPool pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) # Initializing the emcee sampler hod_kwargs = { 'prior_range': prior_range, 'data': fake_obs, 'data_icov': fake_obs_icov, 'observables': observables, 'Mr': data_dict['Mr'] } sampler = emcee.EnsembleSampler(Nwalkers, Ndim, lnPost, pool=pool, kwargs=hod_kwargs) # Initializing Walkers for result in sampler.sample(pos0, iterations=Nchain, storechain=False): position = result[0] #print position f = open(chain_file, 'a') for k in range(position.shape[0]): output_str = '\t'.join(position[k].astype('str')) + '\n' f.write(output_str) f.close() pool.close()
for obv in observables: if obv == 'nbar': data_nbar, data_nbar_var = Data.data_nbar(**data_dict) fake_obs.append(data_nbar) if obv == 'gmf': data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict) fake_obs.append(data_gmf) if obv == 'xi': # import xir and full covariance matrix of xir data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict) # take the inverse of the covariance matrix data_xi_invcov = solve(np.eye(len(data_xi)), data_xi_cov) fake_obs.append(data_xi) # 'True' HOD parameters data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr']) data_hod = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) Ndim = len(data_hod) # Priors prior_min, prior_max = PriorRange(prior_name) prior_range = np.zeros((len(prior_min), 2)) prior_range[:, 0] = prior_min prior_range[:, 1] = prior_max
def ABCvsMCMC_contour(obvs, nwalkers=100, nburns=9000, sigma=False): ''' Plots that compare the ABC posteriors to the MCMC posteriors ''' if obvs == 'nbargmf': abc_dir = ''.join([ ut.dat_dir(), 'paper/ABC', obvs, '/run1/', ]) abc_theta_file = lambda tt: ''.join( [abc_dir, 'nbar_gmf_theta_t', str(tt), '.ABCnbargmf.dat']) tf = 8 mcmc_dir = ''.join([ut.dat_dir(), 'paper/']) mcmc_filename = ''.join([mcmc_dir, 'nbar_gmf.mcmc.mcmc_chain.p']) elif obvs == 'nbarxi': abc_dir = ''.join([ ut.dat_dir(), 'paper/ABC', obvs, '/', ]) abc_theta_file = lambda tt: ''.join( [abc_dir, 'nbar_xi_theta_t', str(tt), '.abc.dat']) tf = 9 mcmc_dir = ''.join([ut.dat_dir(), 'paper/']) mcmc_filename = ''.join([mcmc_dir, 'nbar_xi.mcmc.mcmc_chain.p']) else: raise ValueError prior_min, prior_max = PriorRange('first_try') prior_range = np.zeros((len(prior_min), 2)) prior_range[:, 0] = prior_min prior_range[:, 1] = prior_max # true HOD parameter true_dict = Data.data_hod_param(Mr=21) truths = [ true_dict['logM0'], # log M0 np.log(true_dict['sigma_logM']), # log(sigma) true_dict['logMmin'], # log Mmin true_dict['alpha'], # alpha true_dict['logM1'] # log M1 ] mcmc_sample = pickle.load(open(mcmc_filename, 'rb'))[nburns * nwalkers:, :] abc_sample = np.loadtxt(abc_theta_file(tf)) par_labels = [ r'$\mathtt{log}\;\mathcal{M}_{0}$', r'$\mathtt{log}\;\sigma_\mathtt{log\;M}$', r'$\mathtt{log}\;\mathcal{M}_\mathtt{min}$', r'$\alpha$', r'$\mathtt{log}\;\mathcal{M}_{1}$' ] prettyplot() pretty_colors = prettycolors() fig = plt.figure(1, figsize=(20, 6)) gs = gridspec.GridSpec(1, 3) # first panel for i in [0, 1, 2]: plot_range = np.zeros((2, 2)) if i == 0: col_pair = [2, 3] plot_range[0, 0] = 12.5 plot_range[0, 1] = 13.0 plot_range[1, 0] = prior_range[3, 0] plot_range[1, 1] = prior_range[3, 1] elif i == 2: col_pair = [4, 2] plot_range[0, 0] = 13.6 plot_range[0, 1] = 14.2 plot_range[1, 0] = 12.5 plot_range[1, 1] = 13.0 elif i == 1: col_pair = [3, 4] plot_range[0, 0] = prior_range[3, 0] plot_range[0, 1] = prior_range[3, 1] plot_range[1, 0] = 13.6 plot_range[1, 1] = 14.2 if i == 2: mcmc_label = r'$\mathcal{L}^\mathtt{Gauss}$ MCMC' abc_label = 'ABC-PMC' else: mcmc_label = None abc_label = None mcmc_par1 = mcmc_sample[:, col_pair[0]] mcmc_par2 = mcmc_sample[:, col_pair[1]] abc_par1 = abc_sample[:, col_pair[0]] abc_par2 = abc_sample[:, col_pair[1]] ax = plt.subplot(gs[i]) if sigma: lvls = [1 - np.exp(-0.5), 1 - np.exp(-0.125)] else: lvls = [0.68, 0.95] corner.hist2d(mcmc_par1, mcmc_par2, bins=20, range=plot_range, ax=ax, plot_datapoints=False, levels=lvls, color='#1F77B4', fill_contours=True, smooth=1.0, label=mcmc_label) corner.hist2d(abc_par1, abc_par2, bins=20, range=plot_range, ax=ax, levels=lvls, color='#FF7F0E', fill_contours=True, smooth=1.0, label=abc_label) ax.scatter(np.repeat(truths[col_pair[0]], 2), np.repeat(truths[col_pair[1]], 2), s=100, marker='*', c='k', lw=0, label=None) #ax.axvline(truths[i_col], color='k', ls='--', linewidth=3) #ax.set_xticklabels([]) ax.set_xlim([plot_range[0, 0], plot_range[0, 1]]) ax.set_ylim([plot_range[1, 0], plot_range[1, 1]]) if i == 2: thick_line1 = mlines.Line2D([], [], ls='-', c='#FF7F0E', linewidth=12, alpha=0.5, label='ABC-PMC') ax.legend(loc='upper right', handles=[thick_line1], frameon=False, fontsize=25, handletextpad=0.1, scatteryoffsets=[0.5]) elif i == 1: thick_line2 = mlines.Line2D( [], [], ls='-', c='#1F77B4', linewidth=12, alpha=0.5, label='$\mathcal{L}^\mathtt{Gauss}$ \nMCMC') ax.legend(loc='upper right', handles=[thick_line2], frameon=False, fontsize=25, handletextpad=0.1, scatteryoffsets=[0.5]) ax.set_xlabel(par_labels[col_pair[0]], fontsize=25, labelpad=15) ax.set_ylabel(par_labels[col_pair[1]], fontsize=25) if sigma: sigma_str = '.true1sigma' else: sigma_str = '' fig.subplots_adjust(wspace=0.3) fig_name = ''.join([ ut.fig_dir(), 'paper.ABCvsMCMC.contour', '.', obvs, sigma_str, '.pdf' ]) fig.savefig(fig_name, bbox_inches='tight', dpi=150) return None
for obv in observables: if obv == 'nbar': data_nbar, data_nbar_var = Data.data_nbar(**data_dict) fake_obs.append(data_nbar) if obv == 'gmf': data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict) fake_obs.append(data_gmf) if obv == 'xi': # import xir and full covariance matrix of xir data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict) # take the inverse of the covariance matrix data_xi_invcov = solve(np.eye(len(data_xi)) , data_xi_cov) fake_obs.append(data_xi) # 'True' HOD parameters data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr']) data_hod = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) Ndim = len(data_hod) # Priors prior_min, prior_max = PriorRange(prior_name) prior_range = np.zeros((len(prior_min),2)) prior_range[:,0] = prior_min prior_range[:,1] = prior_max
def mcmc_ipython_par(Nwalkers, Nchains_burn, Nchains_pro, observables=['nbar', 'xi'], data_dict={ 'Mr': 20, 'Nmock': 500 }, prior_name='first_try', threads=1): ''' Standard MCMC implementaion Parameters ----------- - Nwalker : Number of walkers - Nchains_burn : Number of burn-in chains - Nchains_pro : Number of production chains - observables : list of observables. Options are 'nbar', 'gmf', 'xi' - data_dict : dictionary that specifies the observation keywords ''' # data observables fake_obs = [] # list of observables fake_obs_cov = [] for obv in observables: if obv == 'nbar': data_nbar, data_nbar_var = Data.data_nbar(**data_dict) fake_obs.append(data_nbar) fake_obs_cov.append(data_nbar_var) if obv == 'gmf': data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict) fake_obs.append(data_gmf) fake_obs_cov.append(data_gmf) if obv == 'xi': # import xir and full covariance matrix of xir data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict) data_xi_invcov = Data.data_xi_inv_cov(**data_dict) fake_obs.append(data_xi) fake_obs_cov.append(data_xi_invcov) # True HOD parameters data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr']) data_hod = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) Ndim = len(data_hod) # Priors prior_min, prior_max = PriorRange(prior_name) prior_range = np.zeros((len(prior_min), 2)) prior_range[:, 0] = prior_min prior_range[:, 1] = prior_max # Initializing Walkers random_guess = np.array([11., np.log(.4), 11.5, 1.0, 13.5]) pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + \ 1e-1 * np.random.randn(Ndim * Nwalkers).reshape(Nwalkers, Ndim) # Initializing the emcee sampler hod_kwargs = { 'prior_range': prior_range, 'data': fake_obs, 'data_cov': fake_obs_cov, 'observables': observables, 'Mr': data_dict['Mr'] } # Set up the interface to the ipcluster. c = Client() view = c[:] view.push({"lnPost": lnPost}) # Modules necessary in posterior calculation should be called here view.execute("import numpy as np") view.execute("from hod_sim import HODsimulator") # Setting up the Sampler sampler = emcee.EnsembleSampler(Nwalkers, Ndim, lnPost, kwargs=hod_kwargs, pool=view) # Setting up a file for saving the chains chain_file = ''.join([ util.dat_dir(), util.observable_id_flag(observables), '_Mr', str(data_dict["Mr"]), '_theta.mcmc_chain.dat' ]) f = open(chain_file, "w") f.close() # Running the Sampler and writing out the chains for result in sampler.sample(pos0, iterations=Nchains_burn + Nchains_pro, storechain=False): position = result[0] f = open(chain_file, "a") for k in range(position.shape[0]): output_str = '\t'.join(position[k].astype('str')) + '\n' f.write(output_str) f.close()
def ABCvsMCMC_contour(obvs, nwalkers=100, nburns=9000, sigma=False): ''' Plots that compare the ABC posteriors to the MCMC posteriors ''' if obvs == 'nbargmf': abc_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/run1/',]) abc_theta_file = lambda tt: ''.join([abc_dir, 'nbar_gmf_theta_t', str(tt), '.ABCnbargmf.dat']) tf = 8 mcmc_dir = ''.join([ut.dat_dir(), 'paper/']) mcmc_filename = ''.join([mcmc_dir, 'nbar_gmf.mcmc.mcmc_chain.p']) elif obvs == 'nbarxi': abc_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/',]) abc_theta_file = lambda tt: ''.join([abc_dir, 'nbar_xi_theta_t', str(tt), '.abc.dat']) tf = 9 mcmc_dir = ''.join([ut.dat_dir(), 'paper/']) mcmc_filename = ''.join([mcmc_dir, 'nbar_xi.mcmc.mcmc_chain.p']) else: raise ValueError prior_min, prior_max = PriorRange('first_try') prior_range = np.zeros((len(prior_min),2)) prior_range[:,0] = prior_min prior_range[:,1] = prior_max # true HOD parameter true_dict = Data.data_hod_param(Mr=21) truths = [ true_dict['logM0'], # log M0 np.log(true_dict['sigma_logM']), # log(sigma) true_dict['logMmin'], # log Mmin true_dict['alpha'], # alpha true_dict['logM1'] # log M1 ] mcmc_sample = pickle.load(open(mcmc_filename, 'rb'))[nburns*nwalkers:,:] abc_sample = np.loadtxt(abc_theta_file(tf)) par_labels = [ r'$\mathtt{log}\;\mathcal{M}_{0}$', r'$\mathtt{log}\;\sigma_\mathtt{log\;M}$', r'$\mathtt{log}\;\mathcal{M}_\mathtt{min}$', r'$\alpha$', r'$\mathtt{log}\;\mathcal{M}_{1}$' ] prettyplot() pretty_colors = prettycolors() fig = plt.figure(1, figsize=(20,6)) gs = gridspec.GridSpec(1, 3) # first panel for i in [0, 1, 2]: plot_range = np.zeros((2,2)) if i == 0: col_pair = [2, 3] plot_range[0, 0] = 12.5 plot_range[0, 1] = 13.0 plot_range[1, 0] = prior_range[3, 0] plot_range[1, 1] = prior_range[3, 1] elif i == 2: col_pair = [4, 2] plot_range[0, 0] = 13.6 plot_range[0, 1] = 14.2 plot_range[1, 0] = 12.5 plot_range[1, 1] = 13.0 elif i == 1: col_pair = [3, 4] plot_range[0, 0] = prior_range[3, 0] plot_range[0, 1] = prior_range[3, 1] plot_range[1, 0] = 13.6 plot_range[1, 1] = 14.2 if i == 2: mcmc_label = r'$\mathcal{L}^\mathtt{Gauss}$ MCMC' abc_label = 'ABC-PMC' else: mcmc_label = None abc_label = None mcmc_par1 = mcmc_sample[:,col_pair[0]] mcmc_par2 = mcmc_sample[:,col_pair[1]] abc_par1 = abc_sample[:, col_pair[0]] abc_par2 = abc_sample[:, col_pair[1]] ax = plt.subplot(gs[i]) if sigma: lvls = [1-np.exp(-0.5), 1-np.exp(-0.125)] else: lvls = [0.68, 0.95] corner.hist2d(mcmc_par1, mcmc_par2, bins=20, range=plot_range, ax = ax, plot_datapoints=False, levels=lvls, color='#1F77B4', fill_contours=True, smooth=1.0, label=mcmc_label) corner.hist2d(abc_par1, abc_par2, bins=20, range=plot_range, ax = ax, levels=lvls, color='#FF7F0E', fill_contours=True, smooth=1.0, label=abc_label) ax.scatter(np.repeat(truths[col_pair[0]],2), np.repeat(truths[col_pair[1]],2), s=100, marker='*', c='k', lw=0, label=None) #ax.axvline(truths[i_col], color='k', ls='--', linewidth=3) #ax.set_xticklabels([]) ax.set_xlim([plot_range[0, 0] , plot_range[0, 1]]) ax.set_ylim([plot_range[1, 0] , plot_range[1, 1]]) if i == 2: thick_line1 = mlines.Line2D([], [], ls='-', c='#FF7F0E', linewidth=12, alpha=0.5, label='ABC-PMC') ax.legend(loc='upper right', handles=[thick_line1], frameon=False, fontsize=25, handletextpad=0.1, scatteryoffsets=[0.5]) elif i == 1: thick_line2 = mlines.Line2D([], [], ls='-', c='#1F77B4', linewidth=12, alpha=0.5, label='$\mathcal{L}^\mathtt{Gauss}$ \nMCMC') ax.legend(loc='upper right', handles=[thick_line2], frameon=False, fontsize=25, handletextpad=0.1, scatteryoffsets=[0.5]) ax.set_xlabel(par_labels[col_pair[0]], fontsize=25, labelpad=15) ax.set_ylabel(par_labels[col_pair[1]], fontsize=25) if sigma: sigma_str = '.true1sigma' else: sigma_str = '' fig.subplots_adjust(wspace=0.3) fig_name = ''.join([ut.fig_dir(), 'paper.ABCvsMCMC.contour', '.', obvs, sigma_str, '.pdf']) fig.savefig(fig_name, bbox_inches='tight', dpi=150) return None
def PoolEvolution(obvs): ''' Demostrative plot for the evolution of the pool. Illustrate the pool evolution for log M_min versus log M_1, which has the starkest evolution from its prior. ''' if obvs == 'nbargmf': result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/run1/',]) theta_file = lambda tt: ''.join([result_dir, 'nbar_gmf_theta_t', str(tt), '.ABCnbargmf.dat']) t_list = [0, 1, 2, 3, 5, 8] elif obvs == 'nbarxi': result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/']) theta_file = lambda tt: ''.join([result_dir, 'nbar_xi_theta_t', str(tt), '.abc.dat']) t_list = [0, 1, 2, 3, 7, 9] else: raise ValueError prior_min, prior_max = PriorRange('first_try') prior_range = np.zeros((2,2)) prior_range[:,0] = np.array([prior_min[2], prior_min[4]]) prior_range[:,1] = np.array([prior_max[2], prior_max[4]]) # true HOD parameter true_dict = Data.data_hod_param(Mr=21) true_pair = [true_dict['logMmin'], true_dict['logM1']] prettyplot() pretty_colors = prettycolors() fig = plt.figure(figsize=(12,8)) all_fig = fig.add_subplot(111) for i_t, t in enumerate(t_list): sub = fig.add_subplot(2, len(t_list)/2, i_t+1) theta_Mmin, theta_M1 = np.loadtxt(theta_file(t), unpack=True, usecols=[2, 4]) corner.hist2d(theta_Mmin, theta_M1, bins=20, range=prior_range, levels=[0.68, 0.95], color='c', fill_contours=True, smooth=1.0) t_label = r"$\mathtt{t = "+str(t)+"}$" sub.text(13.0, 15.0, t_label, fontsize=25) if i_t == len(t_list) - 1: true_label = r'$``\mathtt{true}"$' else: true_label = None plt.scatter(np.repeat(true_pair[0],2), np.repeat(true_pair[1],2), s=75, marker='*', c='k', lw=0, label=true_label) if i_t == len(t_list) - 1: plt.legend(loc='lower left', scatterpoints=1, markerscale=2.5, handletextpad=-0.25, scatteryoffsets=[0.5]) if i_t == 0: sub.set_xticklabels([]) sub.set_yticklabels([13., 13.5, 14., 14.5, 15., 15.5]) elif (i_t > 0) and (i_t < len(t_list)/2): sub.set_xticklabels([]) sub.set_yticklabels([]) elif i_t == len(t_list)/2: sub.set_yticklabels([13., 13.5, 14., 14.5, 15.]) elif i_t > len(t_list)/2: sub.set_yticklabels([]) all_fig.set_xticklabels([]) all_fig.set_yticklabels([]) all_fig.set_ylabel( r'$\mathtt{log}\;\mathcal{M}_\mathtt{1}$', fontsize=30, labelpad=50) all_fig.set_xlabel( r'$\mathtt{log}\;\mathcal{M}_\mathtt{min}$', fontsize=30, labelpad=25) fig.subplots_adjust(hspace=0.0) fig_name = ''.join([ut.fig_dir(), 'paper_ABC_poolevolution', '.', obvs, '.pdf']) fig.savefig(fig_name, bbox_inches='tight', dpi=150) plt.close()
def ABCvsMCMC_histogram(obvs, nwalkers=100, nburns=9000): ''' Plots that compare the ABC posteriors to the MCMC posteriors ''' if obvs == 'nbargmf': abc_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/run1/',]) abc_theta_file = lambda tt: ''.join([abc_dir, 'nbar_gmf_theta_t', str(tt), '.ABCnbargmf.dat']) tf = 8 mcmc_dir = ''.join([ut.dat_dir(), 'paper/']) mcmc_filename = ''.join([mcmc_dir, 'nbar_gmf.mcmc.mcmc_chain.p']) elif obvs == 'nbarxi': abc_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/',]) abc_theta_file = lambda tt: ''.join([abc_dir, 'nbar_xi_theta_t', str(tt), '.abc.dat']) tf = 9 mcmc_dir = ''.join([ut.dat_dir(), 'paper/']) mcmc_filename = ''.join([mcmc_dir, 'nbar_xi.mcmc.mcmc_chain.p']) else: raise ValueError prior_min, prior_max = PriorRange('first_try') prior_range = np.zeros((len(prior_min),2)) prior_range[:,0] = prior_min prior_range[:,1] = prior_max # true HOD parameter true_dict = Data.data_hod_param(Mr=21) truths = [ true_dict['logM0'], # log M0 np.log(true_dict['sigma_logM']), # log(sigma) true_dict['logMmin'], # log Mmin true_dict['alpha'], # alpha true_dict['logM1'] # log M1 ] mcmc_sample = pickle.load(open(mcmc_filename, 'rb'))[nburns*nwalkers:,:] abc_sample = np.loadtxt(abc_theta_file(tf)) normie = norm() sig1lo = normie.cdf(-1) sig1hi = normie.cdf(1) sig2lo = normie.cdf(-2) sig2hi = normie.cdf(2) nsamples_mcmc = len(mcmc_sample[:, 2]) nsamples_abc = len(abc_sample[:, 2]) sig1lo_mcmc = int(sig1lo * nsamples_mcmc) sig2lo_mcmc = int(sig2lo * nsamples_mcmc) sig1hi_mcmc = int(sig1hi * nsamples_mcmc) sig2hi_mcmc = int(sig2hi * nsamples_mcmc) sig1lo_abc = int(sig1lo * nsamples_abc) sig2lo_abc = int(sig2lo * nsamples_abc) sig1hi_abc = int(sig1hi * nsamples_abc) sig2hi_abc = int(sig2hi * nsamples_abc) par_labels = [ r'$\mathtt{log}\;\mathcal{M}_{0}$', r'$\mathtt{log}\;\sigma_\mathtt{log\;M}$', r'$\mathtt{log}\;\mathcal{M}_\mathtt{min}$', r'$\alpha$', r'$\mathtt{log}\;\mathcal{M}_{1}$' ] prettyplot() pretty_colors = prettycolors() fig = plt.figure(1, figsize=(20,8)) gs = gridspec.GridSpec(2, 3, height_ratios=[2.75, 1]) # first panel for i in [0, 1, 2]: if i == 0: i_col = 2 prior_range[i_col,0] = 12.5 prior_range[i_col,1] = 13. plot_range = prior_range[i_col, :] elif i == 1: i_col = 3 plot_range = prior_range[i_col, :] elif i == 2: i_col = 4 prior_range[i_col,0] = 13.5 prior_range[i_col,1] = 14.25 plot_range = np.array([13.5, 14.5]) ax = plt.subplot(gs[i]) q = ax.hist(mcmc_sample[:,i_col], bins=20, range=[prior_range[i_col,0], prior_range[i_col,1]], normed=True, alpha=0.75, color=pretty_colors[1], linewidth=2, histtype='stepfilled', edgecolor=None ) qq = ax.hist(abc_sample[:,i_col], bins=20, range=[prior_range[i_col,0], prior_range[i_col,1]], normed=True, alpha=0.75, color=pretty_colors[3], linewidth=2, histtype='stepfilled', edgecolor=None ) ax.axvline(truths[i_col], color='k', ls='--', linewidth=3) ax.set_xticklabels([]) ax.set_xlim([plot_range[0] , plot_range[1]]) if i == 2: thick_line2 = mlines.Line2D([], [], ls='-', c=pretty_colors[1], linewidth=4, label='$\mathcal{L}^\mathtt{Gauss}$ \nMCMC') thick_line1 = mlines.Line2D([], [], ls='-', c=pretty_colors[3], linewidth=4, label='ABC-PMC') ax.legend(loc='upper right', handles=[thick_line2, thick_line1], frameon=False, fontsize=25, handletextpad=-0.0) # general box properties boxprops = {'color': 'k'} medianprops = {'alpha': 0.} bplots1 = [] ax = plt.subplot(gs[i+3]) # stats dict for each box bplots1.append({'med': np.median(mcmc_sample[:, i_col]), 'q1': np.sort(mcmc_sample[:, i_col])[sig1lo_mcmc], 'q3': np.sort(mcmc_sample[:, i_col])[sig1hi_mcmc], 'whislo': np.sort(mcmc_sample[:, i_col])[sig2lo_mcmc], 'whishi': np.sort(mcmc_sample[:, i_col])[sig2hi_mcmc], 'fliers': []}) bplots1.append({'med': np.median(abc_sample[:, i_col]), 'q1': np.sort(abc_sample[:, i_col])[sig1lo_abc], 'q3': np.sort(abc_sample[:, i_col])[sig1hi_abc], 'whislo': np.sort(abc_sample[:, i_col])[sig2lo_abc], 'whishi': np.sort(abc_sample[:, i_col])[sig2hi_abc], 'fliers': []}) whiskprop = dict(linestyle='-', linewidth=2, color='k') boxprops = dict(linestyle='-', linewidth=2, color='k') bxp1 = ax.bxp(bplots1, positions=[1,2], vert=False, patch_artist=True, showfliers=False, boxprops=boxprops, medianprops=medianprops, whiskerprops=whiskprop) for ibox, box in enumerate(bxp1['boxes']): if ibox == 0: box.set(facecolor=pretty_colors[1], alpha=0.75) elif ibox == 1: box.set(facecolor=pretty_colors[3], alpha=0.75) ax.axvline(truths[i_col], color='k', ls='--', linewidth=3) ax.set_xlim([plot_range[0] , plot_range[1]]) ax.set_xlabel(par_labels[i_col], fontsize=25, labelpad=15) if i == 0: ax.set_yticks([1,2]) ax.set_yticklabels([r"$\mathtt{MCMC}$", r"$\mathtt{ABC}$"]) else: ax.set_yticks([]) fig.subplots_adjust(wspace=0.2, hspace=0.0) fig_name = ''.join([ut.fig_dir(), 'paper.ABCvsMCMC', '.', obvs, '.pdf']) print fig_name fig.savefig(fig_name, bbox_inches='tight', dpi=150) return None
def ABC_Convergence(weighted=False): ''' Plot the error bars on the parameters as a function of time step ''' prettyplot() pretty_colors = prettycolors() fig = plt.figure(figsize=(20, 10)) for i_obv, obvs in enumerate(['nbargmf', 'nbarxi']): if obvs == 'nbargmf': result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/run1/',]) theta_file = lambda tt: ''.join([result_dir, 'nbar_gmf_theta_t', str(tt), '.ABCnbargmf.dat']) w_file = lambda tt: ''.join([result_dir, 'nbar_gmf_w_t', str(tt), '.ABCnbargmf.dat']) tf = 8 elif obvs == 'nbarxi': result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/']) theta_file = lambda tt: ''.join([result_dir, 'nbar_xi_theta_t', str(tt), '.abc.dat']) w_file = lambda tt: ''.join([result_dir, 'nbar_xi_w_t', str(tt), '.abc.dat']) tf = 9 else: raise ValueError t_list = range(tf+1) columns = [ r"$\mathtt{log}\;\mathcal{M}_0$", r"$\sigma_{\mathtt{log}\;\mathcal{M}}$", r"$\mathtt{log}\;\mathcal{M}_\mathtt{min}$", r"$\alpha$", r"$\mathtt{log}\;\mathcal{M}_1$" ] true_dict = Data.data_hod_param(Mr=21) true_theta = [ true_dict['logM0'], np.log(true_dict['sigma_logM']), true_dict['logMmin'], true_dict['alpha'], true_dict['logM1'] ] prior_min, prior_max = PriorRange('first_try') a_theta = np.zeros((len(t_list), 5)) b_theta = np.zeros((len(t_list), 5)) d_theta = np.zeros((len(t_list), 5)) e_theta = np.zeros((len(t_list), 5)) for i_t, tt in enumerate(t_list): theta_i = np.loadtxt(theta_file(tt), unpack=True) w_i = np.loadtxt(w_file(tt)) for i_par in range(len(theta_i)): if not weighted: a, b, d, e = np.percentile(theta_i[i_par], [2.5, 16, 84, 97.5], axis=0) else: a = ut.quantile_1D(theta_i[i_par], w_i, 0.025) b = ut.quantile_1D(theta_i[i_par], w_i, 0.16) d = ut.quantile_1D(theta_i[i_par], w_i, 0.84) e = ut.quantile_1D(theta_i[i_par], w_i, 0.975) a_theta[i_t, i_par] = a b_theta[i_t, i_par] = b d_theta[i_t, i_par] = d e_theta[i_t, i_par] = e keep_index = [2,3,4] for ii, i in enumerate(keep_index): if i == keep_index[-1]: true_label = r'$``\mathtt{true}"$' abc_1sig_label = r'ABC Pool' else: true_label = None abc_1sig_label = None sub = fig.add_subplot(2, len(keep_index), i_obv * len(keep_index) + ii+1) sub.fill_between(t_list, a_theta[:, i], e_theta[:,i], color=pretty_colors[3], alpha=0.3, edgecolor="none") sub.fill_between(t_list, b_theta[:, i], d_theta[:,i], color=pretty_colors[3], alpha=0.5, edgecolor="none", label=abc_1sig_label) sub.plot(t_list, np.repeat(true_theta[i], len(t_list)), c='k', ls='--', lw=2, label=true_label) if ii == 0: if obvs == 'nbargmf': sub.text(4.85, 13.4, r"$\bar{\mathtt{n}}$ and $\zeta(\mathtt{N})$", fontsize=25) elif obvs == 'nbarxi': sub.text(4.85, 13.4, r"$\bar{\mathtt{n}}$ and $\xi(\mathtt{r})$", fontsize=25) sub.set_ylabel(columns[i], fontsize=25) sub.set_ylim([prior_min[i], prior_max[i]]) sub.set_xlim([-1, 10]) if i_obv == 1: sub.legend(loc='upper right', borderpad=1.) sub.set_xlabel('iterations', fontsize=25) if i == 4: sub.set_yticklabels([13.0, 13.5, 14.0, 14.5, 15.]) else: sub.set_xticklabels([]) fig.subplots_adjust(wspace=0.3, hspace=0.0) if weighted: weight_str = '.weighted' else: weight_str = '' fig_name = ''.join([ut.fig_dir(), 'paper', '.ABCconvergence', weight_str, '.pdf']) fig.savefig(fig_name, bbox_inches='tight', dpi=150) plt.close() return None
def ABC_Coner(obvs, weighted=False): ''' Pretty corner plot ''' if obvs == 'nbargmf': result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/run1/',]) theta_file = lambda tt: ''.join([result_dir, 'nbar_gmf_theta_t', str(tt), '.ABCnbargmf.dat']) w_file = lambda tt: ''.join([result_dir, 'nbar_gmf_w_t', str(tt), '.ABCnbargmf.dat']) tf = 8 elif obvs == 'nbarxi': result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/']) theta_file = lambda tt: ''.join([result_dir, 'nbar_xi_theta_t', str(tt), '.abc.dat']) w_file = lambda tt: ''.join([result_dir, 'nbar_xi_w_t', str(tt), '.abc.dat']) tf = 9 else: raise ValueError theta = np.loadtxt(theta_file(tf)) if weighted: weights = np.loadtxt(w_file(tf)) else: weights = None true_dict = Data.data_hod_param(Mr=21) true_theta = [ true_dict['logM0'], np.log(true_dict['sigma_logM']), true_dict['logMmin'], true_dict['alpha'], true_dict['logM1'] ] prior_min, prior_max = PriorRange('first_try') prior_range = np.zeros((len(prior_min),2)) prior_range[:,0] = np.array(prior_min) prior_range[:,1] = np.array(prior_max) fig = corner.corner( theta, weights=weights, truths=true_theta, truth_color='k', labels=[ r'$\log\;\mathcal{M}_{0}}$', r'$\log\;\sigma_\mathtt{log\;M}}$', r'$\log\;\mathcal{M}_\mathtt{min}}$', r'$\alpha$', r'$\log\;\mathcal{M}_{1}}$' ], label_kwargs={'fontsize': 25}, range=prior_range, quantiles=[0.16,0.5,0.84], show_titles=True, title_args={"fontsize": 12}, plot_datapoints=True, fill_contours=True, levels=[0.68, 0.95], color='#ee6a50', bins=20, smooth=1.0) fig_name = ''.join([ut.fig_dir(), 'paper.ABCcorner', '.', obvs, '.pdf']) fig.savefig(fig_name, bbox_inches='tight', dpi=150) plt.close() return None
def ABC_Convergence(weighted=False): ''' Plot the error bars on the parameters as a function of time step ''' prettyplot() pretty_colors = prettycolors() fig = plt.figure(figsize=(20, 10)) for i_obv, obvs in enumerate(['nbargmf', 'nbarxi']): if obvs == 'nbargmf': result_dir = ''.join([ ut.dat_dir(), 'paper/ABC', obvs, '/run1/', ]) theta_file = lambda tt: ''.join( [result_dir, 'nbar_gmf_theta_t', str(tt), '.ABCnbargmf.dat']) w_file = lambda tt: ''.join( [result_dir, 'nbar_gmf_w_t', str(tt), '.ABCnbargmf.dat']) tf = 8 elif obvs == 'nbarxi': result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/']) theta_file = lambda tt: ''.join( [result_dir, 'nbar_xi_theta_t', str(tt), '.abc.dat']) w_file = lambda tt: ''.join( [result_dir, 'nbar_xi_w_t', str(tt), '.abc.dat']) tf = 9 else: raise ValueError t_list = range(tf + 1) columns = [ r"$\mathtt{log}\;\mathcal{M}_0$", r"$\sigma_{\mathtt{log}\;\mathcal{M}}$", r"$\mathtt{log}\;\mathcal{M}_\mathtt{min}$", r"$\alpha$", r"$\mathtt{log}\;\mathcal{M}_1$" ] true_dict = Data.data_hod_param(Mr=21) true_theta = [ true_dict['logM0'], np.log(true_dict['sigma_logM']), true_dict['logMmin'], true_dict['alpha'], true_dict['logM1'] ] prior_min, prior_max = PriorRange('first_try') a_theta = np.zeros((len(t_list), 5)) b_theta = np.zeros((len(t_list), 5)) d_theta = np.zeros((len(t_list), 5)) e_theta = np.zeros((len(t_list), 5)) for i_t, tt in enumerate(t_list): theta_i = np.loadtxt(theta_file(tt), unpack=True) w_i = np.loadtxt(w_file(tt)) for i_par in range(len(theta_i)): if not weighted: a, b, d, e = np.percentile(theta_i[i_par], [2.5, 16, 84, 97.5], axis=0) else: a = ut.quantile_1D(theta_i[i_par], w_i, 0.025) b = ut.quantile_1D(theta_i[i_par], w_i, 0.16) d = ut.quantile_1D(theta_i[i_par], w_i, 0.84) e = ut.quantile_1D(theta_i[i_par], w_i, 0.975) a_theta[i_t, i_par] = a b_theta[i_t, i_par] = b d_theta[i_t, i_par] = d e_theta[i_t, i_par] = e keep_index = [2, 3, 4] for ii, i in enumerate(keep_index): if i == keep_index[-1]: true_label = r'$``\mathtt{true}"$' abc_1sig_label = r'ABC Pool' else: true_label = None abc_1sig_label = None sub = fig.add_subplot(2, len(keep_index), i_obv * len(keep_index) + ii + 1) sub.fill_between(t_list, a_theta[:, i], e_theta[:, i], color=pretty_colors[3], alpha=0.3, edgecolor="none") sub.fill_between(t_list, b_theta[:, i], d_theta[:, i], color=pretty_colors[3], alpha=0.5, edgecolor="none", label=abc_1sig_label) sub.plot(t_list, np.repeat(true_theta[i], len(t_list)), c='k', ls='--', lw=2, label=true_label) if ii == 0: if obvs == 'nbargmf': sub.text(4.85, 13.4, r"$\bar{\mathtt{n}}$ and $\zeta(\mathtt{N})$", fontsize=25) elif obvs == 'nbarxi': sub.text(4.85, 13.4, r"$\bar{\mathtt{n}}$ and $\xi(\mathtt{r})$", fontsize=25) sub.set_ylabel(columns[i], fontsize=25) sub.set_ylim([prior_min[i], prior_max[i]]) sub.set_xlim([-1, 10]) if i_obv == 1: sub.legend(loc='upper right', borderpad=1.) sub.set_xlabel('iterations', fontsize=25) if i == 4: sub.set_yticklabels([13.0, 13.5, 14.0, 14.5, 15.]) else: sub.set_xticklabels([]) fig.subplots_adjust(wspace=0.3, hspace=0.0) if weighted: weight_str = '.weighted' else: weight_str = '' fig_name = ''.join( [ut.fig_dir(), 'paper', '.ABCconvergence', weight_str, '.pdf']) fig.savefig(fig_name, bbox_inches='tight', dpi=150) plt.close() return None
def plot_mcmc(Nwalkers, Niter=1000, Nchains_burn=200, Mr=21, truths=None, observables=['nbar', 'xi'], plot_range=None): ''' Plot MCMC chains ''' if truths is None: data_hod_dict = Data.data_hod_param(Mr=Mr) truths = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) if plot_range is None: prior_min, prior_max = PriorRange(None) plot_range = np.zeros((len(prior_min), 2)) plot_range[:, 0] = prior_min plot_range[:, 1] = prior_max # chain files chain_file = ''.join([ util.dat_dir(), util.observable_id_flag(observables), '_Mr', str(Mr), '.mcmc_chain.dat' ]) #f = h5py.File(chain_file, 'r') #sample = f['positions'][:] sample = np.loadtxt(chain_file) # Posterior Likelihood Corner Plot fig = corner.corner(sample[Nchains_burn * Nwalkers:], truths=truths, truth_color='#ee6a50', labels=[ r'$\mathtt{\log\;M_{0}}$', r'$\mathtt{\log\;\sigma_{\logM}}$', r'$\mathtt{\log\;M_{min}}$', r'$\mathtt{\alpha}$', r'$\mathtt{\log\;M_{1}}$' ], label_kwargs={'fontsize': 25}, range=plot_range, quantiles=[0.16, 0.5, 0.84], show_titles=True, title_args={"fontsize": 12}, plot_datapoints=True, fill_contours=True, levels=[0.68, 0.95], color='b', bins=16, smooth=1.0) fig_file = ''.join([ util.fig_dir(), util.observable_id_flag(observables), '_Mr', str(Mr), '.Niter', str(Niter), '.Nburn', str(Nchains_burn), '.mcmc_samples.test.png' ]) #print fig_file plt.savefig(fig_file) plt.close() # MCMC Chain plot Ndim = len(sample[0]) Nchain = len(sample) / Nwalkers chain_ensemble = sample.reshape(Nchain, Nwalkers, Ndim) fig, axes = plt.subplots(5, 1, sharex=True, figsize=(10, 12)) labels = [ r'$\mathtt{\log\;M_{0}}$', r'$\mathtt{\log\;\sigma_{\logM}}$', r'$\mathtt{\log\;M_{min}}$', r'$\mathtt{\alpha}$', r'$\mathtt{\log\;M_{1}}$' ] for i in xrange(5): axes[i].plot(chain_ensemble[:, :, i], color="k", alpha=0.4) axes[i].yaxis.set_major_locator(MaxNLocator(5)) axes[i].axhline(truths[i], color="#888888", lw=2) axes[i].vlines(Nchains_burn, plot_range[i, 0], plot_range[i, 1], colors='#ee6a50', linewidth=4, alpha=1) axes[i].set_ylim([plot_range[i, 0], plot_range[i, 1]]) axes[i].set_xlim(0, 6000) axes[i].set_ylabel(labels[i], fontsize=25) axes[4].set_xlabel("Step Number", fontsize=25) fig.tight_layout(h_pad=0.0) fig_file = ''.join([ util.fig_dir(), util.observable_id_flag(observables), '_Mr', str(Mr), '.Niter', str(Niter), '.Nburn', str(Nchains_burn), '.mcmc_time.test.png' ]) plt.savefig(fig_file) plt.close()
def ABCvsMCMC_histogram(obvs, nwalkers=100, nburns=9000): ''' Plots that compare the ABC posteriors to the MCMC posteriors ''' if obvs == 'nbargmf': abc_dir = ''.join([ ut.dat_dir(), 'paper/ABC', obvs, '/run1/', ]) abc_theta_file = lambda tt: ''.join( [abc_dir, 'nbar_gmf_theta_t', str(tt), '.ABCnbargmf.dat']) tf = 8 mcmc_dir = ''.join([ut.dat_dir(), 'paper/']) mcmc_filename = ''.join([mcmc_dir, 'nbar_gmf.mcmc.mcmc_chain.p']) elif obvs == 'nbarxi': abc_dir = ''.join([ ut.dat_dir(), 'paper/ABC', obvs, '/', ]) abc_theta_file = lambda tt: ''.join( [abc_dir, 'nbar_xi_theta_t', str(tt), '.abc.dat']) tf = 9 mcmc_dir = ''.join([ut.dat_dir(), 'paper/']) mcmc_filename = ''.join([mcmc_dir, 'nbar_xi.mcmc.mcmc_chain.p']) else: raise ValueError prior_min, prior_max = PriorRange('first_try') prior_range = np.zeros((len(prior_min), 2)) prior_range[:, 0] = prior_min prior_range[:, 1] = prior_max # true HOD parameter true_dict = Data.data_hod_param(Mr=21) truths = [ true_dict['logM0'], # log M0 np.log(true_dict['sigma_logM']), # log(sigma) true_dict['logMmin'], # log Mmin true_dict['alpha'], # alpha true_dict['logM1'] # log M1 ] mcmc_sample = pickle.load(open(mcmc_filename, 'rb'))[nburns * nwalkers:, :] abc_sample = np.loadtxt(abc_theta_file(tf)) normie = norm() sig1lo = normie.cdf(-1) sig1hi = normie.cdf(1) sig2lo = normie.cdf(-2) sig2hi = normie.cdf(2) nsamples_mcmc = len(mcmc_sample[:, 2]) nsamples_abc = len(abc_sample[:, 2]) sig1lo_mcmc = int(sig1lo * nsamples_mcmc) sig2lo_mcmc = int(sig2lo * nsamples_mcmc) sig1hi_mcmc = int(sig1hi * nsamples_mcmc) sig2hi_mcmc = int(sig2hi * nsamples_mcmc) sig1lo_abc = int(sig1lo * nsamples_abc) sig2lo_abc = int(sig2lo * nsamples_abc) sig1hi_abc = int(sig1hi * nsamples_abc) sig2hi_abc = int(sig2hi * nsamples_abc) par_labels = [ r'$\mathtt{log}\;\mathcal{M}_{0}$', r'$\mathtt{log}\;\sigma_\mathtt{log\;M}$', r'$\mathtt{log}\;\mathcal{M}_\mathtt{min}$', r'$\alpha$', r'$\mathtt{log}\;\mathcal{M}_{1}$' ] prettyplot() pretty_colors = prettycolors() fig = plt.figure(1, figsize=(20, 8)) gs = gridspec.GridSpec(2, 3, height_ratios=[2.75, 1]) # first panel for i in [0, 1, 2]: if i == 0: i_col = 2 prior_range[i_col, 0] = 12.5 prior_range[i_col, 1] = 13. plot_range = prior_range[i_col, :] elif i == 1: i_col = 3 plot_range = prior_range[i_col, :] elif i == 2: i_col = 4 prior_range[i_col, 0] = 13.5 prior_range[i_col, 1] = 14.25 plot_range = np.array([13.5, 14.5]) ax = plt.subplot(gs[i]) q = ax.hist(mcmc_sample[:, i_col], bins=20, range=[prior_range[i_col, 0], prior_range[i_col, 1]], normed=True, alpha=0.75, color=pretty_colors[1], linewidth=2, histtype='stepfilled', edgecolor=None) qq = ax.hist(abc_sample[:, i_col], bins=20, range=[prior_range[i_col, 0], prior_range[i_col, 1]], normed=True, alpha=0.75, color=pretty_colors[3], linewidth=2, histtype='stepfilled', edgecolor=None) ax.axvline(truths[i_col], color='k', ls='--', linewidth=3) ax.set_xticklabels([]) ax.set_xlim([plot_range[0], plot_range[1]]) if i == 2: thick_line2 = mlines.Line2D( [], [], ls='-', c=pretty_colors[1], linewidth=4, label='$\mathcal{L}^\mathtt{Gauss}$ \nMCMC') thick_line1 = mlines.Line2D([], [], ls='-', c=pretty_colors[3], linewidth=4, label='ABC-PMC') ax.legend(loc='upper right', handles=[thick_line2, thick_line1], frameon=False, fontsize=25, handletextpad=-0.0) # general box properties boxprops = {'color': 'k'} medianprops = {'alpha': 0.} bplots1 = [] ax = plt.subplot(gs[i + 3]) # stats dict for each box bplots1.append({ 'med': np.median(mcmc_sample[:, i_col]), 'q1': np.sort(mcmc_sample[:, i_col])[sig1lo_mcmc], 'q3': np.sort(mcmc_sample[:, i_col])[sig1hi_mcmc], 'whislo': np.sort(mcmc_sample[:, i_col])[sig2lo_mcmc], 'whishi': np.sort(mcmc_sample[:, i_col])[sig2hi_mcmc], 'fliers': [] }) bplots1.append({ 'med': np.median(abc_sample[:, i_col]), 'q1': np.sort(abc_sample[:, i_col])[sig1lo_abc], 'q3': np.sort(abc_sample[:, i_col])[sig1hi_abc], 'whislo': np.sort(abc_sample[:, i_col])[sig2lo_abc], 'whishi': np.sort(abc_sample[:, i_col])[sig2hi_abc], 'fliers': [] }) whiskprop = dict(linestyle='-', linewidth=2, color='k') boxprops = dict(linestyle='-', linewidth=2, color='k') bxp1 = ax.bxp(bplots1, positions=[1, 2], vert=False, patch_artist=True, showfliers=False, boxprops=boxprops, medianprops=medianprops, whiskerprops=whiskprop) for ibox, box in enumerate(bxp1['boxes']): if ibox == 0: box.set(facecolor=pretty_colors[1], alpha=0.75) elif ibox == 1: box.set(facecolor=pretty_colors[3], alpha=0.75) ax.axvline(truths[i_col], color='k', ls='--', linewidth=3) ax.set_xlim([plot_range[0], plot_range[1]]) ax.set_xlabel(par_labels[i_col], fontsize=25, labelpad=15) if i == 0: ax.set_yticks([1, 2]) ax.set_yticklabels([r"$\mathtt{MCMC}$", r"$\mathtt{ABC}$"]) else: ax.set_yticks([]) fig.subplots_adjust(wspace=0.2, hspace=0.0) fig_name = ''.join([ut.fig_dir(), 'paper.ABCvsMCMC', '.', obvs, '.pdf']) print fig_name fig.savefig(fig_name, bbox_inches='tight', dpi=150) return None
def mcmc_mpi( Nwalkers, Nchains, observables=["nbar", "xi"], data_dict={"Mr": 21, "b_normal": 0.25}, prior_name="first_try", mcmcrun=None, ): """ Standard MCMC implementaion Parameters ----------- - Nwalker : Number of walkers - Nchains : Number of MCMC chains - observables : list of observables. Options are: ['nbar','xi'],['nbar','gmf'],['xi'] - data_dict : dictionary that specifies the observation keywords """ # Initializing the vector of observables and inverse covariance matrix if observables == ["xi"]: fake_obs = Data.data_xi(**data_dict) # fake_obs_icov = Data.data_inv_cov('xi', **data_dict) fake_obs_icov = Data.data_cov(inference="mcmc", **data_dict)[1:16, 1:16] if observables == ["nbar", "xi"]: fake_obs = np.hstack([Data.data_nbar(**data_dict), Data.data_xi(**data_dict)]) fake_obs_icov = Data.data_cov(inference="mcmc", **data_dict)[:16, :16] if observables == ["nbar", "gmf"]: ##### FIRST BIN OF GMF DROPPED ############### # CAUTION: hardcoded fake_obs = np.hstack([Data.data_nbar(**data_dict), Data.data_gmf(**data_dict)[1:]]) fake_obs_icov = np.zeros((10, 10)) # print Data.data_cov(**data_dict)[17: , 17:].shape # Covariance matrix being adjusted accordingly fake_obs_icov[1:, 1:] = Data.data_cov(inference="mcmc", **data_dict)[17:, 17:] fake_obs_icov[0, 1:] = Data.data_cov(inference="mcmc", **data_dict)[0, 17:] fake_obs_icov[1:, 0] = Data.data_cov(inference="mcmc", **data_dict)[17:, 0] fake_obs_icov[0, 0] = Data.data_cov(inference="mcmc", **data_dict)[0, 0] # True HOD parameters data_hod_dict = Data.data_hod_param(Mr=data_dict["Mr"]) data_hod = np.array( [ data_hod_dict["logM0"], # log M0 np.log(data_hod_dict["sigma_logM"]), # log(sigma) data_hod_dict["logMmin"], # log Mmin data_hod_dict["alpha"], # alpha data_hod_dict["logM1"], # log M1 ] ) Ndim = len(data_hod) # Priors prior_min, prior_max = PriorRange(prior_name) prior_range = np.zeros((len(prior_min), 2)) prior_range[:, 0] = prior_min prior_range[:, 1] = prior_max # mcmc chain output file chain_file = "".join([util.mcmc_dir(), util.observable_id_flag(observables), ".", mcmcrun, ".mcmc_chain.dat"]) # print chain_file if os.path.isfile(chain_file) and continue_chain: print "Continuing previous MCMC chain!" sample = np.loadtxt(chain_file) Nchain = Niter - (len(sample) / Nwalkers) # Number of chains left to finish if Nchain > 0: pass else: raise ValueError print Nchain, " iterations left to finish" # Initializing Walkers from the end of the chain pos0 = sample[-Nwalkers:] else: # new chain f = open(chain_file, "w") f.close() Nchain = Niter # Initializing Walkers random_guess = data_hod pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + 5.0e-2 * np.random.randn( Ndim * Nwalkers ).reshape(Nwalkers, Ndim) # print pos0.shape # Initializing MPIPool pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) # Initializing the emcee sampler hod_kwargs = { "prior_range": prior_range, "data": fake_obs, "data_icov": fake_obs_icov, "observables": observables, "Mr": data_dict["Mr"], } sampler = emcee.EnsembleSampler(Nwalkers, Ndim, lnPost, pool=pool, kwargs=hod_kwargs) # Initializing Walkers for result in sampler.sample(pos0, iterations=Nchain, storechain=False): position = result[0] # print position f = open(chain_file, "a") for k in range(position.shape[0]): output_str = "\t".join(position[k].astype("str")) + "\n" f.write(output_str) f.close() pool.close()
def PoolEvolution(obvs): ''' Demostrative plot for the evolution of the pool. Illustrate the pool evolution for log M_min versus log M_1, which has the starkest evolution from its prior. ''' if obvs == 'nbargmf': result_dir = ''.join([ ut.dat_dir(), 'paper/ABC', obvs, '/run1/', ]) theta_file = lambda tt: ''.join( [result_dir, 'nbar_gmf_theta_t', str(tt), '.ABCnbargmf.dat']) t_list = [0, 1, 2, 3, 5, 8] elif obvs == 'nbarxi': result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/']) theta_file = lambda tt: ''.join( [result_dir, 'nbar_xi_theta_t', str(tt), '.abc.dat']) t_list = [0, 1, 2, 3, 7, 9] else: raise ValueError prior_min, prior_max = PriorRange('first_try') prior_range = np.zeros((2, 2)) prior_range[:, 0] = np.array([prior_min[2], prior_min[4]]) prior_range[:, 1] = np.array([prior_max[2], prior_max[4]]) # true HOD parameter true_dict = Data.data_hod_param(Mr=21) true_pair = [true_dict['logMmin'], true_dict['logM1']] prettyplot() pretty_colors = prettycolors() fig = plt.figure(figsize=(12, 8)) all_fig = fig.add_subplot(111) for i_t, t in enumerate(t_list): sub = fig.add_subplot(2, len(t_list) / 2, i_t + 1) theta_Mmin, theta_M1 = np.loadtxt(theta_file(t), unpack=True, usecols=[2, 4]) corner.hist2d(theta_Mmin, theta_M1, bins=20, range=prior_range, levels=[0.68, 0.95], color='c', fill_contours=True, smooth=1.0) t_label = r"$\mathtt{t = " + str(t) + "}$" sub.text(13.0, 15.0, t_label, fontsize=25) if i_t == len(t_list) - 1: true_label = r'$``\mathtt{true}"$' else: true_label = None plt.scatter(np.repeat(true_pair[0], 2), np.repeat(true_pair[1], 2), s=75, marker='*', c='k', lw=0, label=true_label) if i_t == len(t_list) - 1: plt.legend(loc='lower left', scatterpoints=1, markerscale=2.5, handletextpad=-0.25, scatteryoffsets=[0.5]) if i_t == 0: sub.set_xticklabels([]) sub.set_yticklabels([13., 13.5, 14., 14.5, 15., 15.5]) elif (i_t > 0) and (i_t < len(t_list) / 2): sub.set_xticklabels([]) sub.set_yticklabels([]) elif i_t == len(t_list) / 2: sub.set_yticklabels([13., 13.5, 14., 14.5, 15.]) elif i_t > len(t_list) / 2: sub.set_yticklabels([]) all_fig.set_xticklabels([]) all_fig.set_yticklabels([]) all_fig.set_ylabel(r'$\mathtt{log}\;\mathcal{M}_\mathtt{1}$', fontsize=30, labelpad=50) all_fig.set_xlabel(r'$\mathtt{log}\;\mathcal{M}_\mathtt{min}$', fontsize=30, labelpad=25) fig.subplots_adjust(hspace=0.0) fig_name = ''.join( [ut.fig_dir(), 'paper_ABC_poolevolution', '.', obvs, '.pdf']) fig.savefig(fig_name, bbox_inches='tight', dpi=150) plt.close()
def mcmc_ipython_par(Nwalkers, Nchains_burn, Nchains_pro, observables=['nbar', 'xi'], data_dict={'Mr':20, 'Nmock':500}, prior_name = 'first_try', threads=1): ''' Standard MCMC implementaion Parameters ----------- - Nwalker : Number of walkers - Nchains_burn : Number of burn-in chains - Nchains_pro : Number of production chains - observables : list of observables. Options are 'nbar', 'gmf', 'xi' - data_dict : dictionary that specifies the observation keywords ''' # data observables fake_obs = [] # list of observables fake_obs_cov = [] for obv in observables: if obv == 'nbar': data_nbar, data_nbar_var = Data.data_nbar(**data_dict) fake_obs.append(data_nbar) fake_obs_cov.append(data_nbar_var) if obv == 'gmf': data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict) fake_obs.append(data_gmf) fake_obs_cov.append(data_gmf) if obv == 'xi': # import xir and full covariance matrix of xir data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict) data_xi_invcov = Data.data_xi_inv_cov(**data_dict) fake_obs.append(data_xi) fake_obs_cov.append(data_xi_invcov) # True HOD parameters data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr']) data_hod = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) Ndim = len(data_hod) # Priors prior_min, prior_max = PriorRange(prior_name) prior_range = np.zeros((len(prior_min),2)) prior_range[:,0] = prior_min prior_range[:,1] = prior_max # Initializing Walkers random_guess = np.array([11. , np.log(.4) , 11.5 , 1.0 , 13.5]) pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + \ 1e-1 * np.random.randn(Ndim * Nwalkers).reshape(Nwalkers, Ndim) # Initializing the emcee sampler hod_kwargs = { 'prior_range': prior_range, 'data': fake_obs, 'data_cov': fake_obs_cov, 'observables': observables, 'Mr': data_dict['Mr'] } # Set up the interface to the ipcluster. c = Client() view = c[:] view.push({"lnPost": lnPost}) # Modules necessary in posterior calculation should be called here view.execute("import numpy as np") view.execute("from hod_sim import HODsimulator") # Setting up the Sampler sampler = emcee.EnsembleSampler(Nwalkers, Ndim, lnPost, kwargs=hod_kwargs, pool = view) # Setting up a file for saving the chains chain_file = ''.join([util.dat_dir(), util.observable_id_flag(observables), '_Mr', str(data_dict["Mr"]), '_theta.mcmc_chain.dat']) f = open(chain_file, "w") f.close() # Running the Sampler and writing out the chains for result in sampler.sample(pos0, iterations=Nchains_burn + Nchains_pro, storechain=False): position = result[0] f = open(chain_file, "a") for k in range(position.shape[0]): output_str = '\t'.join(position[k].astype('str')) + '\n' f.write(output_str) f.close()
def ABCpmc_HOD(T, eps_val, N_part=1000, prior_name='first_try', observables=['nbar', 'xi'], abcrun=None, data_dict={'Mr':21, 'b_normal':0.25}): ''' ABC-PMC implementation. Parameters ---------- - T : Number of iterations - eps_val : - N_part : Number of particles - observables : list of observables. Options are 'nbar', 'gmf', 'xi' - data_dict : dictionary that specifies the observation keywords ''' if abcrun is None: raise ValueError("Specify the name of the abcrun!") #Initializing the vector of observables and inverse covariance matrix fake_obs, Cii_list = getObvs(observables, **data_dict) # True HOD parameters data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr']) data_hod = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) # Priors prior_min, prior_max = PriorRange(prior_name) prior = abcpmc.TophatPrior(prior_min, prior_max) prior_range = np.zeros((len(prior_min),2)) prior_range[:,0] = prior_min prior_range[:,1] = prior_max # Simulator our_model = ABC_HODsim(Mr=data_dict['Mr'], b_normal=data_dict['b_normal']) # initialize model kwargs = {'prior_range': prior_range, 'observables': observables} def simz(tt): sim = our_model(tt, **kwargs) if sim is None: pickle.dump(tt, open(util.crash_dir()+"simz_crash_theta.p", 'wb')) pickle.dump(kwargs, open(util.crash_dir()+'simz_crash_kwargs.p', 'wb')) raise ValueError('Simulator is giving NonetType') return sim def multivariate_rho(model, datum): dists = [] if observables == ['nbar','xi']: nbar_Cii = Cii_list[0] xi_Cii = Cii_list[1] dist_nbar = (datum[0] - model[0])**2. / nbar_Cii dist_xi = np.sum((datum[1:] - model[1:])**2. / xi_Cii) dists = [dist_nbar , dist_xi] elif observables == ['nbar','gmf']: nbar_Cii = Cii_list[0] gmf_Cii = Cii_list[1] dist_nbar = (datum[0] - model[0])**2. / nbar_Cii # omitting the first GMF bin in the model ([1:]) dist_gmf = np.sum((datum[1:] - model[1][1:])**2. / gmf_Cii) dists = [dist_nbar , dist_gmf] elif observables == ['xi']: xi_Cii = Cii_list[0] dist_xi = np.sum((datum- model)**2. / xi_Cii) dists = [dist_xi] return np.array(dists) tolerance_file = lambda name: ''.join([util.abc_dir(), "abc_tolerance", '.', name, '.dat']) theta_file = lambda tt, name: ''.join([util.abc_dir(), util.observable_id_flag(observables), '_theta_t', str(tt), '.', name, '.dat']) w_file = lambda tt, name: ''.join([util.abc_dir(), util.observable_id_flag(observables), '_w_t', str(tt), '.', name, '.dat']) dist_file = lambda tt, name: ''.join([util.abc_dir(), util.observable_id_flag(observables), '_dist_t', str(tt), '.', name, '.dat']) def launch(eps_start, init_pool=None): print eps_start eps = abcpmc.ConstEps(T, eps_start) mpi_pool = mpi_util.MpiPool() pools = [] abcpmc_sampler = abcpmc.Sampler( N=N_part, #N_particles Y=fake_obs, #data postfn=simz, #simulator dist=multivariate_rho, #distance function pool=mpi_pool) abcpmc_sampler.particle_proposal_cls = abcpmc.ParticleProposal f = open(tolerance_file(abcrun), "w") f.close() eps_str = '' for pool in abcpmc_sampler.sample(prior, eps): #while pool.ratio > 0.01: new_eps_str = '\t'.join(np.array(pool.eps).astype('str'))+'\n' if eps_str != new_eps_str: # if eps is different, open fiel and append f = open(tolerance_file(abcrun) , "a") eps_str = new_eps_str f.write(eps_str) f.close() print("T:{0},ratio: {1:>.4f}".format(pool.t, pool.ratio)) print pool.eps # write theta, w, and rhos to file np.savetxt(theta_file(pool.t, abcrun), pool.thetas) np.savetxt(w_file(pool.t, abcrun), pool.ws) np.savetxt(dist_file(pool.t, abcrun) , pool.dists) # plot theta plot_thetas(pool.thetas, pool.ws , pool.t, truths=data_hod, plot_range=prior_range, theta_filename=theta_file(pool.t, abcrun), output_dir=util.abc_dir()) eps.eps = np.median(np.atleast_2d(pool.dists), axis = 0) pools.append(pool) abcpmc_sampler.close() return pools print "Initial launch of the sampler" pools = launch(eps_val)
def overlay_pdfs_contours(abc_filename , mcmc_filename , nwalkers , nburns , Mr): import matplotlib as mpl label_size = 35 mpl.rcParams['xtick.labelsize'] = label_size mpl.rcParams['ytick.labelsize'] = label_size MP_LINEWIDTH = 2.4 MP_TICKSIZE = 10. thick_line1 = mlines.Line2D([], [], ls = '-', c = 'blue', linewidth=4, label='ABC-PMC') thick_line2 = mlines.Line2D([], [], ls = '-', c = 'red', linewidth=4, label=r'Gaussian $\mathcal{L}$ +MCMC') mpl.rc('axes', linewidth=MP_LINEWIDTH) data_hod_dict = Data.data_hod_param(Mr=Mr) truths = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) prior_min, prior_max = PriorRange(None) plot_range = np.zeros((len(prior_min),2)) plot_range[:,0] = prior_min plot_range[:,1] = prior_max mcmc_sample = np.loadtxt(mcmc_filename)[nburns*nwalkers:,:] abc_sample = np.loadtxt(abc_filename) # ##################### scatter plots ########################## # fig, axes = plt.subplots(2, 3, figsize=(48, 13)) # fig.subplots_adjust(wspace=0.4, hspace=0.2) # # ax = axes[0] # ax_list = corner.hist2d(mcmc_sample[:,3],mcmc_sample[:,2],bins =20,levels=[0.68,0.95], ax = ax, plot_datapoints = False , # fill_contours = True, alpha = 10. , color = 'r', smooth = 1. , linewidth=4, range = [plot_range[3,:],plot_range[2,:]]) # ax_list = corner.hist2d(abc_sample[:,3],abc_sample[:,2],bins =20,levels=[0.68,0.95], ax = ax, plot_datapoints = False , # fill_contours = True,alpha = 0.1, color = 'b', smooth = 1. , linewidth=4, range = [plot_range[3,:],plot_range[2,:]]) # ax.plot(truths[3] , truths[2] , marker="*", markersize=25 , color = "yellow") # ax.set_ylabel(r'$\log M_{\rm min}$', fontsize = 50) # ax.set_xlabel(r'$\alpha$', fontsize = 50) # ax.set_xlim([plot_range[3,0] , plot_range[3,1]]) # ax.set_ylim([plot_range[2,0] , plot_range[2,1]]) # # # ax = axes[1] # ax_list = corner.hist2d(mcmc_sample[:,2],mcmc_sample[:,4],bins =20,levels=[0.68,0.95], ax = ax, plot_datapoints = False , # fill_contours = True, color = 'r', smooth = 1. , linewidth=4, range = [plot_range[2,:],plot_range[4,:]]) # ax_list = corner.hist2d(abc_sample[:,2],abc_sample[:,4],bins =20,levels=[0.68,0.95], ax = ax, plot_datapoints = False , # fill_contours = True, color = 'b', smooth = 1. , linewidth=4, range = [plot_range[2,:],plot_range[4,:]]) # ax.plot(truths[2] , truths[4] , marker="*", markersize=25 , color = "yellow") # ax.set_xlabel(r'$\log M_{\rm min}$', fontsize = 50) # ax.set_ylabel(r'$\log M_{1}$', fontsize = 50) # ax.set_xlim([plot_range[2,0] , plot_range[2,1]]) # ax.set_ylim([plot_range[4,0] , plot_range[4,1]]) # # # # ax = axes[2] # ax_list = corner.hist2d(mcmc_sample[:,3],mcmc_sample[:,4],bins =20,levels=[0.68,0.95], ax = ax, plot_datapoints = False , # fill_contours = True, color = 'r', smooth = 1. , linewidth=4, range = [plot_range[3,:],plot_range[4,:]]) # ax_list = corner.hist2d(abc_sample[:,3], abc_sample[:,4], bins =20,levels=[0.68,0.95], ax = ax, plot_datapoints = False , # fill_contours = True, color = 'b', smooth = 1. , linewidth=4, range = [plot_range[3,:],plot_range[4,:]]) # ax.plot(truths[3] , truths[4] , marker="*", markersize=25 , color = "yellow") # ax.set_ylabel(r'$\log M_{1}$', fontsize = 50) # ax.set_xlabel(r'$\alpha$', fontsize = 50) # ax.set_xlim([plot_range[3,0] , plot_range[3,1]]) # ax.set_ylim([plot_range[4,0] , plot_range[4,1]]) # # # plt.legend(handles=[thick_line2, thick_line1], frameon=False, loc='best', fontsize=50) # # plt.savefig("contours_nbarxi2.pdf") ################## HISTOGRAMS######################################### # fig, axes = plt.subplots(1, 3, figsize=(48, 13)) # fig.subplots_adjust(wspace=0.4, hspace=0.2) # # ax = axes[0] # q = ax.hist(mcmc_sample[:,2], bins =20, range = [plot_range[2,:].min(),plot_range[2,:].max()] , normed = True , alpha = 1. , color = 'r', linewidth=4 , histtype='step') # qq = ax.hist(abc_sample[:,2], bins =20, range = [plot_range[2,:].min(),plot_range[2,:].max()] ,normed = True , alpha = 1. , color = 'b', linewidth=4 , histtype='step') # ax.vlines(truths[2], 0, max(q[0].max(),qq[0].max()), color = "k", linewidth = 5) # ax.set_xlabel(r'$\log M_{\rm min}$', fontsize = 50) # ax.set_xlim([plot_range[2,0] , plot_range[2,1]]) # # # ax = axes[1] # q = ax.hist(mcmc_sample[:,3], bins =20, range = [plot_range[3,:].min(),plot_range[3,:].max()] ,normed = True , alpha = 1. , color = 'r', linewidth=4, histtype='step') # qq = ax.hist(abc_sample[:,3], bins =20, range = [plot_range[3,:].min(),plot_range[3,:].max()] ,normed = True , alpha = 1. , color = 'b', linewidth=4, histtype='step') # ax.vlines(truths[3], 0, max(q[0].max(),qq[0].max()), color = "k" , linewidth = 5) # ax.set_xlabel(r'$\alpha$', fontsize = 50) # ax.set_xlim([plot_range[3,0] , plot_range[3,1]]) # # # ax = axes[2] # q = ax.hist(mcmc_sample[:,4], bins =20, range = [plot_range[4,:].min(),plot_range[4,:].max()] ,normed = True , alpha = 1. , color = 'r', linewidth=4, histtype='step') # qq = ax.hist(abc_sample[:,4], bins =20, range = [plot_range[4,:].min(),plot_range[4,:].max()] ,normed = True , alpha = 1. , color = 'b', linewidth=4 , histtype='step') # ax.vlines(truths[4] , 0, max(q[0].max(),qq[0].max()), colors='k' , linewidth = 5) # ax.set_xlabel(r'$\log M_{1}$', fontsize = 50) # ax.set_xlim([plot_range[4,0] , plot_range[4,1]]) # # # plt.legend(handles=[thick_line2, thick_line1], frameon=False, loc='best', fontsize=30) # # plt.savefig("histograms_nbarxi2.pdf") # return None # get the 68% and 95% confidence interval indices for sorted chain normie = norm() sig1lo = normie.cdf(-1) sig1hi = normie.cdf(1) sig2lo = normie.cdf(-2) sig2hi = normie.cdf(2) nsamples_mcmc = len(mcmc_sample[:, 2]) nsamples_abc = len(abc_sample[:, 2]) sig1lo_mcmc = int(sig1lo * nsamples_mcmc) sig2lo_mcmc = int(sig2lo * nsamples_mcmc) sig1hi_mcmc = int(sig1hi * nsamples_mcmc) sig2hi_mcmc = int(sig2hi * nsamples_mcmc) sig1lo_abc = int(sig1lo * nsamples_abc) sig2lo_abc = int(sig2lo * nsamples_abc) sig1hi_abc = int(sig1hi * nsamples_abc) sig2hi_abc = int(sig2hi * nsamples_abc) # choose colours here abclr = 'darkcyan' mcmclr = 'darkorchid' fig = plt.figure(1, figsize=(50, 25)) gs = gridspec.GridSpec(2, 3, height_ratios=[3, 1]) ax = plt.subplot(gs[0]) q = ax.hist(mcmc_sample[:,2], bins=20, range=[plot_range[2,:].min(), plot_range[2,:].max()], normed=True, alpha=1., color=mcmclr, linewidth=4, histtype='step') qq = ax.hist(abc_sample[:,2], bins=20, range=[plot_range[2,:].min(), plot_range[2,:].max()], normed=True, alpha=1., color=abclr, linewidth=4, histtype='step') # ax.vlines(truths[2], 0, max(q[0].max(),qq[0].max()), # color = "k", linewidth = 5) ax.axvline(truths[2], color='k', linewidth=5) ax.set_xticklabels([]) ax.set_xlim([plot_range[2,0] , plot_range[2,1]]) ax = plt.subplot(gs[1]) q = ax.hist(mcmc_sample[:,3], bins=20, range=[plot_range[3,:].min(), plot_range[3,:].max()], normed=True, alpha=1., color=mcmclr, linewidth=4, histtype='step') qq = ax.hist(abc_sample[:,3], bins=20, range=[plot_range[3,:].min(), plot_range[3,:].max()], normed=True, alpha=1., color=abclr, linewidth=4, histtype='step') # ax.vlines(truths[3], 0, max(q[0].max(),qq[0].max()), # color="k", linewidth=5) ax.axvline(truths[3], color='k', linewidth=5) ax.set_xticklabels([]) ax.set_xlim([plot_range[3,0], plot_range[3,1]]) ax = plt.subplot(gs[2]) q = ax.hist(mcmc_sample[:,4], bins=20, range=[plot_range[4,:].min(), plot_range[4,:].max()], normed=True, alpha=1., color=mcmclr, linewidth=4, histtype='step') qq = ax.hist(abc_sample[:,4], bins=20, range=[plot_range[4,:].min(), plot_range[4,:].max()], normed=True, alpha=1., color=abclr, linewidth=4, histtype='step') # ax.vlines(truths[4], 0, max(q[0].max(),qq[0].max()), # colors='k', linewidth=5) ax.axvline(truths[4], color='k', linewidth=5) ax.set_xticklabels([]) ax.set_xlim([plot_range[4,0], plot_range[4,1]]) thick_line1 = mlines.Line2D([], [], ls='-', c=abclr, linewidth=4, label='ABC-PMC') thick_line2 = mlines.Line2D([], [], ls='-', c=mcmclr, linewidth=4, label=r'Gaussian $\mathcal{L}$ +MCMC') ax.legend(handles=[thick_line2, thick_line1], frameon=False, loc='best', fontsize=30) # and now the box plots below # general box properties boxprops = {'color': 'k'} medianprops = {'alpha': 0.} bplots1 = [] ax = plt.subplot(gs[3]) # bplots.append(ax.boxplot(mcmc_sample[:,2], # vert=False, patch_artist=True)) # bplots.append(ax.boxplot(abc_sample[:,2], # vert=False, patch_artist=True)) # stats dict for each box bplots1.append({'med': np.median(mcmc_sample[:, 2]), 'q1': np.sort(mcmc_sample[:, 2])[sig1lo_mcmc], 'q3': np.sort(mcmc_sample[:, 2])[sig1hi_mcmc], 'whislo': np.sort(mcmc_sample[:, 2])[sig2lo_mcmc], 'whishi': np.sort(mcmc_sample[:, 2])[sig2hi_mcmc], 'fliers': []}) bplots1.append({'med': np.median(abc_sample[:, 2]), 'q1': np.sort(abc_sample[:, 2])[sig1lo_abc], 'q3': np.sort(abc_sample[:, 2])[sig1hi_abc], 'whislo': np.sort(abc_sample[:, 2])[sig2lo_abc], 'whishi': np.sort(abc_sample[:, 2])[sig2hi_abc], 'fliers': []}) bxp1 = ax.bxp(bplots1, positions=[1,2], vert=False, patch_artist=True, showfliers=False, boxprops=boxprops, medianprops=medianprops) # boxprops=boxprops, medianprops=medianprops, sym='') for i, box in enumerate(bxp1['boxes']): if i == 0: box.set(facecolor=mcmclr, alpha=0.5) elif i == 1: box.set(facecolor=abclr, alpha=0.5) # for patch in bplot['boxes']: # patch.set_facecolor(abclr) # patch.set_alpha(0.5) # ax.vlines(truths[2], 0, ax.get_ylim(), # max(q[0].max(),qq[0].max()), # color = "k", linewidth = 5) ax.axvline(truths[2], color='k', linewidth=5) ax.set_xlim([plot_range[2,0] , plot_range[2,1]]) ax.set_xlabel(r'$\log M_{\rm min}$', fontsize = 50) ax.set_yticks([1,2]) ax.set_yticklabels(["MCMC", "ABC"]) ax = plt.subplot(gs[4]) # bplots.append(ax.boxplot(mcmc_sample[:,3], # vert=False, patch_artist=True)) # bplots.append(ax.boxplot(abc_sample[:,3], # vert=False, patch_artist=True)) bplots2 = [] bplots2.append({'med': np.median(mcmc_sample[:, 3]), 'q1': np.sort(mcmc_sample[:, 3])[sig1lo_mcmc], 'q3': np.sort(mcmc_sample[:, 3])[sig1hi_mcmc], 'whislo': np.sort(mcmc_sample[:, 3])[sig2lo_mcmc], 'whishi': np.sort(mcmc_sample[:, 3])[sig2hi_mcmc], 'fliers': []}) bplots2.append({'med': np.median(abc_sample[:, 3]), 'q1': np.sort(abc_sample[:, 3])[sig1lo_abc], 'q3': np.sort(abc_sample[:, 3])[sig1hi_abc], 'whislo': np.sort(abc_sample[:, 3])[sig2lo_abc], 'whishi': np.sort(abc_sample[:, 3])[sig2hi_abc], 'fliers': []}) # for i, bplot in enumerate(bplots[2:]): # if i == 0: # for patch in bplot['boxes']: # patch.set_facecolor(mcmclr) # patch.set_alpha(0.5) # elif i == 1: # for patch in bplot['boxes']: # patch.set_facecolor(abclr) # patch.set_alpha(0.5) bxp2 = ax.bxp(bplots2, positions=[1,2], vert=False, patch_artist=True, showfliers=False, boxprops=boxprops, medianprops=medianprops) for i, box in enumerate(bxp2['boxes']): if i == 0: box.set(facecolor=mcmclr, alpha=0.5) elif i == 1: box.set(facecolor=abclr, alpha=0.5) # ax.vlines(truths[3], 0, ax.get_ylim(), # max(q[0].max(),qq[0].max()), # color = "k" , linewidth = 5) ax.axvline(truths[3], color='k', linewidth=5) ax.set_xlim([plot_range[3,0], plot_range[3,1]]) ax.set_xlabel(r'$\alpha$', fontsize = 50) ax.set_yticks([]) ax = plt.subplot(gs[5]) # bplots.append(ax.boxplot(mcmc_sample[:,4], # vert=False, patch_artist=True)) # bplots.append(ax.boxplot(abc_sample[:,4], # vert=False, patch_artist=True)) # for i, bplot in enumerate(bplots[4:]): # if i == 0: # for patch in bplot['boxes']: # patch.set_facecolor(mcmclr) # patch.set_alpha(0.5) # elif i == 1: # for patch in bplot['boxes']: # patch.set_facecolor(abclr) # patch.set_alpha(0.5) bplots3 = [] bplots3.append({'med': np.median(mcmc_sample[:, 4]), 'q1': np.sort(mcmc_sample[:, 4])[sig1lo_mcmc], 'q3': np.sort(mcmc_sample[:, 4])[sig1hi_mcmc], 'whislo': np.sort(mcmc_sample[:, 4])[sig2lo_mcmc], 'whishi': np.sort(mcmc_sample[:, 4])[sig2hi_mcmc], 'fliers': []}) bplots3.append({'med': np.median(abc_sample[:, 4]), 'q1': np.sort(abc_sample[:, 4])[sig1lo_abc], 'q3': np.sort(abc_sample[:, 4])[sig1hi_abc], 'whislo': np.sort(abc_sample[:, 4])[sig2lo_abc], 'whishi': np.sort(abc_sample[:, 4])[sig2hi_abc], 'fliers': []}) bxp3 = ax.bxp(bplots3, positions=[1,2], vert=False, patch_artist=True, showfliers=False, boxprops=boxprops, medianprops=medianprops) for i, box in enumerate(bxp3['boxes']): if i == 0: box.set(facecolor=mcmclr, alpha=0.5) elif i == 1: box.set(facecolor=abclr, alpha=0.5) # ax.vlines(truths[4] , 0, ax.get_ylim(), # max(q[0].max(),qq[0].max()), # colors='k' , linewidth = 5) ax.axvline(truths[4], color='k', linewidth=5) ax.set_xlim([plot_range[4,0], plot_range[4,1]]) ax.set_xlabel(r'$\log M_{1}$', fontsize = 50) ax.set_yticks([]) fig.subplots_adjust(wspace=0.05, hspace=0.0) plt.savefig("histograms_nbarxi2_boxplot_fix.pdf") return None