def launch(eps_start, init_pool=None): print eps_start eps = abcpmc.ConstEps(T, eps_start) mpi_pool = mpi_util.MpiPool() pools = [] abcpmc_sampler = abcpmc.Sampler( N=N_part, #N_particles Y=fake_obs, #data postfn=simz, #simulator dist=multivariate_rho, #distance function pool=mpi_pool) abcpmc_sampler.particle_proposal_cls = abcpmc.ParticleProposal f = open(tolerance_file(abcrun), "w") f.close() eps_str = '' for pool in abcpmc_sampler.sample(prior, eps): #while pool.ratio > 0.01: new_eps_str = '\t'.join(np.array(pool.eps).astype('str'))+'\n' if eps_str != new_eps_str: # if eps is different, open fiel and append f = open(tolerance_file(abcrun) , "a") eps_str = new_eps_str f.write(eps_str) f.close() print("T:{0},ratio: {1:>.4f}".format(pool.t, pool.ratio)) print pool.eps # write theta, w, and rhos to file np.savetxt(theta_file(pool.t, abcrun), pool.thetas) np.savetxt(w_file(pool.t, abcrun), pool.ws) np.savetxt(dist_file(pool.t, abcrun) , pool.dists) # plot theta plot_thetas(pool.thetas, pool.ws , pool.t, truths=data_hod, plot_range=prior_range, theta_filename=theta_file(pool.t, abcrun), output_dir=util.abc_dir()) eps.eps = np.median(np.atleast_2d(pool.dists), axis = 0) pools.append(pool) abcpmc_sampler.close() return pools
def ABCpmc_HOD(T, eps_val, N_part=1000, prior_name='first_try', observables=['nbar', 'xi'], data_dict={'Mr': 21}, output_dir=None): ''' ABC-PMC implementation. Parameters ---------- - T : Number of iterations - eps_val : - N_part : Number of particles - observables : list of observables. Options are 'nbar', 'gmf', 'xi' - data_dict : dictionary that specifies the observation keywords ''' if output_dir is None: output_dir = util.dat_dir() else: pass #Initializing the vector of observables and inverse covariance matrix if observables == ['xi']: fake_obs = Data.data_xi(**data_dict) fake_obs_cov = Data.data_cov(**data_dict)[1:16, 1:16] xi_Cii = np.diag(fake_obs_cov) elif observables == ['nbar', 'xi']: fake_obs = np.hstack( [Data.data_nbar(**data_dict), Data.data_xi(**data_dict)]) fake_obs_cov = Data.data_cov(**data_dict)[:16, :16] Cii = np.diag(fake_obs_cov) xi_Cii = Cii[1:] nbar_Cii = Cii[0] elif observables == ['nbar', 'gmf']: fake_obs = np.hstack( [Data.data_nbar(**data_dict), Data.data_gmf(**data_dict)]) fake_obs_cov = Data.data_cov('nbar_gmf', **data_dict) Cii = np.diag(fake_obs_cov) gmf_Cii = Cii[1:] nbar_Cii = Cii[0] # True HOD parameters data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr']) data_hod = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) # Priors prior_min, prior_max = PriorRange(prior_name) prior = abcpmc.TophatPrior(prior_min, prior_max) prior_range = np.zeros((len(prior_min), 2)) prior_range[:, 0] = prior_min prior_range[:, 1] = prior_max # simulator our_model = HODsim(Mr=data_dict['Mr']) # initialize model kwargs = {'prior_range': prior_range, 'observables': observables} def simz(tt): sim = our_model.sum_stat(tt, **kwargs) if sim is None: pickle.dump(tt, open("simz_crash_theta.p", 'wb')) pickle.dump(kwargs, open('simz_crash_kwargs.p', 'wb')) raise ValueError('Simulator is giving NonetType') return sim def multivariate_rho(datum, model): #print datum , model dists = [] if observables == ['nbar', 'xi']: dist_nbar = (datum[0] - model[0])**2. / nbar_Cii dist_xi = np.sum((datum[1:] - model[1:])**2. / xi_Cii) dists = [dist_nbar, dist_xi] elif observables == ['nbar', 'gmf']: dist_nbar = (datum[0] - model[0])**2. / nbar_Cii dist_gmf = np.sum((datum[1:] - model[1:])**2. / gmf_Cii) dists = [dist_nbar, dist_gmf] elif observables == ['xi']: dist_xi = np.sum((datum - model)**2. / xi_Cii) dists = [dist_xi] #print np.array(dists) return np.array(dists) mpi_pool = mpi_util.MpiPool() abcpmc_sampler = abcpmc.Sampler( N=N_part, #N_particles Y=fake_obs, #data postfn=simz, #simulator dist=multivariate_rho, #distance function pool=mpi_pool) abcpmc_sampler.particle_proposal_cls = abcpmc.ParticleProposal eps = abcpmc.MultiConstEps(T, eps_val) pools = [] f = open("abc_tolerance.dat", "w") f.close() eps_str = '' for pool in abcpmc_sampler.sample(prior, eps): #while pool.ratio > 0.01: new_eps_str = '\t'.join(eps(pool.t).astype('str')) + '\n' if eps_str != new_eps_str: # if eps is different, open fiel and append f = open("abc_tolerance.dat", "a") eps_str = new_eps_str f.write(eps_str) f.close() print("T:{0},ratio: {1:>.4f}".format(pool.t, pool.ratio)) print eps(pool.t) # plot theta plot_thetas(pool.thetas, pool.ws, pool.t, Mr=data_dict["Mr"], truths=data_hod, plot_range=prior_range, observables=observables, output_dir=output_dir) if (pool.t < 4) and (pool.t > 2): pool.thetas = np.loadtxt( "/home/mj/abc/halo/dat/gold/nbar_xi_Mr21_theta_t3.mercer.dat") pool.ws = np.loadtxt( "/home/mj/abc/halo/dat/gold/nbar_xi_Mr21_w_t3.mercer.dat") eps.eps = [1.12132735353, 127.215586776] # write theta and w to file theta_file = ''.join([ output_dir, util.observable_id_flag(observables), '_Mr', str(data_dict["Mr"]), '_theta_t', str(pool.t), '.mercer.dat' ]) w_file = ''.join([ output_dir, util.observable_id_flag(observables), '_Mr', str(data_dict["Mr"]), '_w_t', str(pool.t), '.mercer.dat' ]) np.savetxt(theta_file, pool.thetas) np.savetxt(w_file, pool.ws) if pool.t < 3: eps.eps = np.percentile(np.atleast_2d(pool.dists), 50, axis=0) elif (pool.t > 2) and (pool.t < 20): eps.eps = np.percentile(np.atleast_2d(pool.dists), 75, axis=0) abcpmc_sampler.particle_proposal_cls = abcpmc.ParticleProposal else: eps.eps = np.percentile(np.atleast_2d(pool.dists), 90, axis=0) abcpmc_sampler.particle_proposal_cls = abcpmc.ParticleProposal #if eps.eps < eps_min: # eps.eps = eps_min pools.append(pool) #abcpmc_sampler.close() return pools
def ABCpmc_HOD(T, eps_val, N_part=1000, prior_name='first_try', observables=['nbar', 'xi'], data_dict={'Mr':21}, output_dir=None): ''' ABC-PMC implementation. Parameters ---------- - T : Number of iterations - eps_val : - N_part : Number of particles - observables : list of observables. Options are 'nbar', 'gmf', 'xi' - data_dict : dictionary that specifies the observation keywords ''' if output_dir is None: output_dir = util.dat_dir() else: pass #Initializing the vector of observables and inverse covariance matrix if observables == ['xi']: fake_obs = Data.data_xi(**data_dict) fake_obs_cov = Data.data_cov(**data_dict)[1:16 , 1:16] xi_Cii = np.diag(fake_obs_cov) elif observables == ['nbar','xi']: fake_obs = np.hstack([Data.data_nbar(**data_dict), Data.data_xi(**data_dict)]) fake_obs_cov = Data.data_cov(**data_dict)[:16 , :16] Cii = np.diag(fake_obs_cov) xi_Cii = Cii[1:] nbar_Cii = Cii[0] elif observables == ['nbar','gmf']: fake_obs = np.hstack([Data.data_nbar(**data_dict), Data.data_gmf(**data_dict)]) fake_obs_cov = Data.data_cov('nbar_gmf', **data_dict) Cii = np.diag(fake_obs_cov) gmf_Cii = Cii[1:] nbar_Cii = Cii[0] # True HOD parameters data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr']) data_hod = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) # Priors prior_min, prior_max = PriorRange(prior_name) prior = abcpmc.TophatPrior(prior_min, prior_max) prior_range = np.zeros((len(prior_min),2)) prior_range[:,0] = prior_min prior_range[:,1] = prior_max # simulator our_model = HODsim(Mr=data_dict['Mr']) # initialize model kwargs = {'prior_range': prior_range, 'observables': observables} def simz(tt): sim = our_model.sum_stat(tt, **kwargs) if sim is None: pickle.dump(tt, open("simz_crash_theta.p", 'wb')) pickle.dump(kwargs, open('simz_crash_kwargs.p', 'wb')) raise ValueError('Simulator is giving NonetType') return sim def multivariate_rho(datum, model): dists = [] if observables == ['nbar','xi']: dist_nbar = (datum[0] - model[0])**2. / nbar_Cii dist_xi = np.sum((datum[1:] - model[1:])**2. / xi_Cii) dists = [dist_nbar , dist_xi] elif observables == ['nbar','gmf']: dist_nbar = (datum[0] - model[0])**2. / nbar_Cii dist_gmf = np.sum((datum[1:] - model[1:])**2. / gmf_Cii) dists = [dist_nbar , dist_gmf] elif observables == ['xi']: dist_xi = np.sum((datum- model)**2. / xi_Cii) dists = [dist_xi] print np.array(dists) return np.array(dists) mpi_pool = mpi_util.MpiPool() abcpmc_sampler = abcpmc.Sampler( N=N_part, #N_particles Y=fake_obs, #data postfn=simz, #simulator dist=multivariate_rho, #distance function pool=mpi_pool) abcpmc_sampler.particle_proposal_cls = abcpmc.ParticleProposal eps = abcpmc.MultiConstEps(T, eps_val) pools = [] f = open("abc_tolerance.dat" , "w") f.close() eps_str = '' for pool in abcpmc_sampler.sample(prior, eps): #while pool.ratio > 0.01: new_eps_str = '\t'.join(eps(pool.t).astype('str'))+'\n' if eps_str != new_eps_str: # if eps is different, open fiel and append f = open("abc_tolerance.dat" , "a") eps_str = new_eps_str f.write(eps_str) f.close() print("T:{0},ratio: {1:>.4f}".format(pool.t, pool.ratio)) print eps(pool.t) # plot theta plot_thetas(pool.thetas, pool.ws , pool.t, Mr=data_dict["Mr"], truths=data_hod, plot_range=prior_range, observables=observables, output_dir=output_dir) # write theta and w to file theta_file = ''.join([output_dir, util.observable_id_flag(observables), '_Mr', str(data_dict["Mr"]), '_theta_t', str(pool.t), '.mercer.dat']) w_file = ''.join([output_dir, util.observable_id_flag(observables), '_Mr', str(data_dict["Mr"]), '_w_t', str(pool.t), '.mercer.dat']) np.savetxt(theta_file, pool.thetas) np.savetxt(w_file, pool.ws) if pool.t < 3: eps.eps = np.percentile(np.atleast_2d(pool.dists), 50 , axis = 0) elif (pool.t > 2) and (pool.t < 20): eps.eps = np.percentile(np.atleast_2d(pool.dists), 75 , axis = 0) abcpmc_sampler.particle_proposal_cls = abcpmc.ParticleProposal else: eps.eps = np.percentile(np.atleast_2d(pool.dists), 90 , axis = 0) abcpmc_sampler.particle_proposal_cls = abcpmc.ParticleProposal #if eps.eps < eps_min: # eps.eps = eps_min pools.append(pool) #abcpmc_sampler.close() return pools