def getObvs(observables, **data_dict): ''' Given the list of observable strings return data vector and covariance matrices ''' if observables == ['xi']: fake_obs = Data.data_xi(**data_dict) fake_obs_cov = Data.data_cov(inference='abc', **data_dict)[1:16 , 1:16] xi_Cii = np.diag(fake_obs_cov) Cii_list = [xi_Cii] elif observables == ['nbar','xi']: fake_obs = np.hstack([Data.data_nbar(**data_dict), Data.data_xi(**data_dict)]) fake_obs_cov = Data.data_cov(inference='abc', **data_dict)[:16 , :16] Cii = np.diag(fake_obs_cov) xi_Cii = Cii[1:] nbar_Cii = Cii[0] Cii_list = [nbar_Cii, xi_Cii] elif observables == ['nbar','gmf']: ##### FIRST BIN OF GMF DROPPED ############### # CAUTION: hardcoded fake_obs = np.hstack([Data.data_nbar(**data_dict), Data.data_gmf(**data_dict)[1:]]) # CAUTION: Covariance matrix starts at 17 instead fake_obs_cov = Data.data_cov(inference='abc', **data_dict) Cii = np.diag(fake_obs_cov) gmf_Cii = Cii[17:] nbar_Cii = Cii[0] Cii_list = [nbar_Cii, gmf_Cii] return fake_obs, Cii_list
def abc_hodsim(Mr=21, b_normal=0.25): prior_min, prior_max = PriorRange('first_try') prior_range = np.zeros((len(prior_min),2)) prior_range[:,0] = prior_min prior_range[:,1] = prior_max abc_hod = ABC_HODsim() data_hod_dict = Data.data_hod_param(Mr=21) data_hod = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) print Data.data_nbar(Mr=Mr, b_normal=b_normal), Data.data_gmf(Mr=Mr, b_normal=b_normal) print abc_hod(data_hod, prior_range=prior_range, observables=['nbar', 'gmf'])
def abc_hodsim(Mr=21, b_normal=0.25): prior_min, prior_max = PriorRange('first_try') prior_range = np.zeros((len(prior_min), 2)) prior_range[:, 0] = prior_min prior_range[:, 1] = prior_max abc_hod = ABC_HODsim() data_hod_dict = Data.data_hod_param(Mr=21) data_hod = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) print Data.data_nbar(Mr=Mr, b_normal=b_normal), Data.data_gmf(Mr=Mr, b_normal=b_normal) print abc_hod(data_hod, prior_range=prior_range, observables=['nbar', 'gmf'])
def test_nbar(Mr=21, b_normal=0.25): print Data.data_nbar(Mr=Mr, b_normal=b_normal)
def mcmc_mpi(Nwalkers, Nchains, observables=['nbar', 'xi'], data_dict={ 'Mr': 21, 'b_normal': 0.25 }, prior_name='first_try', mcmcrun=None): ''' Standard MCMC implementaion Parameters ----------- - Nwalker : Number of walkers - Nchains : Number of MCMC chains - observables : list of observables. Options are: ['nbar','xi'],['nbar','gmf'],['xi'] - data_dict : dictionary that specifies the observation keywords ''' #Initializing the vector of observables and inverse covariance matrix if observables == ['xi']: fake_obs = Data.data_xi(**data_dict) #fake_obs_icov = Data.data_inv_cov('xi', **data_dict) fake_obs_icov = Data.data_cov(inference='mcmc', **data_dict)[1:16, 1:16] if observables == ['nbar', 'xi']: fake_obs = np.hstack( [Data.data_nbar(**data_dict), Data.data_xi(**data_dict)]) fake_obs_icov = Data.data_cov(inference='mcmc', **data_dict)[:16, :16] if observables == ['nbar', 'gmf']: ##### FIRST BIN OF GMF DROPPED ############### # CAUTION: hardcoded fake_obs = np.hstack( [Data.data_nbar(**data_dict), Data.data_gmf(**data_dict)[1:]]) fake_obs_icov = np.zeros((10, 10)) #print Data.data_cov(**data_dict)[17: , 17:].shape # Covariance matrix being adjusted accordingly fake_obs_icov[1:, 1:] = Data.data_cov(inference='mcmc', **data_dict)[17:, 17:] fake_obs_icov[0, 1:] = Data.data_cov(inference='mcmc', **data_dict)[0, 17:] fake_obs_icov[1:, 0] = Data.data_cov(inference='mcmc', **data_dict)[17:, 0] fake_obs_icov[0, 0] = Data.data_cov(inference='mcmc', **data_dict)[0, 0] # True HOD parameters data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr']) data_hod = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) Ndim = len(data_hod) # Priors prior_min, prior_max = PriorRange(prior_name) prior_range = np.zeros((len(prior_min), 2)) prior_range[:, 0] = prior_min prior_range[:, 1] = prior_max # mcmc chain output file chain_file = ''.join([ util.mcmc_dir(), util.observable_id_flag(observables), '.', mcmcrun, '.mcmc_chain.dat' ]) #print chain_file if os.path.isfile(chain_file) and continue_chain: print 'Continuing previous MCMC chain!' sample = np.loadtxt(chain_file) Nchain = Niter - (len(sample) / Nwalkers ) # Number of chains left to finish if Nchain > 0: pass else: raise ValueError print Nchain, ' iterations left to finish' # Initializing Walkers from the end of the chain pos0 = sample[-Nwalkers:] else: # new chain f = open(chain_file, 'w') f.close() Nchain = Niter # Initializing Walkers random_guess = data_hod pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + \ 5.e-2 * np.random.randn(Ndim * Nwalkers).reshape(Nwalkers, Ndim) #print pos0.shape # Initializing MPIPool pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) # Initializing the emcee sampler hod_kwargs = { 'prior_range': prior_range, 'data': fake_obs, 'data_icov': fake_obs_icov, 'observables': observables, 'Mr': data_dict['Mr'] } sampler = emcee.EnsembleSampler(Nwalkers, Ndim, lnPost, pool=pool, kwargs=hod_kwargs) # Initializing Walkers for result in sampler.sample(pos0, iterations=Nchain, storechain=False): position = result[0] #print position f = open(chain_file, 'a') for k in range(position.shape[0]): output_str = '\t'.join(position[k].astype('str')) + '\n' f.write(output_str) f.close() pool.close()
def mcmc_ipython_par(Nwalkers, Nchains_burn, Nchains_pro, observables=['nbar', 'xi'], data_dict={'Mr':20, 'Nmock':500}, prior_name = 'first_try', threads=1): ''' Standard MCMC implementaion Parameters ----------- - Nwalker : Number of walkers - Nchains_burn : Number of burn-in chains - Nchains_pro : Number of production chains - observables : list of observables. Options are 'nbar', 'gmf', 'xi' - data_dict : dictionary that specifies the observation keywords ''' # data observables fake_obs = [] # list of observables fake_obs_cov = [] for obv in observables: if obv == 'nbar': data_nbar, data_nbar_var = Data.data_nbar(**data_dict) fake_obs.append(data_nbar) fake_obs_cov.append(data_nbar_var) if obv == 'gmf': data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict) fake_obs.append(data_gmf) fake_obs_cov.append(data_gmf) if obv == 'xi': # import xir and full covariance matrix of xir data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict) data_xi_invcov = Data.data_xi_inv_cov(**data_dict) fake_obs.append(data_xi) fake_obs_cov.append(data_xi_invcov) # True HOD parameters data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr']) data_hod = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) Ndim = len(data_hod) # Priors prior_min, prior_max = PriorRange(prior_name) prior_range = np.zeros((len(prior_min),2)) prior_range[:,0] = prior_min prior_range[:,1] = prior_max # Initializing Walkers random_guess = np.array([11. , np.log(.4) , 11.5 , 1.0 , 13.5]) pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + \ 1e-1 * np.random.randn(Ndim * Nwalkers).reshape(Nwalkers, Ndim) # Initializing the emcee sampler hod_kwargs = { 'prior_range': prior_range, 'data': fake_obs, 'data_cov': fake_obs_cov, 'observables': observables, 'Mr': data_dict['Mr'] } # Set up the interface to the ipcluster. c = Client() view = c[:] view.push({"lnPost": lnPost}) # Modules necessary in posterior calculation should be called here view.execute("import numpy as np") view.execute("from hod_sim import HODsimulator") # Setting up the Sampler sampler = emcee.EnsembleSampler(Nwalkers, Ndim, lnPost, kwargs=hod_kwargs, pool = view) # Setting up a file for saving the chains chain_file = ''.join([util.dat_dir(), util.observable_id_flag(observables), '_Mr', str(data_dict["Mr"]), '_theta.mcmc_chain.dat']) f = open(chain_file, "w") f.close() # Running the Sampler and writing out the chains for result in sampler.sample(pos0, iterations=Nchains_burn + Nchains_pro, storechain=False): position = result[0] f = open(chain_file, "a") for k in range(position.shape[0]): output_str = '\t'.join(position[k].astype('str')) + '\n' f.write(output_str) f.close()
def mcmc_mpi( Nwalkers, Nchains, observables=["nbar", "xi"], data_dict={"Mr": 21, "b_normal": 0.25}, prior_name="first_try", mcmcrun=None, ): """ Standard MCMC implementaion Parameters ----------- - Nwalker : Number of walkers - Nchains : Number of MCMC chains - observables : list of observables. Options are: ['nbar','xi'],['nbar','gmf'],['xi'] - data_dict : dictionary that specifies the observation keywords """ # Initializing the vector of observables and inverse covariance matrix if observables == ["xi"]: fake_obs = Data.data_xi(**data_dict) # fake_obs_icov = Data.data_inv_cov('xi', **data_dict) fake_obs_icov = Data.data_cov(inference="mcmc", **data_dict)[1:16, 1:16] if observables == ["nbar", "xi"]: fake_obs = np.hstack([Data.data_nbar(**data_dict), Data.data_xi(**data_dict)]) fake_obs_icov = Data.data_cov(inference="mcmc", **data_dict)[:16, :16] if observables == ["nbar", "gmf"]: ##### FIRST BIN OF GMF DROPPED ############### # CAUTION: hardcoded fake_obs = np.hstack([Data.data_nbar(**data_dict), Data.data_gmf(**data_dict)[1:]]) fake_obs_icov = np.zeros((10, 10)) # print Data.data_cov(**data_dict)[17: , 17:].shape # Covariance matrix being adjusted accordingly fake_obs_icov[1:, 1:] = Data.data_cov(inference="mcmc", **data_dict)[17:, 17:] fake_obs_icov[0, 1:] = Data.data_cov(inference="mcmc", **data_dict)[0, 17:] fake_obs_icov[1:, 0] = Data.data_cov(inference="mcmc", **data_dict)[17:, 0] fake_obs_icov[0, 0] = Data.data_cov(inference="mcmc", **data_dict)[0, 0] # True HOD parameters data_hod_dict = Data.data_hod_param(Mr=data_dict["Mr"]) data_hod = np.array( [ data_hod_dict["logM0"], # log M0 np.log(data_hod_dict["sigma_logM"]), # log(sigma) data_hod_dict["logMmin"], # log Mmin data_hod_dict["alpha"], # alpha data_hod_dict["logM1"], # log M1 ] ) Ndim = len(data_hod) # Priors prior_min, prior_max = PriorRange(prior_name) prior_range = np.zeros((len(prior_min), 2)) prior_range[:, 0] = prior_min prior_range[:, 1] = prior_max # mcmc chain output file chain_file = "".join([util.mcmc_dir(), util.observable_id_flag(observables), ".", mcmcrun, ".mcmc_chain.dat"]) # print chain_file if os.path.isfile(chain_file) and continue_chain: print "Continuing previous MCMC chain!" sample = np.loadtxt(chain_file) Nchain = Niter - (len(sample) / Nwalkers) # Number of chains left to finish if Nchain > 0: pass else: raise ValueError print Nchain, " iterations left to finish" # Initializing Walkers from the end of the chain pos0 = sample[-Nwalkers:] else: # new chain f = open(chain_file, "w") f.close() Nchain = Niter # Initializing Walkers random_guess = data_hod pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + 5.0e-2 * np.random.randn( Ndim * Nwalkers ).reshape(Nwalkers, Ndim) # print pos0.shape # Initializing MPIPool pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) # Initializing the emcee sampler hod_kwargs = { "prior_range": prior_range, "data": fake_obs, "data_icov": fake_obs_icov, "observables": observables, "Mr": data_dict["Mr"], } sampler = emcee.EnsembleSampler(Nwalkers, Ndim, lnPost, pool=pool, kwargs=hod_kwargs) # Initializing Walkers for result in sampler.sample(pos0, iterations=Nchain, storechain=False): position = result[0] # print position f = open(chain_file, "a") for k in range(position.shape[0]): output_str = "\t".join(position[k].astype("str")) + "\n" f.write(output_str) f.close() pool.close()
def mcmc_ipython_par(Nwalkers, Nchains_burn, Nchains_pro, observables=['nbar', 'xi'], data_dict={ 'Mr': 20, 'Nmock': 500 }, prior_name='first_try', threads=1): ''' Standard MCMC implementaion Parameters ----------- - Nwalker : Number of walkers - Nchains_burn : Number of burn-in chains - Nchains_pro : Number of production chains - observables : list of observables. Options are 'nbar', 'gmf', 'xi' - data_dict : dictionary that specifies the observation keywords ''' # data observables fake_obs = [] # list of observables fake_obs_cov = [] for obv in observables: if obv == 'nbar': data_nbar, data_nbar_var = Data.data_nbar(**data_dict) fake_obs.append(data_nbar) fake_obs_cov.append(data_nbar_var) if obv == 'gmf': data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict) fake_obs.append(data_gmf) fake_obs_cov.append(data_gmf) if obv == 'xi': # import xir and full covariance matrix of xir data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict) data_xi_invcov = Data.data_xi_inv_cov(**data_dict) fake_obs.append(data_xi) fake_obs_cov.append(data_xi_invcov) # True HOD parameters data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr']) data_hod = np.array([ data_hod_dict['logM0'], # log M0 np.log(data_hod_dict['sigma_logM']), # log(sigma) data_hod_dict['logMmin'], # log Mmin data_hod_dict['alpha'], # alpha data_hod_dict['logM1'] # log M1 ]) Ndim = len(data_hod) # Priors prior_min, prior_max = PriorRange(prior_name) prior_range = np.zeros((len(prior_min), 2)) prior_range[:, 0] = prior_min prior_range[:, 1] = prior_max # Initializing Walkers random_guess = np.array([11., np.log(.4), 11.5, 1.0, 13.5]) pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + \ 1e-1 * np.random.randn(Ndim * Nwalkers).reshape(Nwalkers, Ndim) # Initializing the emcee sampler hod_kwargs = { 'prior_range': prior_range, 'data': fake_obs, 'data_cov': fake_obs_cov, 'observables': observables, 'Mr': data_dict['Mr'] } # Set up the interface to the ipcluster. c = Client() view = c[:] view.push({"lnPost": lnPost}) # Modules necessary in posterior calculation should be called here view.execute("import numpy as np") view.execute("from hod_sim import HODsimulator") # Setting up the Sampler sampler = emcee.EnsembleSampler(Nwalkers, Ndim, lnPost, kwargs=hod_kwargs, pool=view) # Setting up a file for saving the chains chain_file = ''.join([ util.dat_dir(), util.observable_id_flag(observables), '_Mr', str(data_dict["Mr"]), '_theta.mcmc_chain.dat' ]) f = open(chain_file, "w") f.close() # Running the Sampler and writing out the chains for result in sampler.sample(pos0, iterations=Nchains_burn + Nchains_pro, storechain=False): position = result[0] f = open(chain_file, "a") for k in range(position.shape[0]): output_str = '\t'.join(position[k].astype('str')) + '\n' f.write(output_str) f.close()
# --- Local --- import data as Data from hod_sim import HODsim from group_richness import richness Nwalkers = 10 Nchains_burn = 1 Nchains_pro = 1 # data observables fake_obs = [] # list of observables for obv in observables: if obv == 'nbar': data_nbar, data_nbar_var = Data.data_nbar(**data_dict) fake_obs.append(data_nbar) if obv == 'gmf': data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict) fake_obs.append(data_gmf) if obv == 'xi': # import xir and full covariance matrix of xir data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict) # take the inverse of the covariance matrix data_xi_invcov = solve(np.eye(len(data_xi)) , data_xi_cov) fake_obs.append(data_xi) # 'True' HOD parameters data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr']) data_hod = np.array([ data_hod_dict['logM0'], # log M0
from emcee.utils import MPIPool # --- Local --- import data as Data from hod_sim import HODsim from group_richness import richness Nwalkers = 10 Nchains_burn = 1 Nchains_pro = 1 # data observables fake_obs = [] # list of observables for obv in observables: if obv == 'nbar': data_nbar, data_nbar_var = Data.data_nbar(**data_dict) fake_obs.append(data_nbar) if obv == 'gmf': data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict) fake_obs.append(data_gmf) if obv == 'xi': # import xir and full covariance matrix of xir data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict) # take the inverse of the covariance matrix data_xi_invcov = solve(np.eye(len(data_xi)), data_xi_cov) fake_obs.append(data_xi) # 'True' HOD parameters data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr']) data_hod = np.array([ data_hod_dict['logM0'], # log M0