def fit_sub_mean():
np.random.seed()
for z_id in xrange(3):
norm_gf = growth_factor(float(sim_z[z_id]), Omega_m) / G_0
for space_id in xrange(1):
ifile_Pk = './{}_sub_Pk_2d_wnw_mean_{}_ksorted_mu/{}kave{}.wig_minus_now_mean_sub_a_{}.dat'.format(
sim_run, rec_dirs[rec_id], rec_fprefix[rec_id],
sim_space[space_id], sim_a[z_id])
Pk_wnw_diff_true = np.loadtxt(
ifile_Pk, dtype='f4', comments='#', usecols=(2, )
) # be careful that there are k, \mu, P(k, \mu) columns.
ifile_Cov_Pk = './{}_sub_Cov_Pk_2d_wnw_mean_{}_ksorted_mu/{}kave{}.wig_minus_now_mean_sub_a_{}.dat'.format(
sim_run, rec_dirs[rec_id], rec_fprefix[rec_id],
sim_space[space_id], sim_a[z_id])
Cov_Pk_wnw = np.loadtxt(ifile_Cov_Pk, dtype='f4', comments='#')
ivar_Pk_wnow = N_dataset / np.diag(
Cov_Pk_wnw) # the mean sigma error
params_mcmc = mcmc_routine(N_params, N_walkers, N_walkersteps,
theta, params_T, params_indices,
initial_pvalue, k_p, mu_p,
Pk_wnw_diff_true, ivar_Pk_wnow,
tck_Pk_linw, tck_Pk_sm, norm_gf)
chi_square = chi2(params_mcmc[:, 0], params_indices,
initial_pvalue, k_p, mu_p, Pk_wnw_diff_true,
ivar_Pk_wnow, tck_Pk_linw, tck_Pk_sm, norm_gf)
reduced_chi2 = chi_square / (N_fitbin - N_params)
print('Reduced chi2: ', reduced_chi2)
# output parameters into a file
ofile_params = odir + '{}kave{}.wig-now_b_bscale_mean_sub_a_{}_params{}_isotropic.dat'.format(
rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], ''.join(
map(str, initial_pvalue)))
write_params(ofile_params, params_mcmc, params_name, reduced_chi2)
def Pk_wnw_diff_divideby_Psm():
rec_id = 1 # rec_id=0: before reconstruction; rec_id=1: after reconstruction
odir_prefix = './run2_3_'
for z_id in xrange(3):
for space_id in xrange(1):
# Pk_mnow_obs = np.array([], dtype=np.float64).reshape(0, n_fitbin) # m means matrix
# Pk_mnow_true = np.array([], dtype=np.float64).reshape(0, n_fitbin)
# Pk_mwig_obs = np.array([], dtype=np.float64).reshape(0, n_fitbin)
# Pk_mwig_true = np.array([], dtype=np.float64).reshape(0, n_fitbin)
Pk_mwnw_diff_obs = np.array([], dtype=np.float64).reshape(0, n_fitbin)
Pk_mwnw_diff_true = np.array([], dtype=np.float64).reshape(0, n_fitbin)
for run_id in xrange(2):
for sim_seed_id in xrange(10):
k_i, Pk_now_obs, Pk_now_true = read_fftPk_file(dir0, z_id, rec_id, space_id, sim_wig[run_id][0], sim_seed_id)
# Pk_mnow_obs = np.vstack([Pk_mnow_obs, Pk_now_obs])
# Pk_mnow_true = np.vstack([Pk_mnow_true, Pk_now_true])
k_i, Pk_wig_obs, Pk_wig_true = read_fftPk_file(dir0, z_id, rec_id, space_id, sim_wig[run_id][1], sim_seed_id)
# Pk_mwig_obs = np.vstack([Pk_mwig_obs, Pk_wig_obs])
# Pk_mwig_true = np.vstack([Pk_mwig_true, Pk_wig_true])
Pk_wnw_diff_true = (Pk_wig_true-Pk_now_true)/Pk_sm_obsk
Pk_mwnw_diff_true = np.vstack([Pk_mwnw_diff_true, Pk_wnw_diff_true])
Pk_wnw_diff_true_mean = np.mean(Pk_mwnw_diff_true, axis=0)
for i in xrange(n_fitbin):
Pk_mwnw_diff_true[:, i] = Pk_mwnw_diff_true[:, i] - Pk_wnw_diff_true_mean[i]
Cov_Pk = np.dot(Pk_mwnw_diff_true.T, Pk_mwnw_diff_true)/(N_dataset -1.0)
#print(Cov_Pk, Cov_Pk.shape)
# add the const factor with growth function and bias (b=1.0 for dark matter P(k))
bias = 1.0
z = float(sim_z[z_id])
print("z=", z, "G: ", growth_factor(z, Omega_m)/G_0)
G2b2 = (growth_factor(z, Omega_m)/G_0*bias)**2.0
Pk_wnw_diff_true_mean = Pk_wnw_diff_true_mean/G2b2
Cov_Pk = Cov_Pk/G2b2**2.0
var_name = 'sub_Pk_2d_wnw_mean_{}_ksorted_mu/'.format(rec_dirs[rec_id])
filename = "{}kave{}.wnw_diff_mean_a_{}ga.dat".format(rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id])
header_line = ' After sorting k, the mean (P(k)_wig-P(k)_now)/(G^2b^2) in 2d (k, mu) case\n k mu Pk_wnw_diff_true_mean'
write_output(odir_prefix, var_name, header_line, rec_id, space_id, z_id, np.array([k_p, mu_p, Pk_wnw_diff_true_mean]).T, filename)
var_name = 'sub_Cov_Pk_2d_wnw_mean_{}_ksorted_mu/'.format(rec_dirs[rec_id])
header_line = ' Cov(P_2d_wnw_diff(k1), P_2d_wnw_diff(k2)), (wnw_diff means wig-now, 2d: k, mu; and k1, k2 are the k bin indices).'
write_output(odir_prefix, var_name, header_line, rec_id, space_id, z_id, Cov_Pk, filename)
def fit_fof_mean():
rec_id = 0
#odir = './params_{}_wig-now_b_bscale_fitted_mean_dset/'.format(rec_dirs[rec_id])
odir = './params_{}_wig-now_b_bscale_fitted_mean_dset_broad_prior/'.format(
rec_dirs[rec_id])
if not os.path.exists(odir):
os.makedirs(odir)
for z_id in xrange(3):
norm_gf = growth_factor(float(sim_z[z_id]), Omega_m) / G_0
print(sim_z[z_id], norm_gf)
def fit_fof_mean():
#free_params = ['True', 'True', 'False', 'False', 'True', 'True']
#fix_params = np.array([1., 1., 0., 0., 1., 1.])
for rec_id in xrange(1, 2):
odir = './params_{}_wig-now_b_bscale_fitted_mean_dset/'.format(
rec_dirs[rec_id])
if not os.path.exists(odir):
os.makedirs(odir)
for z_id in xrange(3):
normalized_growth_factor = growth_factor(float(sim_z[z_id]),
Omega_m) / G_0
for mcut_id in xrange(N_masscut):
np.random.seed()
for space_id in xrange(1):
ifile_Pk = './{}_Pk_obs_2d_wnw_mean_{}_ksorted_mu_masscut/{}kave{}.wig_minus_now_mean_fof_a_{}_mcut{}.dat'.format(
sim_run, rec_dirs[rec_id], rec_fprefix[rec_id],
sim_space[space_id], sim_a[z_id],
mcut_Npar_list[z_id][mcut_id])
Pk_wnw_diff_obs = np.loadtxt(
ifile_Pk, dtype='f4', comments='#', usecols=(2, )
) # be careful that there are k, \mu, P(k, \mu) columns.
ifile_Cov_Pk = './{}_Cov_Pk_obs_2d_wnw_mean_{}_ksorted_mu_masscut/{}kave{}.wig_minus_now_mean_fof_a_{}_mcut{}.dat'.format(
sim_run, rec_dirs[rec_id], rec_fprefix[rec_id],
sim_space[space_id], sim_a[z_id],
mcut_Npar_list[z_id][mcut_id])
Cov_Pk_wnw = np.loadtxt(ifile_Cov_Pk,
dtype='f4',
comments='#')
ivar_Pk_wnow = N_dataset / np.diag(
Cov_Pk_wnw) # the mean sigma error
alpha_1, alpha_2, b_0, b_scale = mcmc_routine(
N_params, N_walkers, N_walkersteps, k_p, mu_p,
Pk_wnw_diff_obs, ivar_Pk_wnow, tck_Pk_linw, tck_Pk_sm,
normalized_growth_factor)
chi_square = chi2(
[alpha_1[0], alpha_2[0], b_0[0], b_scale[0]], k_p,
mu_p, Pk_wnw_diff_obs, tck_Pk_linw, tck_Pk_sm,
normalized_growth_factor, ivar_Pk_wnow)
reduced_chi2 = chi_square / (N_fitbin - N_params)
print('Reduced chi2: ', reduced_chi2)
# output parameters into a file
ofile_params = odir + '{}kave{}.wig-now_b_bscale_mean_fof_a_{}_mcut{}_params.dat'.format(
rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id],
mcut_Npar_list[z_id][mcut_id])
write_params(ofile_params, alpha_1, alpha_2, b_0, b_scale,
reduced_chi2)
def plot_Sigma_z(sim_z, sub_Sigma, Sigma_Plinw, odir, figname):
G_index = 1.
z_axis = np.linspace(0.0, 1.2, 200)
fit_Sigma = np.array([(growth_factor(z, Omega_m)/G_0)**G_index for z in z_axis])*sub_Sigma[0]/G_z[0]**G_index
# We don't show error from the integration, because it's not one sigma error.
plt.plot(0.0, Sigma_Plinw, 'rs', label=r'$\Sigma(0)$ from $P_{\mathtt{lin,w}}$')
plt.plot(map(float, sim_z[:]), sub_Sigma, 'ks', label=r'$\Sigma(z)$ from $\hat{P}_{\mathtt{sim,w}}$')
plt.plot(z_axis, fit_Sigma, 'g--', label=r'$\Sigma_0 G(z)$')
plt.legend(frameon=False, loc="upper right", fontsize=20)
plt.title(r"$\Sigma$ calculated {}".format(title_space[0]), fontsize=20)
plt.xlim([-0.05, 1.2])
plt.xlabel(r"$z$", fontsize=24)
plt.ylim([4.0, 9.0])
plt.ylabel(r"$\Sigma$ $[Mpc/h]$", fontsize=24)
plt.savefig(odir+figname)
plt.show()
plt.close()
def fit_fof_sub_mean():
for z_id in xrange(3):
G_z = growth_factor(float(sim_z[z_id]), Omega_m)/G_0
Sigma_z = Sigma_0 * G_z
# Sigma_z = Sigma_z - 1.0
# for count in xrange(5):
print("Sigma_z: ", Sigma_z)
sigma_xy, sigma_z = Sigma_z, Sigma_z
alpha_1, alpha_2, b_0, b_scale = 1.0, 1.0, 1.0, 0.0
all_params = alpha_1, alpha_2, sigma_xy, sigma_z, b_0, b_scale
all_names = "alpha_1", "alpha_2", "sigma_xy", "sigma_z", "b_0", "b_scale"
all_temperature = 0.01, 0.01, 0.1, 0.1, 0.1, 1.0
params_indices = [0, 0, 0, 0, 1, 1] # 0: parameter fixed, 1: parameter free.
fix_params = np.array([], dtype=np.float)
theta = np.array([], dtype=np.float)
params_T = np.array([], dtype=np.float)
params_name = []
N_params = 0
count = 0
for i in params_indices:
if i == 1:
fix_params = np.append(fix_params, 0.)
theta = np.append(theta, all_params[count])
params_T = np.append(params_T, all_temperature[count])
params_name.append(all_names[count])
N_params += 1
else:
fix_params = np.append(fix_params, all_params[count])
count += 1
print(theta, params_name, N_params)
print("fixed params: ", fix_params)
norm_gf = growth_factor(float(sim_z[z_id]), Omega_m)/G_0
np.random.seed()
for space_id in xrange(1):
for mcut_id in xrange(N_masscut):
ifile_Pk = './{}_Pk_obs_2d_wnw_mean_{}_ksorted_mu_masscut/{}kave{}.wig_minus_now_mean_fof_a_{}_mcut{}.dat'.format(sim_run, rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], mcut_Npar_list[z_id][mcut_id])
Pk_wnw_diff_obs = np.loadtxt(ifile_Pk, dtype='f4', comments='#', usecols=(2,)) # be careful that there are k, \mu, P(k, \mu) columns.
ifile_Cov_Pk = './{}_Cov_Pk_obs_2d_wnw_mean_{}_ksorted_mu_masscut/{}kave{}.wig_minus_now_mean_fof_a_{}_mcut{}.dat'.format(sim_run, rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], mcut_Npar_list[z_id][mcut_id])
Cov_Pk_wnw = np.loadtxt(ifile_Cov_Pk, dtype='f4', comments='#')
ivar_Pk_wnow = N_dataset/np.diag(Cov_Pk_wnw) # the mean sigma error
params_mcmc = mcmc_routine(N_params, N_walkers, N_walkersteps, theta, params_T, params_indices, fix_params, k_p, mu_p, Pk_wnw_diff_obs, ivar_Pk_wnow, tck_Pk_linw, tck_Pk_sm, norm_gf, params_name)
chi_square = chi2(params_mcmc[:, 0], params_indices, fix_params, k_p, mu_p, Pk_wnw_diff_obs, ivar_Pk_wnow, tck_Pk_linw, tck_Pk_sm, norm_gf)
reduced_chi2 = chi_square/(N_fitbin-N_params)
print('Reduced chi2: ', reduced_chi2)
# output parameters into a file
##ofile_params = odir + '{}kave{}.wig-now_b_bscale_mean_fof_a_{}_mcut{}_params{}_isotropic.dat'.format(rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], mcut_Npar_list[z_id][mcut_id], ''.join(map(str, params_indices)))
ofile_params = odir + '{}kave{}.wig-now_b_bscale_mean_fof_a_{}_mcut{}_params{}_isotropic_Sigmaz_{}.dat'.format(rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], mcut_Npar_list[z_id][mcut_id], ''.join(map(str, params_indices)), round(Sigma_z, 3))
write_params(ofile_params, params_mcmc, params_name, reduced_chi2)
# Fit for DM power spectrum
ifile_Pk = './{}_sub_Pk_2d_wnw_mean_{}_ksorted_mu/{}kave{}.wig_minus_now_mean_sub_a_{}.dat'.format(sim_run, rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id])
Pk_wnw_diff_true = np.loadtxt(ifile_Pk, dtype='f4', comments='#', usecols=(2,)) # be careful that there are k, \mu, P(k, \mu) columns.
ifile_Cov_Pk = './{}_sub_Cov_Pk_2d_wnw_mean_{}_ksorted_mu/{}kave{}.wig_minus_now_mean_sub_a_{}.dat'.format(sim_run, rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id])
Cov_Pk_wnw = np.loadtxt(ifile_Cov_Pk, dtype='f4', comments='#')
ivar_Pk_wnow = N_dataset/np.diag(Cov_Pk_wnw) # the mean sigma error
params_mcmc = mcmc_routine(N_params, N_walkers, N_walkersteps, theta, params_T, params_indices, fix_params, k_p, mu_p, Pk_wnw_diff_true, ivar_Pk_wnow, tck_Pk_linw, tck_Pk_sm, norm_gf, params_name)
chi_square = chi2(params_mcmc[:, 0], params_indices, fix_params, k_p, mu_p, Pk_wnw_diff_true, ivar_Pk_wnow, tck_Pk_linw, tck_Pk_sm, norm_gf)
reduced_chi2 = chi_square/(N_fitbin-N_params)
print('Reduced chi2: ', reduced_chi2)
# output parameters into a file
##ofile_params = odir + '{}kave{}.wig-now_b_bscale_mean_sub_a_{}_params{}_isotropic.dat'.format(rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], ''.join(map(str, params_indices)))
ofile_params = odir + '{}kave{}.wig-now_b_bscale_mean_sub_a_{}_params{}_isotropic_Sigmaz_{}.dat'.format(rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], ''.join(map(str, params_indices)), round(Sigma_z, 3))
write_params(ofile_params, params_mcmc, params_name, reduced_chi2)
del sampler
return np.array(theta_mcmc)
#################################################################################################
######################################--------main code---------#################################
#################################################################################################
# simulation run name
sim_run = 'run2_3'
N_dataset = 20
N_mu_bin = 100
#N_skip_header = 11
#N_skip_footer = 31977
Omega_m = 0.3075
G_0 = growth_factor(0.0, Omega_m) # G_0 at z=0, normalization factor
N_walkers = 40
N_walkersteps = 5000
rec_id = 0
##Sigma_0 = 8.3364 # the approximated value of \Sigma_xy and \Sigma_z, unit Mpc/h
Sigma_0 = 7.8364 # ~7.8 Mpc/h, the theoretical value
sim_z=['0', '0.6', '1.0']
sim_seed = [0, 9]
sim_wig = ['NW', 'WG']
sim_a = ['1.0000', '0.6250', '0.5000']
sim_space = ['r', 's'] # r for real space; s for redshift space
rec_dirs = ['DD', 'ALL'] # "ALL" folder stores P(k, \mu) after reconstruction process, while DD is before reconstruction.
rec_fprefix = ['', 'R']
def fit_subsamplefof_mean():
parser = argparse.ArgumentParser(
description=
'This is the MCMC code to get the fitting parameters, made by Zhejie Ding.'
)
parser.add_argument(
'-rec_id',
"--rec_id",
help='The id of reconstruction, either 0 or 1.',
required=True) #0: pre-reconstruct; 1: post-reconstruct
parser.add_argument('-space_id',
"--space_id",
help='0 for real space, 1 for redshift space.',
required=True)
parser.add_argument(
'-set_Sigma_xyz_theory',
"--set_Sigma_xyz_theory",
help=
'Determine whether the parameters \Sigma_xy and \Sigma_z are fixed or not, either True or False',
required=True)
parser.add_argument(
'-set_Sigma_sm_theory',
"--set_Sigma_sm_theory",
help=
'Determine whether we use sigma_sm from theory in the fitting model. \
If False, sigma_sm=0 (be careful that sigma_sm=\inf in real space case)',
required=True)
args = parser.parse_args()
print("args: ", args)
rec_id = int(args.rec_id)
space_id = int(args.space_id)
set_Sigma_xyz_theory = args.set_Sigma_xyz_theory
set_Sigma_sm_theory = args.set_Sigma_sm_theory
print("rec_id: ", rec_id, "space_id: ", space_id)
print("set_Sigma_xyz_theory: ", set_Sigma_xyz_theory,
"set_Sigma_sm_theory: ", set_Sigma_sm_theory)
N_walkers = 40 # increase N_walkers would decrease the minimum number of walk steps which make fitting parameters convergent, but running time increases.
N_walkersteps = 5000
# simulation run name
N_dataset = 20
N_mu_bin = 100
#N_skip_header = 11
#N_skip_footer = 31977
Omega_m = 0.3075
G_0 = growth_factor(0.0, Omega_m) # G_0 at z=0, normalization factor
Volume = 1380.0**3.0 # the volume of simulation box
sim_z = ['0', '0.6', '1.0']
sim_seed = [0, 9]
sim_wig = ['NW', 'WG']
sim_a = ['1.0000', '0.6250', '0.5000']
sim_space = ['r', 's'] # r for real space; s for redshift space
rec_dirs = [
'DD', 'ALL'
] # "ALL" folder stores P(k, \mu) after reconstruction process, while DD is before reconstruction.
rec_fprefix = ['', 'R']
mcut_Npar_list = [[37, 149, 516, 1524, 3830], [35, 123, 374, 962, 2105],
[34, 103, 290, 681, 1390]]
N_masscut = np.size(mcut_Npar_list, axis=1)
# Sigma_sm = sqrt(2.* Sig_RR) in post-reconstruction case, for pre-reconstruction, we don't use sub_Sigma_RR.
Sigma_RR_list = [[37, 48.5, 65.5, 84.2, 110], [33, 38, 48.5, 63.5, 91.5],
[31, 38, 49, 65, 86]]
sub_Sigma_RR = 50.0 # note from Hee-Jong's recording
inputf = '../Zvonimir_data/planck_camb_56106182_matterpower_smooth_z0.dat'
k_smooth, Pk_smooth = np.loadtxt(inputf,
dtype='f8',
comments='#',
unpack=True)
tck_Pk_sm = interpolate.splrep(k_smooth, Pk_smooth)
inputf = '../Zvonimir_data/planck_camb_56106182_matterpower_z0.dat'
k_wiggle, Pk_wiggle = np.loadtxt(inputf,
dtype='f8',
comments='#',
unpack=True)
tck_Pk_linw = interpolate.splrep(k_wiggle, Pk_wiggle)
# firstly, read one file and get k bins we want for the fitting range
dir0 = '/Users/ding/Documents/playground/WiggleNowiggle/subsample_FoF_data_HS/Pk_obs_2d_wnw_mean_DD_ksorted_mu_masscut/'
inputf = dir0 + 'fof_kaver.wnw_diff_a_0.6250_mcut35_fraction0.126.dat'
k_p, mu_p = np.loadtxt(inputf,
dtype='f8',
comments='#',
delimiter=' ',
usecols=(0, 1),
unpack=True)
#print(k_p, mu_p)
N_fitbin = len(k_p)
#print('# of (k, mu) bins: ', N_fitbin)
# for output parameters fitted
odir = './params_{}_wig-now_b_bscale_fitted_mean_dset/'.format(
rec_dirs[rec_id])
if not os.path.exists(odir):
os.makedirs(odir)
print("N_walkers: ", N_walkers, "N_walkersteps: ", N_walkersteps, "\n")
if rec_id == 0:
##Sigma_0 = 8.3364 # the approximated value of \Sigma_xy and \Sigma_z, unit Mpc/h, at z=0.
Sigma_0 = 7.8364 # suggested by Zvonimir, at z=0
elif rec_id == 1:
Sigma_0 = 2.84
##space_id = 1 # in redshift space
# 0: parameter fixed, 1: parameter free.
#params_indices = [1, 1, 1, 1, 1, 1] # It doesn't fit \Sigma and bscale well. Yes, it dosen't work well (it's kind of overfitting).
##params_indices = [1, 1, 1, 1, 0, 1, 1, 0, 0] # b0 needs to be fitted. For this case, make sure \Sigma_xy and \Sigma_z positive, which should be set in mcmc_routine.
##params_indices = [0, 1, 0, 1, 1, 0] # make sure \alpha_xy = \alpha_z and \Sigma_xy = \Sigma_z
params_indices = [
1, 1, 1, 1, 0, 0, 0, 0, 0
] # Set sigma_fog=0, f=0, b_scale=0, b_0=1.0 for subsamled DM case in real space.
##params_indices = [1, 1, 0, 0, 1, 1] # with fixed Sigma from theoretical value, then need to set sigma_xy, sigma_z equal to Sigma_z
##params_indices = [0, 1, 0, 0, 1, 1] # For this case, make sure \alpha_1 = \alpha_2 in the function lnlike(..) and set sigma_xy, sigma_z equal to Sigma_z.
print("params_indices: ", params_indices)
##alpha_1, alpha_2, sigma_fog, f, b_0, b_scale = 1.0, 1.0, 2.0, 0.2, 1.0, 0.0
alpha_1, alpha_2, sigma_fog, f, b_0, b_scale = 1.0, 1.0, 0.0, 0.0, 1.0, 0.0 # ! only for real space, i.e., set sigma_fog, f equal to 0.
all_names = "alpha_1", "alpha_2", "sigma_xy", "sigma_z", "sigma_sm", "sigma_fog", "f", "b_0", "b_scale" # the same order for params_indices
all_temperature = 0.01, 0.01, 0.1, 0.1, 0.1, 0.1, 0.1, 0.01, 0.1
pool = MPIPool(loadbalance=True)
for z_id in xrange(3):
norm_gf = growth_factor(float(sim_z[z_id]), Omega_m) / G_0
Sigma_z = Sigma_0 * norm_gf / 2.0 # divided by 2.0 for estimated \Sigma of post-reconstruction
##Sigma_z = Sigma_0* norm_gf
if set_Sigma_xyz_theory == "True":
print("Sigma_z: ", Sigma_z)
sigma_xy, sigma_z = Sigma_z, Sigma_z
else:
if params_indices[2] == 0:
sigma_xy = 0.0
else:
sigma_xy = 10.0
if params_indices[3] == 0:
sigma_z = 0.0
else:
sigma_z = 10.0
np.random.seed()
# Set it for FoF fitting
# for mcut_id in xrange(N_masscut):
# if set_Sigma_sm_theory == "True":
# sigma_sm = (float(Sigma_RR_list[z_id][mcut_id])*2.0)**0.5
# else:
# sigma_sm = 0.0
#
# all_params = alpha_1, alpha_2, sigma_xy, sigma_z, sigma_sm, sigma_fog, f, b_0, b_scale
# N_params, theta, fix_params, params_T, params_name = set_params(all_params, params_indices, all_names, all_temperature)
#
# ifile_Pk = './run2_3_Pk_obs_2d_wnw_mean_{}_ksorted_mu_masscut/{}kave{}.wig_minus_now_mean_fof_a_{}_mcut{}.dat'.format(rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], mcut_Npar_list[z_id][mcut_id])
# Pk_wnw_diff_obs = np.loadtxt(ifile_Pk, dtype='f4', comments='#', usecols=(2,)) # be careful that there are k, \mu, P(k, \mu) columns.
#
# ifile_Cov_Pk = './run2_3_Cov_Pk_obs_2d_wnw_{}_ksorted_mu_masscut/{}kave{}.wig_minus_now_mean_fof_a_{}_mcut{}.dat'.format(rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], mcut_Npar_list[z_id][mcut_id])
# Cov_Pk_wnw = np.loadtxt(ifile_Cov_Pk, dtype='f4', comments='#')
# ivar_Pk_wnow = N_dataset/np.diag(Cov_Pk_wnw) # the mean sigma error
#
# params_mcmc = mcmc_routine(N_params, N_walkers, N_walkersteps, theta, params_T, params_indices, fix_params, k_p, mu_p, Pk_wnw_diff_obs, ivar_Pk_wnow, tck_Pk_linw, tck_Pk_sm, norm_gf, params_name, pool)
#
# chi_square = chi2(params_mcmc[:, 0], params_indices, fix_params, k_p, mu_p, Pk_wnw_diff_obs, ivar_Pk_wnow, tck_Pk_linw, tck_Pk_sm, norm_gf)
# reduced_chi2 = chi_square/(N_fitbin-N_params)
# print("Reduced chi2: {}\n".format(reduced_chi2))
# # output parameters into a file
# if set_Sigma_xyz_theory == "False":
# ofile_params = odir + 'fof_{}kave{}.wnw_diff_a_{}_mcut{}_params{}_Sigma_sm{}.dat'.format(rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], mcut_Npar_list[z_id][mcut_id], ''.join(map(str, params_indices)), round(sigma_sm,3))
# else:
# ofile_params = odir + 'fof_{}kave{}.wnw_diff_a_{}_mcut{}_params{}_isotropic_Sigmaz_{}_Sigma_sm{}.dat'.format(rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], mcut_Npar_list[z_id][mcut_id], ''.join(map(str, params_indices)), round(Sigma_z, 3), round(sigma_sm,3))
# print(ofile_params)
# write_params(ofile_params, params_mcmc, params_name, reduced_chi2)
# set it for DM subsample fitting
sub_sigma_sm = (sub_Sigma_RR * 2.0)**0.5
print("sub_sigma_sm: ", sub_sigma_sm)
all_params = alpha_1, alpha_2, sigma_xy, sigma_z, sub_sigma_sm, sigma_fog, f, b_0, b_scale # set \Sigma_sm = sqrt(50*2)=10, for post-rec
N_params, theta, fix_params, params_T, params_name = set_params(
all_params, params_indices, all_names, all_temperature)
# Fit for DM power spectrum
ifile_Pk = './run2_3_sub_Pk_2d_wnw_mean_{}_ksorted_mu/{}kave{}.wig_minus_now_mean_sub_a_{}.dat'.format(
rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id],
sim_a[z_id])
Pk_wnw_diff_true = np.loadtxt(
ifile_Pk, dtype='f4', comments='#', usecols=(
2, )) # be careful that there are k, \mu, P(k, \mu) columns.
ifile_Cov_Pk = './run2_3_sub_Cov_Pk_2d_wnw_{}_ksorted_mu/{}kave{}.wig_minus_now_mean_sub_a_{}.dat'.format(
rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id],
sim_a[z_id])
Cov_Pk_wnw = np.loadtxt(ifile_Cov_Pk, dtype='f4', comments='#')
ivar_Pk_wnow = N_dataset / np.diag(Cov_Pk_wnw) # the mean sigma error
params_mcmc = mcmc_routine(N_params, N_walkers, N_walkersteps, theta,
params_T, params_indices, fix_params, k_p,
mu_p, Pk_wnw_diff_true, ivar_Pk_wnow,
tck_Pk_linw, tck_Pk_sm, norm_gf,
params_name, pool)
chi_square = chi2(params_mcmc[:, 0], params_indices, fix_params, k_p,
mu_p, Pk_wnw_diff_true, ivar_Pk_wnow, tck_Pk_linw,
tck_Pk_sm, norm_gf)
reduced_chi2 = chi_square / (N_fitbin - N_params)
print('Reduced chi2: {}\n'.format(reduced_chi2))
if rec_id == 1:
if set_Sigma_xyz_theory == "False":
ofile_params = odir + 'sub_{}kave{}.wnw_diff_a_{}_params{}_Sigma_sm{}.dat'.format(
rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id],
''.join(map(str, params_indices)), round(sub_sigma_sm, 3))
else:
ofile_params = odir + 'sub_{}kave{}.wnw_diff_a_{}_params{}_isotropic_Sigmaz_{}_Sigma_sm{}.dat'.format(
rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id],
''.join(map(str, params_indices)), round(Sigma_z, 3),
round(sub_sigma_sm, 3))
elif rec_id == 0:
if set_Sigma_xyz_theory == "False":
ofile_params = odir + 'sub_{}kave{}.wnw_diff_a_{}_params{}.dat'.format(
rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id],
''.join(map(str, params_indices)))
else:
ofile_params = odir + 'sub_{}kave{}.wnw_diff_a_{}_params{}_isotropic_Sigmaz_{}.dat'.format(
rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id],
''.join(map(str, params_indices)), round(Sigma_z, 3))
# write_params(ofile_params, params_mcmc, params_name, reduced_chi2)
pool.close()
def fit_subsamplefof_mean():
parser = argparse.ArgumentParser(
description=
'This is the MCMC code to get the fitting parameters, made by Zhejie Ding.'
)
parser.add_argument(
'-rec_id',
"--rec_id",
help='The id of reconstruction, either 0 or 1.',
required=True) #0: pre-reconstruct; 1: post-reconstruct
parser.add_argument(
'-space_id',
"--space_id",
help=
'The type of space for fitting, 0 for real-space and 1 for redshift space.',
required=True)
args = parser.parse_args()
print("args: ", args)
rec_id = int(args.rec_id)
print("rec_id: ", rec_id)
space_id = int(args.space_id)
print("space_id: ", space_id)
N_walkers = 40
# simulation run name
N_dataset = 20
N_mu_bin = 100
#N_skip_header = 11
#N_skip_footer = 31977
Omega_m = 0.3075
G_0 = growth_factor(0.0, Omega_m) # G_0 at z=0, normalization factor
Volume = 1380.0**3.0 # the volume of simulation box
sim_z = ['0', '0.6', '1.0']
sim_seed = [0, 9]
sim_wig = ['NW', 'WG']
sim_a = ['1.0000', '0.6250', '0.5000']
sim_space = ['r', 's'] # r for real space; s for redshift space
rec_dirs = [
'DD', 'ALL'
] # "ALL" folder stores P(k, \mu) after reconstruction process, while DD is before reconstruction.
rec_fprefix = ['', 'R']
mcut_Npar_list = [[37, 149, 516, 1524, 3830], [35, 123, 374, 962, 2105],
[34, 103, 290, 681, 1390]]
N_masscut = np.size(mcut_Npar_list, axis=1)
inputf = '../Zvonimir_data/planck_camb_56106182_matterpower_smooth_z0.dat'
k_smooth, Pk_smooth = np.loadtxt(inputf,
dtype='f8',
comments='#',
unpack=True)
tck_Pk_sm = interpolate.splrep(k_smooth, Pk_smooth)
inputf = '../Zvonimir_data/planck_camb_56106182_matterpower_z0.dat'
k_wiggle, Pk_wiggle = np.loadtxt(inputf,
dtype='f8',
comments='#',
unpack=True)
tck_Pk_linw = interpolate.splrep(k_wiggle, Pk_wiggle)
# firstly, read one file and get k bins we want for the fitting range
dir0 = '/Users/ding/Documents/playground/WiggleNowiggle/subsample_FoF_data_HS/Pk_obs_2d_wnw_mean_DD_ksorted_mu_masscut/'
inputf = dir0 + 'fof_kaver.wnw_diff_a_0.6250_mcut35_fraction0.126.dat'
k_p, mu_p = np.loadtxt(inputf,
dtype='f8',
comments='#',
delimiter=' ',
usecols=(0, 1),
unpack=True)
#print(k_p, mu_p)
N_fitbin = len(k_p)
#print('# of (k, mu) bins: ', N_fitbin)
# for output parameters fitted
odir = './ZV_lagrange_params_{}_wig-now_mean_dset/'.format(
rec_dirs[rec_id])
if not os.path.exists(odir):
os.makedirs(odir)
#space_id = 1 # in redshift space
if rec_id == 0:
##Sigma_0 = 8.3364 # the approximated value of \Sigma_xy and \Sigma_z, unit Mpc/h, at z=0.
Sigma_0 = 7.8364 # suggested by Zvonimir, at z=0
elif rec_id == 1:
Sigma_0 = 2.84
# 0: parameter fixed, 1: parameter free.
#params_indices = [1, 1, 1, 1, 1, 1, 0, 0] # Only for DM case. b0 needs to be fixed for DM subsample power spectrum. For this case, make sure \Sigma_xy and \Sigma_z positive, which should be set in mcmc_routine.
if space_id == 0:
# It may not work in real-space just setting f=0.
params_indices = [1, 1, 1, 0, 1, 1] # Set f=0 for real-space.
f = 0.0
elif space_id == 1:
params_indices = [1, 1, 1, 1, 1, 1] # for redshift space
f = 1.0
print("params_indices: ", params_indices)
alpha_1, alpha_2, sigma, b_0, b_scale = 1.0, 1.0, Sigma_0, 1.0, 0.0
all_names = "alpha_1", "alpha_2", "Sigma_qmax", "f", "b_1", "b_partial" # the same order for params_indices
all_temperature = 0.01, 0.01, 0.1, 0.1, 0.01, 0.1
pool = MPIPool(loadbalance=True)
for z_id in xrange(3):
norm_gf = growth_factor(float(sim_z[z_id]), Omega_m) / G_0
# ##Sigma_z = Sigma_0* norm_gf
#
# if set_Sigma_xyz_theory == "True":
# print("Sigma_z: ", Sigma_z)
# sigma_xy, sigma_z = Sigma_z, Sigma_z
# else:
# if params_indices[2] == 0:
# sigma_xy = 0.0
# else:
# sigma_xy = 10.0
# if params_indices[3] == 0:
# sigma_z = 0.0
# else:
# sigma_z = 10.0
all_params = alpha_1, alpha_2, sigma, f, b_0, b_scale
np.random.seed()
for mcut_id in xrange(N_masscut):
N_params, theta, fix_params, params_T, params_name = set_params(
all_params, params_indices, all_names, all_temperature)
ifile_Pk = './run2_3_Pk_obs_2d_wnw_mean_{}_ksorted_mu_masscut/{}kave{}.wig_minus_now_mean_fof_a_{}_mcut{}.dat'.format(
rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id],
sim_a[z_id], mcut_Npar_list[z_id][mcut_id])
Pk_wnw_diff_obs = np.loadtxt(
ifile_Pk, dtype='f4', comments='#', usecols=(2, )
) # be careful that there are k, \mu, P(k, \mu) columns.
ifile_Cov_Pk = './run2_3_Cov_Pk_obs_2d_wnw_{}_ksorted_mu_masscut/{}kave{}.wig_minus_now_mean_fof_a_{}_mcut{}.dat'.format(
rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id],
sim_a[z_id], mcut_Npar_list[z_id][mcut_id])
Cov_Pk_wnw = np.loadtxt(ifile_Cov_Pk, dtype='f4', comments='#')
ivar_Pk_wnow = N_dataset / np.diag(
Cov_Pk_wnw) # the mean sigma error
params_mcmc = mcmc_routine(N_params, N_walkers, theta, params_T,
params_indices, fix_params, k_p, mu_p,
Pk_wnw_diff_obs, ivar_Pk_wnow,
tck_Pk_linw, tck_Pk_sm, norm_gf,
params_name, pool)
chi_square = chi2(params_mcmc[:, 0], params_indices, fix_params,
k_p, mu_p, Pk_wnw_diff_obs, ivar_Pk_wnow,
tck_Pk_linw, tck_Pk_sm, norm_gf)
reduced_chi2 = chi_square / (N_fitbin - N_params)
print("Reduced chi2: {}\n".format(reduced_chi2))
# output parameters into a file
ofile_params = odir + 'fof_{}kave{}.wnw_diff_a_{}_mcut{}_params{}.dat'.format(
rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id],
mcut_Npar_list[z_id][mcut_id], ''.join(map(
str, params_indices)))
print(ofile_params)
write_params(ofile_params, params_mcmc, params_name, reduced_chi2)
np.random.seed()
N_params, theta, fix_params, params_T, params_name = set_params(
all_params, params_indices, all_names, all_temperature)
# Fit for DM power spectrum
ifile_Pk = './run2_3_sub_Pk_2d_wnw_mean_{}_ksorted_mu/{}kave{}.wig_minus_now_mean_sub_a_{}.dat'.format(
rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id],
sim_a[z_id])
Pk_wnw_diff_true = np.loadtxt(
ifile_Pk, dtype='f4', comments='#', usecols=(
2, )) # be careful that there are k, \mu, P(k, \mu) columns.
print(ifile_Pk)
ifile_Cov_Pk = './run2_3_sub_Cov_Pk_2d_wnw_{}_ksorted_mu/{}kave{}.wig_minus_now_mean_sub_a_{}.dat'.format(
rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id],
sim_a[z_id])
Cov_Pk_wnw = np.loadtxt(ifile_Cov_Pk, dtype='f4', comments='#')
ivar_Pk_wnow = N_dataset / np.diag(Cov_Pk_wnw) # the mean sigma error
params_mcmc = mcmc_routine(N_params, N_walkers, theta, params_T,
params_indices, fix_params, k_p, mu_p,
Pk_wnw_diff_true, ivar_Pk_wnow, tck_Pk_linw,
tck_Pk_sm, norm_gf, params_name, pool)
chi_square = chi2(params_mcmc[:, 0], params_indices, fix_params, k_p,
mu_p, Pk_wnw_diff_true, ivar_Pk_wnow, tck_Pk_linw,
tck_Pk_sm, norm_gf)
reduced_chi2 = chi_square / (N_fitbin - N_params)
print('Reduced chi2: {}\n'.format(reduced_chi2))
ofile_params = odir + 'sub_{}kave{}.wnw_diff_a_{}_params{}.dat'.format(
rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], ''.join(
map(str, params_indices)))
write_params(ofile_params, params_mcmc, params_name, reduced_chi2)
pool.close()
def fit_BAO(args):
kmin = float(args.kmin)
kmax = float(args.kmax)
params_str = args.params_str
Pk_type = args.Pk_type
params_indices = [int(i) for i in params_str]
old_stdout = sys.stdout
odir = './fit_kmin{}_kmax{}_{}/'.format(kmin, kmax, Pk_type)
if not os.path.exists(odir):
os.makedirs(odir)
ofile = odir + "mcmc_fit_params{}.log".format(params_str)
log_file = open(ofile, "w")
sys.stdout = log_file
print('Arguments for the fitting: ', args)
ifile = '/Users/mehdi/work/quicksurvey/ELG/run8/planck_camb_56106182_matterpower_z0.dat'
klin, Pk_linw = np.loadtxt(ifile, dtype='f8', comments='#', unpack=True)
Pwig_spl = InterpolatedUnivariateSpline(klin, Pk_linw)
ifile = '/Users/mehdi/work/quicksurvey/ELG/run8/planck_camb_56106182_matterpower_smooth_z0.dat'
klin, Pk_sm = np.loadtxt(ifile, dtype='f8', comments='#', unpack=True)
Psm_spl = InterpolatedUnivariateSpline(klin, Pk_sm)
norm_gf = 1.0
N_walkers = 40
##params_indices = [1, 0, 0] # 1: free parameter; 0: fixed parameter
all_param_names = 'alpha', 'Sigma2_xy', 'A', 'B'
all_temperature = 0.01, 1.0, 0.1, 0.1
Omega_m = 0.3075 # matter density
G_0 = growth_factor(0.0, Omega_m)
Sigma_0 = 7.7840 # This is exactly calculated from theoretical prediction with q_{BAO}=110 Mpc/h.
z_list = [0.625] #, 0.875, 1.125, 1.375] # z list for the data files
cut_list = ['F'] #, 'T']
# initial guess for fitting, Sigma2_xy=31.176 at z=0.65 is from theory prediction
alpha, A, B = 1.0, 1.0, 10.0
idir = '../kp0kp2knmodes/surveyscaled-nmodes/'
odir = './mcmc_fit_params_{}/kmin{}_kmax{}/'.format(Pk_type, kmin, kmax)
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if rank == 0:
if not os.path.exists(odir):
os.makedirs(odir)
pool = MPIPool(loadbalance=True)
for z_value in z_list:
norm_gf = growth_factor(z_value, Omega_m) / G_0
Sigma2_xy = (Sigma_0 * norm_gf)**2.0
print('z, Sigma2_xy: ', z_value, Sigma2_xy)
all_params = alpha, Sigma2_xy, A, B
N_params, theta, fix_params, params_T, params_name = set_params(
all_params, params_indices, all_param_names, all_temperature)
for cut_type in cut_list:
ifile = idir + 'kp0kp2knmodes_z{}RADECcut{}.dat'.format(
z_value, cut_type)
print(ifile)
data_m = np.loadtxt(ifile, dtype='f8',
comments='#') # k, P0(k), P2(k), N_modes
indices = np.argwhere((data_m[:, 0] >= kmin)
& (data_m[:, 0] <= kmax)).flatten()
N_fitbin = len(indices)
k_obs, Pk_wig_obs, N_modes = data_m[indices,
0], data_m[indices,
1], data_m[indices,
3]
ivar_Pk_wig = N_modes / (2.0 * Pk_wig_obs**2.0)
#print('ivar_Pk_wig', ivar_Pk_wig)
params_mcmc = mcmc_routine(N_params, N_walkers, theta, params_T,
params_indices, fix_params, k_obs,
Pk_wig_obs, ivar_Pk_wig, Pwig_spl,
Psm_spl, norm_gf, params_name, pool)
print(params_mcmc)
chi_square = chi2(params_mcmc[:, 0], params_indices, fix_params,
k_obs, Pk_wig_obs, ivar_Pk_wig, Pwig_spl,
Psm_spl, norm_gf)
reduced_chi2 = chi_square / (N_fitbin - N_params)
print("chi^2/dof: ", reduced_chi2, "\n")
ofile_params = odir + 'fit_p0_z{}RADECcut{}_params{}.dat'.format(
z_value, cut_type, params_str)
write_params(ofile_params, params_mcmc, params_name, reduced_chi2)
pool.close()
sys.stdout = old_stdout
log_file.close()
def show_sigma_Pwnw():
rec_id = 1
indices_pn = indices_p1
k_p = np.array(k_sorted[indices_pn])
Pk_sm_obsk = interpolate.splev(
k_p, tck_Pk_smooth,
der=0) # interpolate theoretical linear P(k) at points k_p
mu_p = np.array(mu_sorted[indices_pn])
Nmode_p = np.array(Nmode_sorted[indices_pn])
Nmode_p = Nmode_p * N_dset # count the number of data sets (N_dset)
mu_boundary = np.round([min(mu_p), max(mu_p)], decimals=1)
print(mu_boundary)
n_fitbin = np.size(mu_p)
print(n_fitbin)
idir_Pk = [
'./run2_3_Pk_obs_2d_wnw_mean_{}_ksorted_mu_masscut/'.format(
rec_dirs[rec_id]),
'./run2_3_Pk_2d_wnw_mean_{}_ksorted_mu_masscut/'.format(
rec_dirs[rec_id])
]
idir_covPk = [
'./run2_3_Cov_Pk_obs_2d_wnw_mean_{}_ksorted_mu_masscut/'.format(
rec_dirs[rec_id]),
'./run2_3_Cov_Pk_2d_wnw_mean_{}_ksorted_mu_masscut/'.format(
rec_dirs[rec_id])
]
odir = './{}_figs_fofPkobs_subPktrue/'.format(rec_dirs[rec_id])
#odir = './{}_figs_barn_mean/'.format(rec_dirs[rec_id])
for space_id in xrange(1):
for z_id in xrange(3):
for mcut_id in xrange(4, 5): #(N_masscut[z_id]):
bias = bias_array[z_id][mcut_id]
z = float(sim_z[z_id])
print("z=", z, "G: ", growth_factor(z, Omega_m) / G_0)
G2b2 = (growth_factor(z, Omega_m) / G_0 * bias)**2.0
inputf = idir_Pk[
1] + '{}kave{}.wnw_mean_fof_a_{}_mcut{}.dat'.format(
rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id],
mcut_Npar_list[z_id][mcut_id])
Pk_wnw = np.loadtxt(inputf,
dtype='f4',
comments='#',
usecols=(2, ))
inputf = idir_covPk[
1] + '{}kave{}.wnw_mean_fof_a_{}_mcut{}.dat'.format(
rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id],
mcut_Npar_list[z_id][mcut_id])
Cov_Pk_a = np.loadtxt(inputf, dtype='f4', comments='#')
sigma_Pk_a = (np.diag(Cov_Pk_a)[indices_pn])**0.5
sigma_Pk_wnw = sigma_Pk_a / math.sqrt(float(N_dset))
##sigma_Pk_wnw = sigma_Pk_a/math.sqrt(float(N_dset))* G2b2 * Pk_sm_obsk # get sigma_(Pwig/Pnow)*P_{Lin,w}
relative_sigma_Pk_wnw = sigma_Pk_wnw / Pk_wnw[
indices_pn] # get the relative error
print(sigma_Pk_wnw.shape)
inputf = idir_Pk[
0] + '{}kave{}.wnw_diff_mean_fof_a_{}_mcut{}.dat'.format(
rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id],
mcut_Npar_list[z_id][mcut_id])
Pk_wnw_diff = np.loadtxt(inputf,
dtype='f4',
comments='#',
usecols=(2, ))
Pk_wig_minus_now = Pk_wnw_diff[
indices_pn] * G2b2 * Pk_sm_obsk # get back P_wig-P_now
inputf = idir_covPk[
0] + '{}kave{}.wnw_diff_mean_fof_a_{}_mcut{}.dat'.format(
rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id],
mcut_Npar_list[z_id][mcut_id])
Cov_Pk_a = np.loadtxt(inputf, dtype='f4', comments='#')
sigma_Pk_a = (np.diag(Cov_Pk_a)[indices_pn])**0.5
sigma_Pk_wnw_diff = sigma_Pk_a / math.sqrt(float(N_dset))
##sigma_Pk_wnw_diff = sigma_Pk_wnw_diff * G2b2 * Pk_sm_obsk # get back sigma_Pk_wnw_diff
##relative_sigma_Pk_wnw_diff = sigma_Pk_wnw_diff/np.abs(Pk_wnw_diff[indices_pn]) # get the relative error
relative_sigma_Pk_wnw_diff = sigma_Pk_wnw_diff / (
abs(Pk_wig_minus_now) + 1.0
) # get the "relative" error, add 1.0 in the denominator
#print sigma_Pk_a.shape
plot_sigma_Pk_wnw_mean(k_p, sigma_Pk_wnw, sigma_Pk_wnw_diff,
Nmode_p, mu_boundary, rec_id, space_id,
z_id, mcut_id, odir)
def Pk_wnw_diff_divideby_b2G2Psm():
rec_id = 0 # rec_id=0: before reconstruction; rec_id=1: after reconstruction
odir_prefix = './run2_3_'
for z_id in xrange(1, 3):
for mcut_id in xrange(N_masscut):
for space_id in xrange(1):
Pk_mwnw_diff_obs = np.array([], dtype=np.float64).reshape(
0, n_fitbin)
for run_id in xrange(2):
for sim_seed_id in xrange(10):
k_obs, Pk_now_obs, Pk_now_true = read_fftPk_file(
dir0, z_id, mcut_id, rec_id, space_id,
sim_wig[run_id][0], sim_seed_id)
k_obs, Pk_wig_obs, Pk_wig_true = read_fftPk_file(
dir0, z_id, mcut_id, rec_id, space_id,
sim_wig[run_id][1], sim_seed_id)
Pk_wnw_diff_obs = (Pk_wig_obs -
Pk_now_obs) / Pk_sm_obsk
Pk_mwnw_diff_obs = np.vstack(
[Pk_mwnw_diff_obs, Pk_wnw_diff_obs])
Pk_wnw_diff_obs_mean = np.mean(Pk_mwnw_diff_obs, axis=0)
for i in xrange(n_fitbin):
Pk_mwnw_diff_obs[:,
i] = Pk_mwnw_diff_obs[:,
i] - Pk_wnw_diff_obs_mean[
i]
Cov_Pk = np.dot(Pk_mwnw_diff_obs.T,
Pk_mwnw_diff_obs) / (N_dataset - 1.0)
#print(Cov_Pk, Cov_Pk.shape)
# add the const factor with growth function and bias b
bias = bias_list[z_id][mcut_id]
z = float(sim_z[z_id])
print("z=", z, "G: ", growth_factor(z, Omega_m) / G_0)
G2b2 = (growth_factor(z, Omega_m) / G_0 * bias)**2.0
Pk_wnw_diff_obs_mean = Pk_wnw_diff_obs_mean / G2b2
Cov_Pk = Cov_Pk / G2b2**2.0
var_name = 'Pk_obs_2d_wnw_mean_{}_ksorted_mu_masscut/'.format(
rec_dirs[rec_id])
filename = "{}kave{}.wnw_diff_mean_fof_a_{}_mcut{}.dat".format(
rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id],
mcut_Npar_list[z_id][mcut_id])
header_line = ' After sorting k, the mean (P(k)_wig-P(k)_now)/(G^2*b^2*Plinnw) in 2d (k, mu) case\n k mu Pk_wnw_diff_obs_mean'
write_output(odir_prefix, var_name, header_line, rec_id,
space_id, z_id, mcut_id,
np.array([k_p, mu_p,
Pk_wnw_diff_obs_mean]).T, filename)
var_name = 'Cov_Pk_obs_2d_wnw_mean_{}_ksorted_mu_masscut/'.format(
rec_dirs[rec_id])
header_line = ' Cov(P_obs_2d_wnw_diff_b2G2(k1), P_obs_2d_wnw_diff_b2G2(k2)), (obs: including shot noise; wnw_diff_b2G2 means (Pwig-Pnow)/(b^2 G^2 Psm), 2d: k, mu; and k1, k2 are the k bin indices).'
filename = "{}kave{}.wnw_diff_mean_fof_a_{}_mcut{}.dat".format(
rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id],
mcut_Npar_list[z_id][mcut_id])
write_output(odir_prefix, var_name, header_line, rec_id,
space_id, z_id, mcut_id, Cov_Pk, filename)
sim_z=['0', '0.6', '1.0']
sim_seed = [0, 9]
sim_wig = ['NW', 'WG']
sim_a = ['1.0000', '0.6250', '0.5000']
sim_space = ['r', 's'] # r for real space; s for redshift space
title_space = ['in real space', 'in redshift space']
rec_dirs = ['DD', 'ALL'] # "ALL" folder stores P(k, \mu) after reconstruction process, while DD is before reconstruction.
rec_fprefix = ['', 'R']
R_bao = 110. # unit: Mpc/h
space_id = 0
k_cut = 0.3
ns = 0.965
Omega_m = 0.3075
G_0 = growth_factor(0.0, Omega_m) # G_0 at z=0, normalization factor
G_z = np.array([growth_factor(float(z), Omega_m)/G_0 for z in sim_z])
print("Growth factor: ", G_z)
inputf = '../Zvonimir_data/planck_camb_56106182_matterpower_z0.dat'
k_wiggle, Pk_wiggle = np.loadtxt(inputf, dtype='f8', comments='#', unpack=True)
tck_Pk_linw = interpolate.splrep(k_wiggle, Pk_wiggle, k=3)
print(k_wiggle[0], k_wiggle[-1], len(k_wiggle))
const = 1./(6.0*math.pi**2.0)
##sigma_v2_err = quad(lambda x: interpolate.splev(x, tck_Pk_linw, der=0), k_wiggle[0], k_wiggle[-1])
sigma_v2_err = quad(lambda x: interpolate.splev(x, tck_Pk_linw, der=0)*(1.0-math.sin(x*R_bao)/(x*R_bao)), k_wiggle[0], 100.0) # the expression is from Zvonimir's suggestion.
print(sigma_v2_err)
sigma_v2 = sigma_v2_err[0]*const
Sigma_Plinw = (2*sigma_v2)**0.5
print("Sigma_Plinw: ", Sigma_Plinw)
def fit_subsamplefof_mean():
parser = argparse.ArgumentParser(description='This is the MCMC code to get the fitting parameters using Zvonimir model, made by Zhejie Ding.')
parser.add_argument('-rec_id', "--rec_id", help='The id of reconstruction, either 0 or 1.', required=True) #0: pre-reconstruct; 1: post-reconstruct
args = parser.parse_args()
rec_id = int(args.rec_id)
print("rec_id: ", rec_id)
N_walkers = 200
N_walkersteps = 20000
space_id = 1 # in redshift space
print("N_walkers: ", N_walkers, "N_walkersteps: ", N_walkersteps, "\n")
# 0: parameter fixed, 1: parameter free.
params_indices = [1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1] # b0 needs to be fitted. For this case, make sure \Sigma_0,2,4 positive, which should be set in mcmc_routine.
##params_indices = [0, 1, 0, 0, 1, 1] # For this case, make sure \alpha_1 = \alpha_2 in the function lnlike(..) and set sigma_xy, sigma_z equal to Sigma_z.
print("params_indices: ", params_indices)
# simulation run name
N_dataset = 20
N_mu_bin = 100
#N_skip_header = 11
#N_skip_footer = 31977
Omega_m = 0.3075
G_0 = growth_factor(0.0, Omega_m) # G_0 at z=0, normalization factor
Volume = 1380.0**3.0 # the volume of simulation box
sim_z=['0', '0.6', '1.0']
sim_seed = [0, 9]
sim_wig = ['NW', 'WG']
sim_a = ['1.0000', '0.6250', '0.5000']
sim_space = ['r', 's'] # r for real space; s for redshift space
rec_dirs = ['DD', 'ALL'] # "ALL" folder stores P(k, \mu) after reconstruction process, while DD is before reconstruction.
rec_fprefix = ['', 'R']
mcut_Npar_list = [[37, 149, 516, 1524, 3830],
[35, 123, 374, 962, 2105],
[34, 103, 290, 681, 1390]]
N_masscut = np.size(mcut_Npar_list, axis=1)
# Sigma_sm = sqrt(2.* Sig_RR) in post-reconstruction case, for pre-reconstruction, we don't use sub_Sigma_RR.
Sigma_RR_list = [[37, 48.5, 65.5, 84.2, 110],
[33, 38, 48.5, 63.5, 91.5],
[31, 38, 49, 65, 86]]
sub_Sigma_RR = 50.0 # note from Hee-Jong's recording
inputf = '../Zvonimir_data/planck_camb_56106182_matterpower_smooth_z0.dat'
k_smooth, Pk_smooth = np.loadtxt(inputf, dtype='f8', comments='#', unpack=True)
tck_Pk_sm = interpolate.splrep(k_smooth, Pk_smooth)
inputf = '../Zvonimir_data/planck_camb_56106182_matterpower_z0.dat'
k_wiggle, Pk_wiggle = np.loadtxt(inputf, dtype='f8', comments='#', unpack=True)
tck_Pk_linw = interpolate.splrep(k_wiggle, Pk_wiggle)
# firstly, read one file and get k bins we want for the fitting range
dir0='/Users/ding/Documents/playground/WiggleNowiggle/subsample_FoF_data_HS/Pk_obs_2d_wnw_mean_DD_ksorted_mu_masscut/'
inputf = dir0 +'fof_kaver.wnw_diff_a_0.6250_mcut35_fraction0.126.dat'
k_p, mu_p = np.loadtxt(inputf, dtype='f8', comments='#', delimiter=' ', usecols=(0,1), unpack=True)
#print(k_p, mu_p)
N_fitbin = len(k_p)
#print('# of (k, mu) bins: ', N_fitbin)
# for output parameters fitted
odir = './params_{}_wig-now_b_bscale_fitted_mean_dset/'.format(rec_dirs[rec_id])
if not os.path.exists(odir):
os.makedirs(odir)
alpha_1, alpha_2, sigma_0, sigma_2, sigma_4, f, b_0, b_scale0, b_scale2, b_scale4 = 1.0, 1.0, 8.0, 8.0, 8.0, 0.2, 1.0, 0.0, 0.0, 0.0
all_names = "alpha_1", "alpha_2", "sigma_0", "sigma_2", "sigma_4", "sigma_sm", "f", "b_0", "b_scale0", "b_scale2", "b_scale4" # the same order for params_indices
all_temperature = 0.01, 0.01, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 1.0, 1.0, 1.0
pool = MPIPool(loadbalance=True)
for z_id in xrange(1):
norm_gf = growth_factor(float(sim_z[z_id]), Omega_m)/G_0
# for mcut_id in xrange(N_masscut):
# np.random.seed()
# sigma_sm = (float(Sigma_RR_list[z_id][mcut_id])*2.0)**0.5
# all_params = alpha_1, alpha_2, sigma_0, sigma_2, sigma_4, sigma_sm, f, b_0, b_scale0, b_scale2, b_scale4
# N_params, theta, fix_params, params_T, params_name = set_params(all_params, params_indices, all_names, all_temperature)
#
# ifile_Pk = './run2_3_Pk_obs_2d_wnw_mean_{}_ksorted_mu_masscut/{}kave{}.wig_minus_now_mean_fof_a_{}_mcut{}.dat'.format(rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], mcut_Npar_list[z_id][mcut_id])
# print(ifile_Pk)
# Pk_wnw_diff_obs = np.loadtxt(ifile_Pk, dtype='f4', comments='#', usecols=(2,)) # be careful that there are k, \mu, P(k, \mu) columns.
#
# ifile_Cov_Pk = './run2_3_Cov_Pk_obs_2d_wnw_{}_ksorted_mu_masscut/{}kave{}.wig_minus_now_mean_fof_a_{}_mcut{}.dat'.format(rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], mcut_Npar_list[z_id][mcut_id])
# Cov_Pk_wnw = np.loadtxt(ifile_Cov_Pk, dtype='f4', comments='#')
# ivar_Pk_wnow = N_dataset/np.diag(Cov_Pk_wnw) # the mean sigma error
#
# params_mcmc = mcmc_routine(N_params, N_walkers, N_walkersteps, theta, params_T, params_indices, fix_params, k_p, mu_p, Pk_wnw_diff_obs, ivar_Pk_wnow, tck_Pk_linw, tck_Pk_sm, norm_gf, params_name, pool)
#
# chi_square = chi2(params_mcmc[:, 0], params_indices, fix_params, k_p, mu_p, Pk_wnw_diff_obs, ivar_Pk_wnow, tck_Pk_linw, tck_Pk_sm, norm_gf)
# reduced_chi2 = chi_square/(N_fitbin-N_params)
# print("Reduced chi2: {}\n".format(reduced_chi2))
# # output parameters into a file
# ofile_params = odir + 'ZV_fof_{}kave{}.wnw_diff_a_{}_mcut{}_params{}_Sigma_sm{}.dat'.format(rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], mcut_Npar_list[z_id][mcut_id], ''.join(map(str, params_indices)), round(sigma_sm, 3))
# write_params(ofile_params, params_mcmc, params_name, reduced_chi2)
np.random.seed()
sigma_sm = 10.0 # for all redshifts, it's the same value.
all_params = alpha_1, alpha_2, sigma_0, sigma_2, sigma_4, sigma_sm, f, b_0, b_scale0, b_scale2, b_scale4
N_params, theta, fix_params, params_T, params_name = set_params(all_params, params_indices, all_names, all_temperature)
# Fit for DM power spectrum
ifile_Pk = './run2_3_sub_Pk_2d_wnw_mean_{}_ksorted_mu/{}kave{}.wig_minus_now_mean_sub_a_{}.dat'.format(rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id])
Pk_wnw_diff_true = np.loadtxt(ifile_Pk, dtype='f4', comments='#', usecols=(2,)) # be careful that there are k, \mu, P(k, \mu) columns.
print(ifile_Pk)
ifile_Cov_Pk = './run2_3_sub_Cov_Pk_2d_wnw_{}_ksorted_mu/{}kave{}.wig_minus_now_mean_sub_a_{}.dat'.format(rec_dirs[rec_id], rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id])
Cov_Pk_wnw = np.loadtxt(ifile_Cov_Pk, dtype='f4', comments='#')
ivar_Pk_wnow = N_dataset/np.diag(Cov_Pk_wnw) # the mean sigma error
params_mcmc = mcmc_routine(N_params, N_walkers, N_walkersteps, theta, params_T, params_indices, fix_params, k_p, mu_p, Pk_wnw_diff_true, ivar_Pk_wnow, tck_Pk_linw, tck_Pk_sm, norm_gf, params_name, pool)
chi_square = chi2(params_mcmc[:, 0], params_indices, fix_params, k_p, mu_p, Pk_wnw_diff_true, ivar_Pk_wnow, tck_Pk_linw, tck_Pk_sm, norm_gf)
reduced_chi2 = chi_square/(N_fitbin-N_params)
print('Reduced chi2: {}\n'.format(reduced_chi2))
ofile_params = odir + 'ZV_sub_{}kave{}.wnw_diff_a_{}_params{}_Sigma_sm{}.dat'.format(rec_fprefix[rec_id], sim_space[space_id], sim_a[z_id], ''.join(map(str, params_indices)), round(sigma_sm, 3))
write_params(ofile_params, params_mcmc, params_name, reduced_chi2)
pool.close()
