def execute(self, processes):
pwr_noise = ps.PowerSpectrum(self.params["p_noise"])
noise2d_agg = pwr_noise.agg_stat_2d_pwrspec()
transfer_dict = pe.load_transferfunc(
self.params["apply_2d_beamtransfer"],
self.params["apply_2d_modetransfer"], pwr_noise.treatment_cases)
weightfile = h5py.File(self.params["outfile"], "w")
for treatment in pwr_noise.treatment_cases:
# note that we sum in constant |k| annuli so k^3 in k^3 P(k) is a
# constant factor
comb = pwr_noise.comb_cases[0]
pwrcase = "%s:%s" % (comb, treatment)
weight = np.abs(pwr_noise.counts_2d[pwrcase])
weight /= noise2d_agg[treatment]["mean"]**2.
if transfer_dict is not None:
weight *= transfer_dict[treatment]**2.
# this is a little excessive, but to be safe:
nanmask = np.isnan(weight)
infmask = np.isnan(weight)
zeromask = (pwr_noise.counts_2d[pwrcase] == 0)
mask = np.where(
np.logical_or(np.logical_or(nanmask, infmask), zeromask))
weight[mask] = 0.
weightfile[treatment] = weight
weightfile.close()
def execute(self, processes):
pwr_physsim = ps.PowerSpectrum(self.params["pwr_file"])
pwr_physsim.convert_2d_to_1d()
pwr_physsim_summary = pwr_physsim.agg_stat_1d_pwrspec(from_2d=True)
k_vec = pwr_physsim.k_1d_from_2d["center"]
treatment = "phys"
mean_sim = pwr_physsim_summary[treatment]["mean"]
std_sim = pwr_physsim_summary[treatment]["std"]
outfile = "%s/pk_%s.dat" % (self.params["outdir"], treatment)
file_tools.print_multicolumn(k_vec, mean_sim, std_sim, outfile=outfile)
print "writing to " + outfile
def execute(self, processes):
print "using data: ", self.params["p_data"]
print "using mock: ", self.params["p_mock"]
pwr_data = ps.PowerSpectrum(self.params["p_data"])
pwr_mock = ps.PowerSpectrum(self.params["p_mock"])
transfer_dict = pe.load_transferfunc(
self.params["apply_2d_beamtransfer"],
self.params["apply_2d_modetransfer"],
pwr_mock.treatment_cases)
if transfer_dict is not None:
pwr_mock.apply_2d_trans_by_treatment(transfer_dict)
pwr_data.apply_2d_trans_by_treatment(transfer_dict)
# gather the rms of each 2D k-bin over the realizations of mock
# catalogs; this is used for the 2D->1D weighting
#mock2d_agg = pwr_mock.agg_stat_2d_pwrspec(debug="./plotheap/")
mock2d_agg = pwr_mock.agg_stat_2d_pwrspec()
weights_2d = {}
# weight by the variance as determined in the mock runs
for treatment in pwr_mock.treatment_cases:
weight_for_treatment = 1. / (mock2d_agg[treatment]["std"] * \
mock2d_agg[treatment]["std"])
kx = pwr_mock.kx_2d['center']
ky = pwr_mock.ky_2d['center']
kmin_x = self.params['kmin_xy'][0]
kmin_y = self.params['kmin_xy'][1]
if kmin_x is not None:
restrict = np.where(kx < kmin_x)
weight_for_treatment[restrict, :] = 0.
if kmin_y is not None:
restrict = np.where(ky < kmin_y)
weight_for_treatment[:, restrict] = 0.
weight_for_treatment[np.isnan(weight_for_treatment)] = 0.
weight_for_treatment[np.isinf(weight_for_treatment)] = 0.
weights_2d[treatment] = weight_for_treatment
outplot_power_file = "%s/power_2d_%s.png" % \
(self.params['outdir'], treatment)
outplot_transfer_file = "%s/transfer_2d_%s.png" % \
(self.params['outdir'], treatment)
outplot_weight_file = "%s/noiseweight_2d_%s.png" % \
(self.params['outdir'], treatment)
outplot_count_file = "%s/countweight_2d_%s.png" % \
(self.params['outdir'], treatment)
# comb is used for autopower AxB etc.; here just use a placeholder
logkx = np.log10(pwr_mock.kx_2d['center'])
logky = np.log10(pwr_mock.ky_2d['center'])
comb = pwr_data.comb_cases[0]
pwrcase = "%s:%s" % (comb, treatment)
plot_slice.simpleplot_2D(outplot_power_file,
np.abs(pwr_data.pwrspec_2d[pwrcase]),
logkx, logky,
["logkx", "logky"], 1., "2d power",
"log(abs(2d power))", logscale=True)
if (modetransfer_2d is not None) or (beamtransfer_2d is not None):
plot_slice.simpleplot_2D(outplot_transfer_file,
transfer_dict[treatment],
logkx, logky,
["logkx", "logky"], 1., "2d trans",
"2d trans", logscale=False)
#if treatment == "20modes":
# np.set_printoptions(threshold=np.nan)
# print mock2d_agg[treatment]["std"]
# print np.abs(weights_2d[treatment])
plot_slice.simpleplot_2D(outplot_weight_file,
np.abs(weights_2d[treatment]),
logkx, logky,
["logkx", "logky"], 1., "Log-weight",
"log-weight", logscale=True)
comb = pwr_mock.comb_cases[0]
pwrcase = "%s:%s" % (comb, treatment)
plot_slice.simpleplot_2D(outplot_count_file,
np.abs(pwr_mock.counts_2d[pwrcase]),
logkx, logky,
["logkx", "logky"], 1., "Log-counts",
"log-counts", logscale=True)
if self.params["use_noiseweights_2dto1d"]:
pwr_data.convert_2d_to_1d(weights_2d=weights_2d)
pwr_mock.convert_2d_to_1d(weights_2d=weights_2d)
else:
# use the counts to weight instead by default
# just be sure the transfer function is applied
pwr_data.convert_2d_to_1d()
pwr_mock.convert_2d_to_1d()
pwr_data_summary = pwr_data.agg_stat_1d_pwrspec(from_2d=True)
pwr_mock_summary = pwr_mock.agg_stat_1d_pwrspec(from_2d=True)
# build the hd5 summary file
sum_filename = "%s/cross_summary.hd5" % self.params["outdir"]
summaryfile = h5py.File(sum_filename, "w")
power_1d_out = summaryfile.create_group("power_1d")
power_1d_cov_out = summaryfile.create_group("power_1d_cov")
kvec_out = summaryfile.create_group("kvec")
k_vec = pwr_data.k_1d_from_2d["center"]
kvec_out["k_1d_left"] = pwr_data.k_1d_from_2d["left"]
kvec_out["k_1d_center"] = pwr_data.k_1d_from_2d["center"]
kvec_out["k_1d_right"] = pwr_data.k_1d_from_2d["right"]
for treatment in pwr_data.treatment_cases:
mean_data = pwr_data_summary[treatment]["mean"]
mean_mock = pwr_mock_summary[treatment]["mean"]
std_mock = pwr_mock_summary[treatment]["std"]
cov_mock = pwr_mock_summary[treatment]["cov"]
corr_mock = pwr_mock_summary[treatment]["corr"]
power_1d_out[treatment] = mean_data
power_1d_cov_out[treatment] = cov_mock
logk = np.log10(k_vec)
outplot_corr_file = "%s/covariance_1d_%s.png" % \
(self.params['outdir'], treatment)
plot_slice.simpleplot_2D(outplot_corr_file,
corr_mock,
logk, logk,
["log k", "log k"], 1., "1d corr",
"1d corr", logscale=False)
outfile = "%s/pk_%s.dat" % (self.params["outdir"], treatment)
file_tools.print_multicolumn(k_vec, mean_data, mean_mock, std_mock,
outfile=outfile)
print "writing to " + outfile
summaryfile.close()
def produce_summary(self, filelist, outfile, debug=False):
num_sim = len(filelist)
print "number of simulations to aggregate: %d" % num_sim
data_subtract = None
if self.params['subtract_pwrspec'] is not None:
print "agg WARNING: you are subtracting a power spectrum"
print "from file ", self.params['subtract_pwrspec']
data_subtract = ps.PowerSpectrum(self.params['subtract_pwrspec'])
# accumulate information about this set of files by loading just the
# first one:
sim_toread = ps.PowerSpectrum(filelist[0])
# run this here just to get the 2D->1D bin centers
sim_toread.convert_2d_to_1d()
k_1d = sim_toread.k_1d
k_1d_from_2d = sim_toread.k_1d_from_2d
num_k_1d = sim_toread.num_k_1d
num_k_1d_from_2d = sim_toread.num_k_1d_from_2d
(kx_2d, ky_2d) = (sim_toread.kx_2d, sim_toread.ky_2d)
(kx, ky) = (kx_2d['center'], ky_2d['center'])
(num_kx, num_ky) = (sim_toread.num_kx, sim_toread.num_ky)
print "k_1d bins: %d, %d, kx bins: %d, ky bins: %d" % \
(num_k_1d, num_k_1d_from_2d, num_kx, num_ky)
print "AggregateSummary: treatment cases: ", sim_toread.treatment_cases
# mask any parts of k space (useful for making signal simulations)
if self.params["kpar_range"] is not None:
print "restricting k_par to ", self.params["kpar_range"]
restrict_par = np.where(
np.logical_or(ky < self.params["kpar_range"][0],
ky > self.params["kpar_range"][1]))
else:
restrict_par = None
if self.params["kperp_range"] is not None:
print "restricting k_perp to ", self.params["kperp_range"]
restrict_perp = np.where(
np.logical_or(kx < self.params["kperp_range"][0],
kx > self.params["kperp_range"][1]))
else:
restrict_perp = None
# load the transfer functions and 2d->1d noise weights
# if applying a mode correction at fixed treatment (for Gaussian noise
# model)
fixed_treatment = self.params["fix_weight_treatment"]
if fixed_treatment:
checklist = None
else:
checklist = sim_toread.treatment_cases
full_transfer_dict = pe.load_transferfunc(
self.params["apply_2d_beamtransfer"],
self.params["apply_2d_modetransfer"], checklist)
transfer_dict = None
if full_transfer_dict is not None:
transfer_dict = {}
for treatment in sim_toread.treatment_cases:
if fixed_treatment is not None:
print "fixing transfer for %s to value at %s" % \
(treatment, fixed_treatment)
transfer_dict[treatment] = full_transfer_dict[
fixed_treatment]
else:
transfer_dict[treatment] = full_transfer_dict[treatment]
weights_2d = None
if self.params["noiseweights_2dto1d"] is not None:
print "applying 2D noise weights: " + \
self.params["noiseweights_2dto1d"]
weightfile = h5py.File(self.params["noiseweights_2dto1d"], "r")
weights_2d = {}
for treatment in sim_toread.treatment_cases:
if fixed_treatment is not None:
print "fixing weight for %s to value at %s" % \
(treatment, fixed_treatment)
weights_2d[treatment] = weightfile[fixed_treatment].value
else:
weights_2d[treatment] = weightfile[treatment].value
if restrict_perp is not None:
weights_2d[treatment][restrict_perp, :] = 0.
if restrict_par is not None:
weights_2d[treatment][:, restrict_par] = 0.
#outplot_weight_file = "cool_noiseweight_2d_%s.png" % treatment
#plot_slice.simpleplot_2D(outplot_weight_file,
# np.abs(weights_2d[treatment]),
# np.log10(kx), np.log10(ky),
# ["logkx", "logky"], 1., "Log-weight",
# "log-weight", logscale=True)
weightfile.close()
result_dict = {}
for treatment in sim_toread.treatment_cases:
treatment_dict = {}
treatment_dict["pk_1d"] = np.zeros((num_sim, num_k_1d))
treatment_dict["pk_1d_from_2d"] = \
np.zeros((num_sim, num_k_1d_from_2d))
treatment_dict["pkstd_1d"] = np.zeros((num_sim, num_k_1d))
treatment_dict["pkstd_1d_from_2d"] = \
np.zeros((num_sim, num_k_1d_from_2d))
treatment_dict["pk_2d"] = np.zeros((num_sim, num_kx, num_ky))
treatment_dict["counts_1d"] = np.zeros((num_sim, num_k_1d))
treatment_dict["counts_1d_from_2d"] = \
np.zeros((num_sim, num_k_1d_from_2d))
treatment_dict["counts_2d"] = np.zeros((num_sim, num_kx, num_ky))
result_dict[treatment] = treatment_dict
for (simfile, index) in zip(filelist, range(num_sim)):
print "processing ", simfile, index
sim_toread = ps.PowerSpectrum(simfile)
# optionally subtract some reference spectrum
if data_subtract is not None:
print "agg WARNING: you are subtracting a power spectrum"
# assuming these both have the same treatments
for pwrspec_case in sim_toread.pwrspec_1d:
sim_toread.pwrspec_1d[pwrspec_case] -= \
data_subtract.pwrspec_1d[pwrspec_case]
sim_toread.pwrspec_2d[pwrspec_case] -= \
data_subtract.pwrspec_2d[pwrspec_case]
sim_toread.apply_2d_trans_by_treatment(transfer_dict)
sim_toread.convert_2d_to_1d(weights_2d=weights_2d)
agg1d = sim_toread.agg_stat_1d_pwrspec()
agg1d_from_2d = sim_toread.agg_stat_1d_pwrspec(from_2d=True)
agg2d = sim_toread.agg_stat_2d_pwrspec()
for treatment in sim_toread.treatment_cases:
result_dict[treatment]["pk_1d"][index, :] = \
agg1d[treatment]['mean']
result_dict[treatment]["pk_1d_from_2d"][index, :] = \
agg1d_from_2d[treatment]['mean']
result_dict[treatment]["pkstd_1d"][index, :] = \
agg1d[treatment]['std']
result_dict[treatment]["pkstd_1d_from_2d"][index, :] = \
agg1d_from_2d[treatment]['std']
result_dict[treatment]["pk_2d"][index, :, :] = \
agg2d[treatment]['mean']
result_dict[treatment]["counts_1d"][index, :] = \
agg1d[treatment]['counts']
result_dict[treatment]["counts_1d_from_2d"][index, :] = \
agg1d_from_2d[treatment]['counts']
result_dict[treatment]["counts_2d"][index, :, :] = \
agg2d[treatment]['counts']
# package all simulations of all treatments into a file
#out_dicttree = shelve.open(outfile, "n", protocol=-1)
out_dicttree = {}
out_dicttree["k_1d"] = k_1d
out_dicttree["k_1d_from_2d"] = k_1d_from_2d
out_dicttree["kx_2d"] = kx_2d
out_dicttree["ky_2d"] = ky_2d
out_dicttree["results"] = result_dict
file_tools.convert_numpytree_hdf5(out_dicttree, outfile)
def execute(self, processes):
pwr_map = ps.PowerSpectrum(self.params["p_map"])
pwr_noise = ps.PowerSpectrum(self.params["p_noise"])
pwr_map_plussim = ps.PowerSpectrum(self.params["p_map_plussim"])
pwr_cleaned_sim = ps.PowerSpectrum(self.params["p_cleaned_sim"])
#plussim_basename = self.params["p_map_plussim"].split(".")[0]
#plussim_list = glob.glob("%s_[0-9]*.shelve" % plussim_basename)
#plussim_1d_list = []
#for filename in plussim_list:
# print filename
# pwr_map_plussim = shelve.open(filename, "r")
# plussim_1d_list.append(pe.repackage_1d_power(pwr_map_plussim)[4])
# pwr_map_plussim.close()
#avg_plussim = {}
#for treatment in pwr_cases["treatment"]:
# shape_plussim = plussim_1d_list[0][treatment].shape
# ndim_plussim = len(shape_plussim)
# num_plussim = len(plussim_1d_list)
# shape_avg = shape_plussim + (num_plussim,)
# avg_treatment = np.zeros(shape_avg)
# for (plussim_realization, ind) in \
# zip(plussim_1d_list, range(num_plussim)):
# avg_treatment[..., ind] = plussim_realization[treatment]
# avg_plussim[treatment] = np.mean(avg_treatment, axis=ndim_plussim)
# print avg_plussim[treatment].shape, shape_plussim
# TODOi!!!!!: is this a proper function of treatment?
if self.params["apply_2d_transfer"] is not None:
# copy the same beam transfer function for all cases
trans_shelve = shelve.open(self.params["apply_2d_transfer"])
transfer_2d = trans_shelve["transfer_2d"]
trans_shelve.close()
transfer_dict = {}
for treatment in pwr_map.treatment_cases:
transfer_dict[treatment] = transfer_2d
pwr_map.apply_2d_trans_by_treatment(transfer_dict)
pwr_map_plussim.apply_2d_trans_by_treatment(transfer_dict)
pwr_cleaned_sim.apply_2d_trans_by_treatment(transfer_dict)
# combine AxA, BxB, CxC, DxD into a noise bias estimate
noise2d_agg = pwr_noise.agg_stat_2d_pwrspec()
# also combine the AxB, etc. into a signal piece that is subtracted
signal2d_agg = pwr_map.agg_stat_2d_pwrspec()
weights_2d = {}
# weight by the variance as determined in the mock runs
for treatment in pwr_map.treatment_cases:
weight = noise2d_agg[treatment]["mean"] - \
signal2d_agg[treatment]["mean"]
print weight
weights_2d[treatment] = weight * weight
if self.params["use_noiseweights_2dto1d"]:
pwr_map.convert_2d_to_1d(weights_2d=weights_2d)
pwr_map_plussim.convert_2d_to_1d(weights_2d=weights_2d)
pwr_cleaned_sim.convert_2d_to_1d(weights_2d=weights_2d)
else:
pwr_map.convert_2d_to_1d()
pwr_map_plussim.convert_2d_to_1d()
pwr_cleaned_sim.convert_2d_to_1d()
pwr_map_summary = pwr_map.agg_stat_1d_pwrspec(from_2d=True)
pwr_map_plussim_summary = \
pwr_map_plussim.agg_stat_1d_pwrspec(from_2d=True)
pwr_cleaned_sim_summary = \
pwr_cleaned_sim.agg_stat_1d_pwrspec(from_2d=True)
reference_pwr = pwr_cleaned_sim_summary["0modes"]["mean"]
k_vec = pwr_map.k_1d_from_2d["center"]
for treatment in pwr_map.treatment_cases:
mean_map_plussim = pwr_map_plussim_summary[treatment]["mean"]
mean_map = pwr_map_summary[treatment]["mean"]
std_map = pwr_map_summary[treatment]["std"]
trans = (mean_map_plussim - mean_map) / reference_pwr
#mean_map_plussim = np.mean(avg_plussim[treatment], axis=1)
##trans = (pwr_map_plussim_1d[treatment]-pwr_map_1d[treatment]) / \
## pwr_cleaned_sim_1d["0modes"]
#trans = (avg_plussim[treatment]-pwr_map_1d[treatment]) / \
# pwr_cleaned_sim_1d["0modes"]
#corrected_pwr = pwr_map_1d[treatment]/trans
#trans = np.mean(trans, axis=1)
#mean_map = np.mean(corrected_pwr, axis=1)
#std_map = np.mean(corrected_pwr, axis=1)
outfile = "%s/pk_%s.dat" % (self.params["outdir"], treatment)
file_tools.print_multicolumn(k_vec,
reference_pwr,
trans,
mean_map,
std_map,
outfile=outfile)
print "writing to " + outfile
from quadratic_products import power_spectrum
root = "/mnt/raid-project/gmrt/eswitzer/GBT/pwrspec/GBT_15hr_map_oldcal_x_WiggleZ_15hr_blackman_order1/"
datafile = root + "GBT_15hr_map_oldcal_x_WiggleZ_15hr_data.shelve"
mockfile = root + "GBT_15hr_map_oldcal_x_WiggleZ_15hr_mock.shelve"
dataobj = power_spectrum.PowerSpectrum(datafile)
mockobj = power_spectrum.PowerSpectrum(mockfile)
dataobj.summarize_2d_agg_pwrspec("20modes", "xspec_data.dat")
mockobj.summarize_2d_agg_pwrspec("20modes", "xspec_mock.dat")
def execute(self, processes):
pwr_map = ps.PowerSpectrum(self.params["p_map"])
transfer_dict = pe.load_transferfunc(
self.params["apply_2d_beamtransfer"],
self.params["apply_2d_modetransfer"], pwr_map.treatment_cases)
pwr_map.apply_2d_trans_by_treatment(transfer_dict)
# also combine the AxB, etc. into a signal piece that is subtracted
signal2d_agg = pwr_map.agg_stat_2d_pwrspec()
kx = pwr_map.kx_2d["center"]
ky = pwr_map.ky_2d["center"]
logkx = np.log10(kx)
logky = np.log10(ky)
if self.params["kpar_range"] is not None:
print "restricting k_par to ", self.params["kpar_range"]
restrict_par = np.where(
np.logical_or(ky < self.params["kpar_range"][0],
ky > self.params["kpar_range"][1]))
else:
restrict_par = None
if self.params["kperp_range"] is not None:
print "restricting k_perp to ", self.params["kperp_range"]
restrict_perp = np.where(
np.logical_or(kx < self.params["kperp_range"][0],
kx > self.params["kperp_range"][1]))
else:
restrict_perp = None
for treatment in pwr_map.treatment_cases:
# comb is used for autopower AxB etc.; here just use a placeholder
comb = pwr_map.comb_cases[0]
pwrcase = "%s:%s" % (comb, treatment)
outplot_power_file = "%s/power_2d_%s.png" % \
(self.params['outdir'], treatment)
plot_slice.simpleplot_2D(outplot_power_file,
np.abs(pwr_map.pwrspec_2d[pwrcase]),
logkx,
logky, ["logkx", "logky"],
1.,
"2d power",
"log(abs(2d power))",
logscale=True)
outplot_transfer_file = "%s/transfer_2d_%s.png" % \
(self.params['outdir'], treatment)
if transfer_dict is not None:
plot_slice.simpleplot_2D(outplot_transfer_file,
transfer_dict[treatment],
logkx,
logky, ["logkx", "logky"],
1.,
"2d trans",
"2d trans",
logscale=False)
else:
plot_slice.simpleplot_2D(outplot_transfer_file,
np.ones_like(
pwr_map.counts_2d[pwrcase]),
logkx,
logky, ["logkx", "logky"],
1.,
"2d trans",
"2d trans",
logscale=False)
outplot_count_file = "%s/countweight_2d_%s.png" % \
(self.params['outdir'], treatment)
plot_slice.simpleplot_2D(outplot_count_file,
np.abs(pwr_map.counts_2d[pwrcase]),
logkx,
logky, ["logkx", "logky"],
1.,
"Log-counts",
"log-counts",
logscale=True)
if self.params["noiseweights_2dto1d"] is not None:
print "applying 2D noise weights: " + \
self.params["noiseweights_2dto1d"]
weightfile = h5py.File(self.params["noiseweights_2dto1d"], "r")
weights_2d = {}
for treatment in pwr_map.treatment_cases:
weights_2d[treatment] = weightfile[treatment].value
if restrict_perp is not None:
weights_2d[treatment][restrict_perp, :] = 0.
if restrict_par is not None:
weights_2d[treatment][:, restrict_par] = 0.
outplot_weight_file = "%s/noiseweight_2d_%s.png" % \
(self.params['outdir'], treatment)
plot_slice.simpleplot_2D(outplot_weight_file,
np.abs(weights_2d[treatment]),
logkx,
logky, ["logkx", "logky"],
1.,
"Log-weight",
"log-weight",
logscale=True)
weightfile.close()
pwr_map.convert_2d_to_1d(weights_2d=weights_2d)
else:
pwr_map.convert_2d_to_1d()
# open the output hd5 and attach directories for each treatment
summary_struct = h5py.File(self.params["summaryfile"], "w")
summary_by_treatment = {}
for treatment in pwr_map.treatment_cases:
summary_by_treatment[treatment] = \
summary_struct.create_group(treatment)
pwr_map_summary = pwr_map.agg_stat_1d_pwrspec(from_2d=True)
field_list = ["mean", "std", "gauss_std", "cov", "corr"]
for treatment in pwr_map.treatment_cases:
# copy data from the summary to the output hd5
for field in field_list:
summary_by_treatment[treatment][field] = \
pwr_map_summary[treatment][field]
summary_by_treatment[treatment]["k_vec_left"] = \
pwr_map.k_1d_from_2d["left"]
summary_by_treatment[treatment]["k_vec"] = \
pwr_map.k_1d_from_2d["center"]
summary_by_treatment[treatment]["k_vec_right"] = \
pwr_map.k_1d_from_2d["right"]
outfile = "%s/pk_%s.dat" % (self.params["outdir"], treatment)
file_tools.print_multicolumn(pwr_map.k_1d_from_2d["center"],
pwr_map_summary[treatment]["mean"],
pwr_map_summary[treatment]["std"],
outfile=outfile)
print "writing to " + outfile
summary_struct.close()