def combination_array_2d(self, counts=False, debug=None):
r"""pack the various pair combinations for each treatment into an array
"""
summary_treatment = {}
for treatment in self.treatment_cases:
pwr_treatment = np.zeros((self.num_kx, self.num_ky,
self.num_comb))
for comb, comb_index in zip(self.comb_cases, range(self.num_comb)):
pwrcase = "%s:%s" % (comb, treatment)
if counts:
pwr_treatment[:, :, comb_index] = self.counts_2d[pwrcase]
if debug is not None:
outplot_power_file = "%s/counts_%s_%s.png" % \
(debug, treatment, comb)
else:
pwr_treatment[:, :, comb_index] = self.pwrspec_2d[pwrcase]
if debug is not None:
outplot_power_file = "%s/power_%s_%s.png" % \
(debug, treatment, comb)
if (debug is not None) and (treatment == "20modes"):
plot_slice.simpleplot_2D(outplot_power_file,
np.abs(pwr_treatment[:, :, comb_index]),
np.log10(self.kx_2d['center']),
np.log10(self.ky_2d['center']),
["logkx", "logky"], 1., "2d array",
"log(abs(array))", logscale=True)
summary_treatment[treatment] = pwr_treatment
return summary_treatment
def aggregate_1d_statistics(self, treatment, from2d=False):
r"""Find the mean, std, cov, corr etc of 1D p(k)'s"""
results = self.summary["results"][treatment]
if from2d:
k_1d = self.summary["k_1d_from_2d"]
trial_array_1d = results["pk_1d_from_2d"].value
error_array_1d = results["pkstd_1d_from_2d"].value
counts_array_1d = results["counts_1d_from_2d"].value
outfile = "%s/power_1d_from_2d_%s.dat" % \
(self.params['outplotdir'], treatment)
else:
k_1d = self.summary["k_1d"]
trial_array_1d = results["pk_1d"].value
error_array_1d = results["pkstd_1d"].value
counts_array_1d = results["counts_1d"].value
outfile = "%s/power_1d_%s.dat" % \
(self.params['outplotdir'], treatment)
stat_1d = self.calc_stat_1d(trial_array_1d)
error_stat_1d = self.calc_stat_1d(error_array_1d)
counts_1d = self.calc_stat_1d(counts_array_1d)
file_tools.print_multicolumn(k_1d["left"].value,
k_1d["center"].value,
k_1d["right"].value,
counts_1d["mean"],
stat_1d["mean"],
stat_1d["std"],
error_stat_1d["mean"],
outfile=outfile)
logk_1d = np.log10(k_1d['center'])
if self.make_plot:
outplot_file = "%s/sim_corr_1d_%s.png" % \
(self.params['outplotdir'], treatment)
plot_slice.simpleplot_2D(outplot_file, stat_1d['corr'],
logk_1d, logk_1d,
["logk", "logk"], 1.,
"1D corr", "corr")
outplot_file = "%s/sim_cov_1d_%s.png" % \
(self.params['outplotdir'], treatment)
plot_slice.simpleplot_2D(outplot_file, stat_1d['cov'],
logk_1d, logk_1d,
["logk", "logk"], 1.,
"1D covariance", "cov")
return (stat_1d, error_stat_1d, counts_1d)
def aggregate_1d_statistics(self, treatment, from2d=False):
r"""Find the mean, std, cov, corr etc of 1D p(k)'s"""
results = self.summary["results"][treatment]
if from2d:
k_1d = self.summary["k_1d_from_2d"]
trial_array_1d = results["pk_1d_from_2d"].value
error_array_1d = results["pkstd_1d_from_2d"].value
counts_array_1d = results["counts_1d_from_2d"].value
outfile = "%s/power_1d_from_2d_%s.dat" % \
(self.params['outplotdir'], treatment)
else:
k_1d = self.summary["k_1d"]
trial_array_1d = results["pk_1d"].value
error_array_1d = results["pkstd_1d"].value
counts_array_1d = results["counts_1d"].value
outfile = "%s/power_1d_%s.dat" % \
(self.params['outplotdir'], treatment)
stat_1d = self.calc_stat_1d(trial_array_1d)
error_stat_1d = self.calc_stat_1d(error_array_1d)
counts_1d = self.calc_stat_1d(counts_array_1d)
file_tools.print_multicolumn(k_1d["left"].value,
k_1d["center"].value,
k_1d["right"].value,
counts_1d["mean"],
stat_1d["mean"],
stat_1d["std"],
error_stat_1d["mean"],
outfile=outfile)
logk_1d = np.log10(k_1d['center'])
if self.make_plot:
outplot_file = "%s/sim_corr_1d_%s.png" % \
(self.params['outplotdir'], treatment)
plot_slice.simpleplot_2D(outplot_file, stat_1d['corr'], logk_1d,
logk_1d, ["logk", "logk"], 1., "1D corr",
"corr")
outplot_file = "%s/sim_cov_1d_%s.png" % \
(self.params['outplotdir'], treatment)
plot_slice.simpleplot_2D(outplot_file, stat_1d['cov'], logk_1d,
logk_1d, ["logk", "logk"], 1.,
"1D covariance", "cov")
return (stat_1d, error_stat_1d, counts_1d)
def execute(self, processes):
"""produce a list of files to combine and run"""
transferfile = h5py.File(self.params["transferfile"], "w")
inkey = self.treatments_in[0]
for treatment in self.treatments_out:
stats_2d_in = self.stats_in[inkey]["pk_2d_stat"]["mean"].value
stats_2d_out = self.stats_out[treatment]["pk_2d_stat"][
"mean"].value
counts_2d_in = self.stats_in[inkey]["pk_2d_counts"]["mean"].value
counts_2d_out = self.stats_out[treatment]["pk_2d_counts"][
"mean"].value
counts_prod = counts_2d_in * counts_2d_out
transfer_2d = stats_2d_out / (stats_2d_in * self.multiplier)
transfer_2d[counts_prod == 0] = 0.
if self.params["regulate_transfer"]:
transfer_2d[transfer_2d < 0.] = 0.
transfer_2d[transfer_2d > 1.] = 1.
if self.params["square_transfer"]:
transfer_2d *= transfer_2d
transferfile[treatment] = transfer_2d
transfer_2d_plot = copy.deepcopy(transfer_2d)
transfer_2d_plot[transfer_2d_plot < 0.] = 0.
transfer_2d_plot[transfer_2d_plot > 1.] = 1.
outplot_file = "%s/transfer_2d_%s.png" % \
(self.params['outplotdir'], treatment)
print "plotting trans to ", outplot_file
logkx = np.log10(self.stats_in[inkey]["kx_2d"].value)
logky = np.log10(self.stats_in[inkey]["ky_2d"].value)
plot_slice.simpleplot_2D(outplot_file, transfer_2d_plot, logkx,
logky, ["logkx", "logky"], 1.,
"2D beam transfer", "T")
transferfile.close()
self.stats_in.close()
self.stats_out.close()
def execute(self, processes):
"""produce a list of files to combine and run"""
transferfile = h5py.File(self.params["transferfile"], "w")
inkey = self.treatments_in[0]
for treatment in self.treatments_out:
stats_2d_in = self.stats_in[inkey]["pk_2d_stat"]["mean"].value
stats_2d_out = self.stats_out[treatment]["pk_2d_stat"]["mean"].value
counts_2d_in = self.stats_in[inkey]["pk_2d_counts"]["mean"].value
counts_2d_out = self.stats_out[treatment]["pk_2d_counts"]["mean"].value
counts_prod = counts_2d_in * counts_2d_out
transfer_2d = stats_2d_out / (stats_2d_in * self.multiplier)
transfer_2d[counts_prod == 0] = 0.
if self.params["regulate_transfer"]:
transfer_2d[transfer_2d < 0.] = 0.
transfer_2d[transfer_2d > 1.] = 1.
if self.params["square_transfer"]:
transfer_2d *= transfer_2d
transferfile[treatment] = transfer_2d
transfer_2d_plot = copy.deepcopy(transfer_2d)
transfer_2d_plot[transfer_2d_plot < 0.] = 0.
transfer_2d_plot[transfer_2d_plot > 1.] = 1.
outplot_file = "%s/transfer_2d_%s.png" % \
(self.params['outplotdir'], treatment)
print "plotting trans to ", outplot_file
logkx = np.log10(self.stats_in[inkey]["kx_2d"].value)
logky = np.log10(self.stats_in[inkey]["ky_2d"].value)
plot_slice.simpleplot_2D(outplot_file, transfer_2d_plot,
logkx, logky,
["logkx", "logky"], 1.,
"2D beam transfer", "T")
transferfile.close()
self.stats_in.close()
self.stats_out.close()
def aggregate_2d_statistics(self, treatment):
kx_2d = self.summary["kx_2d"]
ky_2d = self.summary["ky_2d"]
trial_array_2d = self.summary["results"][treatment]["pk_2d"]
counts_array_2d = self.summary["results"][treatment]["counts_2d"]
outfile = "%s/power_2d_%s.dat" % \
(self.params['outplotdir'], treatment)
stat_2d = self.calc_stat_2d(trial_array_2d)
counts_2d = self.calc_stat_2d(counts_array_2d)
#mean2d[np.isnan(mean2d)] = 0.
#mean2d[np.isnan(mean2d)] = 1.e-16
np.set_printoptions(threshold='nan')
logkx = np.log10(kx_2d['center'])
logky = np.log10(ky_2d['center'])
if self.make_plot:
outplot_file = "%s/sim_mean_2d_%s.png" % \
(self.params['outplotdir'], treatment)
plot_slice.simpleplot_2D(outplot_file,
stat_2d['mean'],
logkx,
logky, ["logkx", "logky"],
1.,
"2D power",
"logP(k)",
logscale=False)
# can use C or F to do column or row-major
#outplot_file = "%s/sim_corr_2d_%s.png" % \
# (self.params['outplotdir'], treatment)
#plot_slice.simpleplot_2D(outplot_file, stat_2d['corr'],
# stat_2d['flat_axis'],
# stat_2d['flat_axis'],
# ["k", "k"], 1.,
# "2D power corr", "corr")
return (stat_2d, counts_2d)
def aggregate_2d_statistics(self, treatment):
kx_2d = self.summary["kx_2d"]
ky_2d = self.summary["ky_2d"]
trial_array_2d = self.summary["results"][treatment]["pk_2d"]
counts_array_2d = self.summary["results"][treatment]["counts_2d"]
outfile = "%s/power_2d_%s.dat" % \
(self.params['outplotdir'], treatment)
stat_2d = self.calc_stat_2d(trial_array_2d)
counts_2d = self.calc_stat_2d(counts_array_2d)
#mean2d[np.isnan(mean2d)] = 0.
#mean2d[np.isnan(mean2d)] = 1.e-16
np.set_printoptions(threshold='nan')
logkx = np.log10(kx_2d['center'])
logky = np.log10(ky_2d['center'])
if self.make_plot:
outplot_file = "%s/sim_mean_2d_%s.png" % \
(self.params['outplotdir'], treatment)
plot_slice.simpleplot_2D(outplot_file, stat_2d['mean'],
logkx, logky,
["logkx", "logky"], 1.,
"2D power", "logP(k)",
logscale=False)
# can use C or F to do column or row-major
#outplot_file = "%s/sim_corr_2d_%s.png" % \
# (self.params['outplotdir'], treatment)
#plot_slice.simpleplot_2D(outplot_file, stat_2d['corr'],
# stat_2d['flat_axis'],
# stat_2d['flat_axis'],
# ["k", "k"], 1.,
# "2D power corr", "corr")
return (stat_2d, counts_2d)
def execute(self, processes):
pwr_sim = ps.PowerSpectrum(self.params["p_sim"])
pwr_mock = ps.PowerSpectrum(self.params["p_mock"])
modetransfer_2d = None
beamtransfer_2d = None
if (self.params["apply_2d_beamtransfer"] is not None) or \
(self.params["apply_2d_modetransfer"] is not None):
if self.params["apply_2d_modetransfer"] is not None:
print "Applying 2d transfer from " + \
self.params["apply_2d_modetransfer"]
modetransfer_2d = h5py.File(
self.params["apply_2d_modetransfer"], "r")
if self.params["apply_2d_beamtransfer"] is not None:
print "Applying 2d transfer from " + \
self.params["apply_2d_beamtransfer"]
beamtransfer_2d = h5py.File(
self.params["apply_2d_beamtransfer"], "r")
transfer_dict = {}
for treatment in pwr_data.treatment_cases:
if modetransfer_2d is not None:
transfer_dict[treatment] = modetransfer_2d[treatment].value
if beamtransfer_2d is not None:
transfer_dict[treatment] *= \
beamtransfer_2d["0modes"].value
else:
transfer_dict[treatment] = beamtransfer_2d["0modes"].value
if (modetransfer_2d is not None) or (beamtransfer_2d is not None):
pwr_mock.apply_2d_trans_by_treatment(transfer_dict)
pwr_data.apply_2d_trans_by_treatment(transfer_dict)
# gather the rms of each 2D k-bin over the realizations of mock
# catalogs; this is used for the 2D->1D weighting
#mock2d_agg = pwr_mock.agg_stat_2d_pwrspec(debug="./plotheap/")
mock2d_agg = pwr_mock.agg_stat_2d_pwrspec()
weights_2d = {}
# weight by the variance as determined in the mock runs
for treatment in pwr_mock.treatment_cases:
weight_for_treatment = 1. / (mock2d_agg[treatment]["std"] * \
mock2d_agg[treatment]["std"])
kx = pwr_mock.kx_2d['center']
ky = pwr_mock.ky_2d['center']
kmin_x = self.params['kmin_xy'][0]
kmin_y = self.params['kmin_xy'][1]
if kmin_x is not None:
restrict = np.where(kx < kmin_x)
weight_for_treatment[restrict, :] = 0.
if kmin_y is not None:
restrict = np.where(ky < kmin_y)
weight_for_treatment[:, restrict] = 0.
weight_for_treatment[np.isnan(weight_for_treatment)] = 0.
weight_for_treatment[np.isinf(weight_for_treatment)] = 0.
weights_2d[treatment] = weight_for_treatment
outplot_power_file = "%s/power_2d_%s.png" % \
(self.params['outdir'], treatment)
outplot_transfer_file = "%s/transfer_2d_%s.png" % \
(self.params['outdir'], treatment)
outplot_weight_file = "%s/noiseweight_2d_%s.png" % \
(self.params['outdir'], treatment)
outplot_count_file = "%s/countweight_2d_%s.png" % \
(self.params['outdir'], treatment)
# comb is used for autopower AxB etc.; here just use a placeholder
logkx = np.log10(pwr_mock.kx_2d['center'])
logky = np.log10(pwr_mock.ky_2d['center'])
comb = pwr_data.comb_cases[0]
pwrcase = "%s:%s" % (comb, treatment)
plot_slice.simpleplot_2D(outplot_power_file,
np.abs(pwr_data.pwrspec_2d[pwrcase]),
logkx, logky,
["logkx", "logky"], 1., "2d power",
"log(abs(2d power))", logscale=True)
if (modetransfer_2d is not None) or (beamtransfer_2d is not None):
plot_slice.simpleplot_2D(outplot_transfer_file,
transfer_dict[treatment],
logkx, logky,
["logkx", "logky"], 1., "2d trans",
"2d trans", logscale=False)
#if treatment == "20modes":
# np.set_printoptions(threshold=np.nan)
# print mock2d_agg[treatment]["std"]
# print np.abs(weights_2d[treatment])
plot_slice.simpleplot_2D(outplot_weight_file,
np.abs(weights_2d[treatment]),
logkx, logky,
["logkx", "logky"], 1., "Log-weight",
"log-weight", logscale=True)
comb = pwr_mock.comb_cases[0]
pwrcase = "%s:%s" % (comb, treatment)
plot_slice.simpleplot_2D(outplot_count_file,
np.abs(pwr_mock.counts_2d[pwrcase]),
logkx, logky,
["logkx", "logky"], 1., "Log-counts",
"log-counts", logscale=True)
if self.params["use_noiseweights_2dto1d"]:
pwr_data.convert_2d_to_1d(weights_2d=weights_2d)
pwr_mock.convert_2d_to_1d(weights_2d=weights_2d)
else:
# use the counts to weight instead by default
# just be sure the transfer function is applied
pwr_data.convert_2d_to_1d()
pwr_mock.convert_2d_to_1d()
pwr_data_summary = pwr_data.agg_stat_1d_pwrspec(from_2d=True)
pwr_mock_summary = pwr_mock.agg_stat_1d_pwrspec(from_2d=True)
# build the hd5 summary file
sum_filename = "%s/cross_summary.hd5" % self.params["outdir"]
summaryfile = h5py.File(sum_filename, "w")
power_1d_out = summaryfile.create_group("power_1d")
power_1d_cov_out = summaryfile.create_group("power_1d_cov")
kvec_out = summaryfile.create_group("kvec")
k_vec = pwr_data.k_1d_from_2d["center"]
kvec_out["k_1d_left"] = pwr_data.k_1d_from_2d["left"]
kvec_out["k_1d_center"] = pwr_data.k_1d_from_2d["center"]
kvec_out["k_1d_right"] = pwr_data.k_1d_from_2d["right"]
for treatment in pwr_data.treatment_cases:
mean_data = pwr_data_summary[treatment]["mean"]
mean_mock = pwr_mock_summary[treatment]["mean"]
std_mock = pwr_mock_summary[treatment]["std"]
cov_mock = pwr_mock_summary[treatment]["cov"]
corr_mock = pwr_mock_summary[treatment]["corr"]
power_1d_out[treatment] = mean_data
power_1d_cov_out[treatment] = cov_mock
logk = np.log10(k_vec)
outplot_corr_file = "%s/covariance_1d_%s.png" % \
(self.params['outdir'], treatment)
plot_slice.simpleplot_2D(outplot_corr_file,
corr_mock,
logk, logk,
["log k", "log k"], 1., "1d corr",
"1d corr", logscale=False)
outfile = "%s/pk_%s.dat" % (self.params["outdir"], treatment)
file_tools.print_multicolumn(k_vec, mean_data, mean_mock, std_mock,
outfile=outfile)
print "writing to " + outfile
summaryfile.close()
def execute(self, processes):
#sim_auto_stats = self.sim_auto['stats']['0modes']['pk_1d_from_2d_stat']
#sim_xspec_stats = self.sim_xspec['stats']['0modes']['pk_1d_from_2d_stat']
sim_auto_stats = self.sim_auto['0modes']['pk_1d_from_2d_stat']
sim_xspec_stats = self.sim_xspec['0modes']['pk_1d_from_2d_stat']
sim_xspec_mean = sim_xspec_stats['mean'].value
sim_xspec_cov = sim_xspec_stats['cov'].value
sim_auto_mean = sim_auto_stats['mean'].value
#k_vec_sim = self.sim_auto['k_1d_from_2d']['center']
k_vec_sim = self.sim_auto['0modes']['k_1d_from_2d'].value
k_vec = self.data_xspec['0modes']['k_vec'].value
assert np.array_equal(k_vec_sim, k_vec), "simulation and data unaligned kvecs"
log_k_vec = np.log10(k_vec)
for treatment in self.data_xspec:
est_data_cov = np.zeros_like(sim_xspec_cov)
data_xspec_mean = self.data_xspec[treatment]['mean'].value
data_auto_mean = self.data_auto[treatment]['mean'].value
data_xspec_std = self.data_xspec[treatment]['std'].value
data_auto_std = self.data_auto[treatment]['std'].value
print treatment
outfile = "%s/pk_terms_%s.dat" % (self.params["outdir"], treatment)
file_tools.print_multicolumn(k_vec, data_xspec_mean,
data_xspec_std, data_auto_mean,
data_auto_std, sim_xspec_mean,
sim_auto_mean,
outfile=outfile)
data_cov_fac = 7. / 6. * data_xspec_mean * data_xspec_mean
data_cov_fac += 2. / 3. * data_xspec_mean * data_auto_mean
data_cov_fac += 1. / 6. * data_auto_mean * data_auto_mean
sim_cov_fac = 7. / 6. * sim_xspec_mean * sim_xspec_mean
sim_cov_fac += 2. / 3. * sim_xspec_mean * sim_auto_mean
sim_cov_fac += 1. / 6. * sim_auto_mean * sim_auto_mean
cov_fac_ratio = data_cov_fac / sim_cov_fac
error_multiplier = np.outer(np.sqrt(cov_fac_ratio),
np.sqrt(cov_fac_ratio))
est_data_cov = sim_xspec_cov * error_multiplier
np.set_printoptions(threshold=np.nan)
print est_data_cov
outfile = "%s/cov_fac_%s.dat" % (self.params["outdir"], treatment)
file_tools.print_multicolumn(k_vec, data_cov_fac, sim_cov_fac,
outfile=outfile)
outplot_cov_file = "%s/covariance_%s.png" % \
(self.params['outdir'], treatment)
plot_slice.simpleplot_2D(outplot_cov_file, est_data_cov,
log_k_vec, log_k_vec,
["logk", "logk"], 1.,
"1D covariance", "cov")
new_data_std = np.sqrt(np.diag(est_data_cov))
outfile = "%s/pk_%s.dat" % (self.params["outdir"], treatment)
file_tools.print_multicolumn(k_vec, data_xspec_mean,
data_xspec_std, new_data_std,
outfile=outfile)
def load_mixing_matrix(filename, outplot_tag):
r"""load a mixing matrix produced by 'calculate_mixing'
The goal is to unmixing the 2D k-bins, but many of the k-bins have zero
counts, e.g. no bins in 3d k-space for that 2d bin. The inversion is
therefore performed on a subset of the indices.
"""
mixing_shelve = shelve.open(filename, "r")
# first find the number of 2D k bins that have more than 0 modes
counts_reference = mixing_shelve['results'][0][1].flatten()
counts_nonzero = (counts_reference != 0)
n_nonzero_counts = np.sum(counts_nonzero)
where_nonzero_counts = np.where(counts_nonzero)[0]
# alternately: (these should be the same)
bins3d_nonzero = []
for results_entry in mixing_shelve['results']:
kindex = results_entry[0]
bins3d_nonzero.append(kindex)
# these are produced in a batch which is not necessarily ordered
bins3d_nonzero.sort()
bins3d_trans = {}
nonzero_index = 0
for full_index in bins3d_nonzero:
bins3d_trans[repr(full_index)] = nonzero_index
nonzero_index += 1
n_nonzero_bins3d = len(bins3d_nonzero)
print bins3d_nonzero
print where_nonzero_counts
assert np.array_equal(bins3d_nonzero, where_nonzero_counts), \
"load_mixing_matrix: unmatching bins"
mixing_array = np.zeros((n_nonzero_counts, n_nonzero_counts))
counts_array = np.zeros((n_nonzero_counts, n_nonzero_counts))
for result_entry in mixing_shelve['results']:
kindex = result_entry[0]
if kindex in bins3d_nonzero:
counts_vec = result_entry[1].flatten()
mixing_vec = result_entry[2].flatten()
counts_vec_nonzero = counts_vec[bins3d_nonzero]
mixing_vec_nonzero = mixing_vec[bins3d_nonzero]
nonzero_index = bins3d_trans[repr(kindex)]
counts_array[nonzero_index, :] = counts_vec_nonzero
mixing_array[nonzero_index, :] = mixing_vec_nonzero
mixing_shelve.close()
#masked_mixing = ma.masked_invalid(mixing_array)
zeroed_mixing = copy.deepcopy(mixing_array)
zeroed_mixing[np.isnan(zeroed_mixing)] = 0.
zeroed_mixing[np.isinf(zeroed_mixing)] = 0.
#print zeroed_mixing
inv_mixing = LA.pinv(zeroed_mixing)
filename = "mixing_%s.png" % outplot_tag
plot_slice.simpleplot_2D(filename, zeroed_mixing,
range(n_nonzero_bins3d), range(n_nonzero_bins3d),
["X", "Y"], 1., "2D mixing", "mixing", logscale=False)
print np.diag(inv_mixing)
filename = "inv_mixing_%s.png" % outplot_tag
#inv_mixing[inv_mixing < 0.] = -np.sqrt(-inv_mixing[inv_mixing < 0.])
#inv_mixing[inv_mixing > 0.] = np.sqrt(inv_mixing[inv_mixing > 0.])
plot_slice.simpleplot_2D(filename, inv_mixing,
range(n_nonzero_bins3d), range(n_nonzero_bins3d),
["X", "Y"], 1., "2D mixing", "mixing", logscale=False)
filename = "counts_%s.png" % outplot_tag
plot_slice.simpleplot_2D(filename, counts_array,
range(n_nonzero_bins3d), range(n_nonzero_bins3d),
["X", "Y"], 1., "2D counts", "counts", logscale=False)
return {'params': mixing_shelve['params'],
'kflat': mixing_shelve['kflat'],
'counts_reference': counts_reference,
'bins3d_nonzero': bins3d_nonzero,
'counts_array': counts_array,
'mixing_array': mixing_array,
'inv_mixing': inv_mixing}
def execute(self, processes):
print "using data: ", self.params["p_data"]
print "using mock: ", self.params["p_mock"]
pwr_data = ps.PowerSpectrum(self.params["p_data"])
pwr_mock = ps.PowerSpectrum(self.params["p_mock"])
transfer_dict = pe.load_transferfunc(
self.params["apply_2d_beamtransfer"],
self.params["apply_2d_modetransfer"],
pwr_mock.treatment_cases)
if transfer_dict is not None:
pwr_mock.apply_2d_trans_by_treatment(transfer_dict)
pwr_data.apply_2d_trans_by_treatment(transfer_dict)
# gather the rms of each 2D k-bin over the realizations of mock
# catalogs; this is used for the 2D->1D weighting
#mock2d_agg = pwr_mock.agg_stat_2d_pwrspec(debug="./plotheap/")
mock2d_agg = pwr_mock.agg_stat_2d_pwrspec()
weights_2d = {}
# weight by the variance as determined in the mock runs
for treatment in pwr_mock.treatment_cases:
weight_for_treatment = 1. / (mock2d_agg[treatment]["std"] * \
mock2d_agg[treatment]["std"])
kx = pwr_mock.kx_2d['center']
ky = pwr_mock.ky_2d['center']
kmin_x = self.params['kmin_xy'][0]
kmin_y = self.params['kmin_xy'][1]
if kmin_x is not None:
restrict = np.where(kx < kmin_x)
weight_for_treatment[restrict, :] = 0.
if kmin_y is not None:
restrict = np.where(ky < kmin_y)
weight_for_treatment[:, restrict] = 0.
weight_for_treatment[np.isnan(weight_for_treatment)] = 0.
weight_for_treatment[np.isinf(weight_for_treatment)] = 0.
weights_2d[treatment] = weight_for_treatment
outplot_power_file = "%s/power_2d_%s.png" % \
(self.params['outdir'], treatment)
outplot_transfer_file = "%s/transfer_2d_%s.png" % \
(self.params['outdir'], treatment)
outplot_weight_file = "%s/noiseweight_2d_%s.png" % \
(self.params['outdir'], treatment)
outplot_count_file = "%s/countweight_2d_%s.png" % \
(self.params['outdir'], treatment)
# comb is used for autopower AxB etc.; here just use a placeholder
logkx = np.log10(pwr_mock.kx_2d['center'])
logky = np.log10(pwr_mock.ky_2d['center'])
comb = pwr_data.comb_cases[0]
pwrcase = "%s:%s" % (comb, treatment)
plot_slice.simpleplot_2D(outplot_power_file,
np.abs(pwr_data.pwrspec_2d[pwrcase]),
logkx, logky,
["logkx", "logky"], 1., "2d power",
"log(abs(2d power))", logscale=True)
if (modetransfer_2d is not None) or (beamtransfer_2d is not None):
plot_slice.simpleplot_2D(outplot_transfer_file,
transfer_dict[treatment],
logkx, logky,
["logkx", "logky"], 1., "2d trans",
"2d trans", logscale=False)
#if treatment == "20modes":
# np.set_printoptions(threshold=np.nan)
# print mock2d_agg[treatment]["std"]
# print np.abs(weights_2d[treatment])
plot_slice.simpleplot_2D(outplot_weight_file,
np.abs(weights_2d[treatment]),
logkx, logky,
["logkx", "logky"], 1., "Log-weight",
"log-weight", logscale=True)
comb = pwr_mock.comb_cases[0]
pwrcase = "%s:%s" % (comb, treatment)
plot_slice.simpleplot_2D(outplot_count_file,
np.abs(pwr_mock.counts_2d[pwrcase]),
logkx, logky,
["logkx", "logky"], 1., "Log-counts",
"log-counts", logscale=True)
if self.params["use_noiseweights_2dto1d"]:
pwr_data.convert_2d_to_1d(weights_2d=weights_2d)
pwr_mock.convert_2d_to_1d(weights_2d=weights_2d)
else:
# use the counts to weight instead by default
# just be sure the transfer function is applied
pwr_data.convert_2d_to_1d()
pwr_mock.convert_2d_to_1d()
pwr_data_summary = pwr_data.agg_stat_1d_pwrspec(from_2d=True)
pwr_mock_summary = pwr_mock.agg_stat_1d_pwrspec(from_2d=True)
# build the hd5 summary file
sum_filename = "%s/cross_summary.hd5" % self.params["outdir"]
summaryfile = h5py.File(sum_filename, "w")
power_1d_out = summaryfile.create_group("power_1d")
power_1d_cov_out = summaryfile.create_group("power_1d_cov")
kvec_out = summaryfile.create_group("kvec")
k_vec = pwr_data.k_1d_from_2d["center"]
kvec_out["k_1d_left"] = pwr_data.k_1d_from_2d["left"]
kvec_out["k_1d_center"] = pwr_data.k_1d_from_2d["center"]
kvec_out["k_1d_right"] = pwr_data.k_1d_from_2d["right"]
for treatment in pwr_data.treatment_cases:
mean_data = pwr_data_summary[treatment]["mean"]
mean_mock = pwr_mock_summary[treatment]["mean"]
std_mock = pwr_mock_summary[treatment]["std"]
cov_mock = pwr_mock_summary[treatment]["cov"]
corr_mock = pwr_mock_summary[treatment]["corr"]
power_1d_out[treatment] = mean_data
power_1d_cov_out[treatment] = cov_mock
logk = np.log10(k_vec)
outplot_corr_file = "%s/covariance_1d_%s.png" % \
(self.params['outdir'], treatment)
plot_slice.simpleplot_2D(outplot_corr_file,
corr_mock,
logk, logk,
["log k", "log k"], 1., "1d corr",
"1d corr", logscale=False)
outfile = "%s/pk_%s.dat" % (self.params["outdir"], treatment)
file_tools.print_multicolumn(k_vec, mean_data, mean_mock, std_mock,
outfile=outfile)
print "writing to " + outfile
summaryfile.close()
def execute(self, processes):
pwr_map = ps.PowerSpectrum(self.params["p_map"])
transfer_dict = pe.load_transferfunc(
self.params["apply_2d_beamtransfer"],
self.params["apply_2d_modetransfer"],
pwr_map.treatment_cases)
pwr_map.apply_2d_trans_by_treatment(transfer_dict)
# also combine the AxB, etc. into a signal piece that is subtracted
signal2d_agg = pwr_map.agg_stat_2d_pwrspec()
kx = pwr_map.kx_2d["center"]
ky = pwr_map.ky_2d["center"]
logkx = np.log10(kx)
logky = np.log10(ky)
if self.params["kpar_range"] is not None:
print "restricting k_par to ", self.params["kpar_range"]
restrict_par = np.where(np.logical_or(
ky < self.params["kpar_range"][0],
ky > self.params["kpar_range"][1]))
else:
restrict_par = None
if self.params["kperp_range"] is not None:
print "restricting k_perp to ", self.params["kperp_range"]
restrict_perp = np.where(np.logical_or(
kx < self.params["kperp_range"][0],
kx > self.params["kperp_range"][1]))
else:
restrict_perp = None
for treatment in pwr_map.treatment_cases:
# comb is used for autopower AxB etc.; here just use a placeholder
comb = pwr_map.comb_cases[0]
pwrcase = "%s:%s" % (comb, treatment)
outplot_power_file = "%s/power_2d_%s.png" % \
(self.params['outdir'], treatment)
plot_slice.simpleplot_2D(outplot_power_file,
np.abs(pwr_map.pwrspec_2d[pwrcase]),
logkx, logky,
["logkx", "logky"], 1., "2d power",
"log(abs(2d power))", logscale=True)
outplot_transfer_file = "%s/transfer_2d_%s.png" % \
(self.params['outdir'], treatment)
if transfer_dict is not None:
plot_slice.simpleplot_2D(outplot_transfer_file,
transfer_dict[treatment],
logkx, logky,
["logkx", "logky"], 1., "2d trans",
"2d trans", logscale=False)
else:
plot_slice.simpleplot_2D(outplot_transfer_file,
np.ones_like(pwr_map.counts_2d[pwrcase]),
logkx, logky,
["logkx", "logky"], 1., "2d trans",
"2d trans", logscale=False)
outplot_count_file = "%s/countweight_2d_%s.png" % \
(self.params['outdir'], treatment)
plot_slice.simpleplot_2D(outplot_count_file,
np.abs(pwr_map.counts_2d[pwrcase]),
logkx, logky,
["logkx", "logky"], 1., "Log-counts",
"log-counts", logscale=True)
if self.params["noiseweights_2dto1d"] is not None:
print "applying 2D noise weights: " + \
self.params["noiseweights_2dto1d"]
weightfile = h5py.File(self.params["noiseweights_2dto1d"], "r")
weights_2d = {}
for treatment in pwr_map.treatment_cases:
weights_2d[treatment] = weightfile[treatment].value
if restrict_perp is not None:
weights_2d[treatment][restrict_perp, :] = 0.
if restrict_par is not None:
weights_2d[treatment][:, restrict_par] = 0.
outplot_weight_file = "%s/noiseweight_2d_%s.png" % \
(self.params['outdir'], treatment)
plot_slice.simpleplot_2D(outplot_weight_file,
np.abs(weights_2d[treatment]),
logkx, logky,
["logkx", "logky"], 1., "Log-weight",
"log-weight", logscale=True)
weightfile.close()
pwr_map.convert_2d_to_1d(weights_2d=weights_2d)
else:
pwr_map.convert_2d_to_1d()
# open the output hd5 and attach directories for each treatment
summary_struct = h5py.File(self.params["summaryfile"], "w")
summary_by_treatment = {}
for treatment in pwr_map.treatment_cases:
summary_by_treatment[treatment] = \
summary_struct.create_group(treatment)
pwr_map_summary = pwr_map.agg_stat_1d_pwrspec(from_2d=True)
field_list = ["mean", "std", "gauss_std", "cov", "corr"]
for treatment in pwr_map.treatment_cases:
# copy data from the summary to the output hd5
for field in field_list:
summary_by_treatment[treatment][field] = \
pwr_map_summary[treatment][field]
summary_by_treatment[treatment]["k_vec_left"] = \
pwr_map.k_1d_from_2d["left"]
summary_by_treatment[treatment]["k_vec"] = \
pwr_map.k_1d_from_2d["center"]
summary_by_treatment[treatment]["k_vec_right"] = \
pwr_map.k_1d_from_2d["right"]
outfile = "%s/pk_%s.dat" % (self.params["outdir"], treatment)
file_tools.print_multicolumn(pwr_map.k_1d_from_2d["center"],
pwr_map_summary[treatment]["mean"],
pwr_map_summary[treatment]["std"],
outfile=outfile)
print "writing to " + outfile
summary_struct.close()
def load_mixing_matrix(filename, outplot_tag):
r"""load a mixing matrix produced by 'calculate_mixing'
The goal is to unmixing the 2D k-bins, but many of the k-bins have zero
counts, e.g. no bins in 3d k-space for that 2d bin. The inversion is
therefore performed on a subset of the indices.
"""
mixing_shelve = shelve.open(filename, "r")
# first find the number of 2D k bins that have more than 0 modes
counts_reference = mixing_shelve['results'][0][1].flatten()
counts_nonzero = (counts_reference != 0)
n_nonzero_counts = np.sum(counts_nonzero)
where_nonzero_counts = np.where(counts_nonzero)[0]
# alternately: (these should be the same)
bins3d_nonzero = []
for results_entry in mixing_shelve['results']:
kindex = results_entry[0]
bins3d_nonzero.append(kindex)
# these are produced in a batch which is not necessarily ordered
bins3d_nonzero.sort()
bins3d_trans = {}
nonzero_index = 0
for full_index in bins3d_nonzero:
bins3d_trans[repr(full_index)] = nonzero_index
nonzero_index += 1
n_nonzero_bins3d = len(bins3d_nonzero)
print bins3d_nonzero
print where_nonzero_counts
assert np.array_equal(bins3d_nonzero, where_nonzero_counts), \
"load_mixing_matrix: unmatching bins"
mixing_array = np.zeros((n_nonzero_counts, n_nonzero_counts))
counts_array = np.zeros((n_nonzero_counts, n_nonzero_counts))
for result_entry in mixing_shelve['results']:
kindex = result_entry[0]
if kindex in bins3d_nonzero:
counts_vec = result_entry[1].flatten()
mixing_vec = result_entry[2].flatten()
counts_vec_nonzero = counts_vec[bins3d_nonzero]
mixing_vec_nonzero = mixing_vec[bins3d_nonzero]
nonzero_index = bins3d_trans[repr(kindex)]
counts_array[nonzero_index, :] = counts_vec_nonzero
mixing_array[nonzero_index, :] = mixing_vec_nonzero
mixing_shelve.close()
#masked_mixing = ma.masked_invalid(mixing_array)
zeroed_mixing = copy.deepcopy(mixing_array)
zeroed_mixing[np.isnan(zeroed_mixing)] = 0.
zeroed_mixing[np.isinf(zeroed_mixing)] = 0.
#print zeroed_mixing
inv_mixing = LA.pinv(zeroed_mixing)
filename = "mixing_%s.png" % outplot_tag
plot_slice.simpleplot_2D(filename,
zeroed_mixing,
range(n_nonzero_bins3d),
range(n_nonzero_bins3d), ["X", "Y"],
1.,
"2D mixing",
"mixing",
logscale=False)
print np.diag(inv_mixing)
filename = "inv_mixing_%s.png" % outplot_tag
#inv_mixing[inv_mixing < 0.] = -np.sqrt(-inv_mixing[inv_mixing < 0.])
#inv_mixing[inv_mixing > 0.] = np.sqrt(inv_mixing[inv_mixing > 0.])
plot_slice.simpleplot_2D(filename,
inv_mixing,
range(n_nonzero_bins3d),
range(n_nonzero_bins3d), ["X", "Y"],
1.,
"2D mixing",
"mixing",
logscale=False)
filename = "counts_%s.png" % outplot_tag
plot_slice.simpleplot_2D(filename,
counts_array,
range(n_nonzero_bins3d),
range(n_nonzero_bins3d), ["X", "Y"],
1.,
"2D counts",
"counts",
logscale=False)
return {
'params': mixing_shelve['params'],
'kflat': mixing_shelve['kflat'],
'counts_reference': counts_reference,
'bins3d_nonzero': bins3d_nonzero,
'counts_array': counts_array,
'mixing_array': mixing_array,
'inv_mixing': inv_mixing
}
def execute(self, processes):
pwr_map = ps.PowerSpectrum(self.params["p_map"])
transfer_dict = pe.load_transferfunc(
self.params["apply_2d_beamtransfer"],
self.params["apply_2d_modetransfer"], pwr_map.treatment_cases)
pwr_map.apply_2d_trans_by_treatment(transfer_dict)
# also combine the AxB, etc. into a signal piece that is subtracted
signal2d_agg = pwr_map.agg_stat_2d_pwrspec()
kx = pwr_map.kx_2d["center"]
ky = pwr_map.ky_2d["center"]
logkx = np.log10(kx)
logky = np.log10(ky)
if self.params["kpar_range"] is not None:
print "restricting k_par to ", self.params["kpar_range"]
restrict_par = np.where(
np.logical_or(ky < self.params["kpar_range"][0],
ky > self.params["kpar_range"][1]))
else:
restrict_par = None
if self.params["kperp_range"] is not None:
print "restricting k_perp to ", self.params["kperp_range"]
restrict_perp = np.where(
np.logical_or(kx < self.params["kperp_range"][0],
kx > self.params["kperp_range"][1]))
else:
restrict_perp = None
for treatment in pwr_map.treatment_cases:
# comb is used for autopower AxB etc.; here just use a placeholder
comb = pwr_map.comb_cases[0]
pwrcase = "%s:%s" % (comb, treatment)
outplot_power_file = "%s/power_2d_%s.png" % \
(self.params['outdir'], treatment)
plot_slice.simpleplot_2D(outplot_power_file,
np.abs(pwr_map.pwrspec_2d[pwrcase]),
logkx,
logky, ["logkx", "logky"],
1.,
"2d power",
"log(abs(2d power))",
logscale=True)
outplot_transfer_file = "%s/transfer_2d_%s.png" % \
(self.params['outdir'], treatment)
if transfer_dict is not None:
plot_slice.simpleplot_2D(outplot_transfer_file,
transfer_dict[treatment],
logkx,
logky, ["logkx", "logky"],
1.,
"2d trans",
"2d trans",
logscale=False)
else:
plot_slice.simpleplot_2D(outplot_transfer_file,
np.ones_like(
pwr_map.counts_2d[pwrcase]),
logkx,
logky, ["logkx", "logky"],
1.,
"2d trans",
"2d trans",
logscale=False)
outplot_count_file = "%s/countweight_2d_%s.png" % \
(self.params['outdir'], treatment)
plot_slice.simpleplot_2D(outplot_count_file,
np.abs(pwr_map.counts_2d[pwrcase]),
logkx,
logky, ["logkx", "logky"],
1.,
"Log-counts",
"log-counts",
logscale=True)
if self.params["noiseweights_2dto1d"] is not None:
print "applying 2D noise weights: " + \
self.params["noiseweights_2dto1d"]
weightfile = h5py.File(self.params["noiseweights_2dto1d"], "r")
weights_2d = {}
for treatment in pwr_map.treatment_cases:
weights_2d[treatment] = weightfile[treatment].value
if restrict_perp is not None:
weights_2d[treatment][restrict_perp, :] = 0.
if restrict_par is not None:
weights_2d[treatment][:, restrict_par] = 0.
outplot_weight_file = "%s/noiseweight_2d_%s.png" % \
(self.params['outdir'], treatment)
plot_slice.simpleplot_2D(outplot_weight_file,
np.abs(weights_2d[treatment]),
logkx,
logky, ["logkx", "logky"],
1.,
"Log-weight",
"log-weight",
logscale=True)
weightfile.close()
pwr_map.convert_2d_to_1d(weights_2d=weights_2d)
else:
pwr_map.convert_2d_to_1d()
# open the output hd5 and attach directories for each treatment
summary_struct = h5py.File(self.params["summaryfile"], "w")
summary_by_treatment = {}
for treatment in pwr_map.treatment_cases:
summary_by_treatment[treatment] = \
summary_struct.create_group(treatment)
pwr_map_summary = pwr_map.agg_stat_1d_pwrspec(from_2d=True)
field_list = ["mean", "std", "gauss_std", "cov", "corr"]
for treatment in pwr_map.treatment_cases:
# copy data from the summary to the output hd5
for field in field_list:
summary_by_treatment[treatment][field] = \
pwr_map_summary[treatment][field]
summary_by_treatment[treatment]["k_vec_left"] = \
pwr_map.k_1d_from_2d["left"]
summary_by_treatment[treatment]["k_vec"] = \
pwr_map.k_1d_from_2d["center"]
summary_by_treatment[treatment]["k_vec_right"] = \
pwr_map.k_1d_from_2d["right"]
outfile = "%s/pk_%s.dat" % (self.params["outdir"], treatment)
file_tools.print_multicolumn(pwr_map.k_1d_from_2d["center"],
pwr_map_summary[treatment]["mean"],
pwr_map_summary[treatment]["std"],
outfile=outfile)
print "writing to " + outfile
summary_struct.close()
