def execute(self, processes):
"""produce a list of files to combine and run"""
stat_results = {}
for treatment in self.treatments:
stat_summary = {}
(stat_1d, error_stat_1d, counts_1d) = \
self.aggregate_1d_statistics(treatment, from2d=False)
stat_summary["pk_1d_stat"] = stat_1d
stat_summary["pk_1d_errstat"] = error_stat_1d
stat_summary["pk_1d_counts"] = counts_1d
(stat_1d, error_stat_1d, counts_1d) = \
self.aggregate_1d_statistics(treatment, from2d=True)
stat_summary["pk_1d_from_2d_stat"] = stat_1d
stat_summary["pk_1d_from_2d_errstat"] = error_stat_1d
stat_summary["pk_1d_from_2d_counts"] = counts_1d
(stat_2d, counts_2d) = self.aggregate_2d_statistics(treatment)
stat_summary["pk_2d_stat"] = stat_2d
stat_summary["pk_2d_counts"] = counts_2d
stat_summary["k_1d"] = self.summary["k_1d"]["center"].value
stat_summary["k_1d_from_2d"] = \
self.summary["k_1d_from_2d"]["center"].value
stat_summary["kx_2d"] = self.summary["kx_2d"]["center"].value
stat_summary["ky_2d"] = self.summary["ky_2d"]["center"].value
stat_results[treatment] = copy.deepcopy(stat_summary)
# in shelve file model
#self.summary["stats"] = stat_results
#print self.summary["stats"]["0modes"].keys()
self.summary.close()
file_tools.convert_numpytree_hdf5(stat_results,
self.params["statfile_out"])
def execute(self, processes):
"""produce a list of files to combine and run"""
stat_results = {}
for treatment in self.treatments:
stat_summary = {}
(stat_1d, error_stat_1d, counts_1d) = \
self.aggregate_1d_statistics(treatment, from2d=False)
stat_summary["pk_1d_stat"] = stat_1d
stat_summary["pk_1d_errstat"] = error_stat_1d
stat_summary["pk_1d_counts"] = counts_1d
(stat_1d, error_stat_1d, counts_1d) = \
self.aggregate_1d_statistics(treatment, from2d=True)
stat_summary["pk_1d_from_2d_stat"] = stat_1d
stat_summary["pk_1d_from_2d_errstat"] = error_stat_1d
stat_summary["pk_1d_from_2d_counts"] = counts_1d
(stat_2d, counts_2d) = self.aggregate_2d_statistics(treatment)
stat_summary["pk_2d_stat"] = stat_2d
stat_summary["pk_2d_counts"] = counts_2d
stat_summary["k_1d"] = self.summary["k_1d"]["center"].value
stat_summary["k_1d_from_2d"] = \
self.summary["k_1d_from_2d"]["center"].value
stat_summary["kx_2d"] = self.summary["kx_2d"]["center"].value
stat_summary["ky_2d"] = self.summary["ky_2d"]["center"].value
stat_results[treatment] = copy.deepcopy(stat_summary)
# in shelve file model
#self.summary["stats"] = stat_results
#print self.summary["stats"]["0modes"].keys()
self.summary.close()
file_tools.convert_numpytree_hdf5(stat_results,
self.params["statfile_out"])
def produce_summary(self, filelist, outfile, debug=False):
num_sim = len(filelist)
print "number of simulations to aggregate: %d" % num_sim
data_subtract = None
if self.params['subtract_pwrspec'] is not None:
print "agg WARNING: you are subtracting a power spectrum"
print "from file ", self.params['subtract_pwrspec']
data_subtract = ps.PowerSpectrum(self.params['subtract_pwrspec'])
# accumulate information about this set of files by loading just the
# first one:
sim_toread = ps.PowerSpectrum(filelist[0])
# run this here just to get the 2D->1D bin centers
sim_toread.convert_2d_to_1d()
k_1d = sim_toread.k_1d
k_1d_from_2d = sim_toread.k_1d_from_2d
num_k_1d = sim_toread.num_k_1d
num_k_1d_from_2d = sim_toread.num_k_1d_from_2d
(kx_2d, ky_2d) = (sim_toread.kx_2d, sim_toread.ky_2d)
(kx, ky) = (kx_2d['center'], ky_2d['center'])
(num_kx, num_ky) = (sim_toread.num_kx, sim_toread.num_ky)
print "k_1d bins: %d, %d, kx bins: %d, ky bins: %d" % \
(num_k_1d, num_k_1d_from_2d, num_kx, num_ky)
print "AggregateSummary: treatment cases: ", sim_toread.treatment_cases
# mask any parts of k space (useful for making signal simulations)
if self.params["kpar_range"] is not None:
print "restricting k_par to ", self.params["kpar_range"]
restrict_par = np.where(np.logical_or(
ky < self.params["kpar_range"][0],
ky > self.params["kpar_range"][1]))
else:
restrict_par = None
if self.params["kperp_range"] is not None:
print "restricting k_perp to ", self.params["kperp_range"]
restrict_perp = np.where(np.logical_or(
kx < self.params["kperp_range"][0],
kx > self.params["kperp_range"][1]))
else:
restrict_perp = None
# load the transfer functions and 2d->1d noise weights
# if applying a mode correction at fixed treatment (for Gaussian noise
# model)
fixed_treatment = self.params["fix_weight_treatment"]
if fixed_treatment:
checklist = None
else:
checklist = sim_toread.treatment_cases
full_transfer_dict = pe.load_transferfunc(
self.params["apply_2d_beamtransfer"],
self.params["apply_2d_modetransfer"],
checklist)
transfer_dict = None
if full_transfer_dict is not None:
transfer_dict = {}
for treatment in sim_toread.treatment_cases:
if fixed_treatment is not None:
print "fixing transfer for %s to value at %s" % \
(treatment, fixed_treatment)
transfer_dict[treatment] = full_transfer_dict[fixed_treatment]
else:
transfer_dict[treatment] = full_transfer_dict[treatment]
weights_2d = None
if self.params["noiseweights_2dto1d"] is not None:
print "applying 2D noise weights: " + \
self.params["noiseweights_2dto1d"]
weightfile = h5py.File(self.params["noiseweights_2dto1d"], "r")
weights_2d = {}
for treatment in sim_toread.treatment_cases:
if fixed_treatment is not None:
print "fixing weight for %s to value at %s" % \
(treatment, fixed_treatment)
weights_2d[treatment] = weightfile[fixed_treatment].value
else:
weights_2d[treatment] = weightfile[treatment].value
if restrict_perp is not None:
weights_2d[treatment][restrict_perp, :] = 0.
if restrict_par is not None:
weights_2d[treatment][:, restrict_par] = 0.
#outplot_weight_file = "cool_noiseweight_2d_%s.png" % treatment
#plot_slice.simpleplot_2D(outplot_weight_file,
# np.abs(weights_2d[treatment]),
# np.log10(kx), np.log10(ky),
# ["logkx", "logky"], 1., "Log-weight",
# "log-weight", logscale=True)
weightfile.close()
result_dict = {}
for treatment in sim_toread.treatment_cases:
treatment_dict = {}
treatment_dict["pk_1d"] = np.zeros((num_sim, num_k_1d))
treatment_dict["pk_1d_from_2d"] = \
np.zeros((num_sim, num_k_1d_from_2d))
treatment_dict["pkstd_1d"] = np.zeros((num_sim, num_k_1d))
treatment_dict["pkstd_1d_from_2d"] = \
np.zeros((num_sim, num_k_1d_from_2d))
treatment_dict["pk_2d"] = np.zeros((num_sim, num_kx, num_ky))
treatment_dict["counts_1d"] = np.zeros((num_sim, num_k_1d))
treatment_dict["counts_1d_from_2d"] = \
np.zeros((num_sim, num_k_1d_from_2d))
treatment_dict["counts_2d"] = np.zeros((num_sim, num_kx, num_ky))
result_dict[treatment] = treatment_dict
for (simfile, index) in zip(filelist, range(num_sim)):
print "processing ", simfile, index
sim_toread = ps.PowerSpectrum(simfile)
# optionally subtract some reference spectrum
if data_subtract is not None:
print "agg WARNING: you are subtracting a power spectrum"
# assuming these both have the same treatments
for pwrspec_case in sim_toread.pwrspec_1d:
sim_toread.pwrspec_1d[pwrspec_case] -= \
data_subtract.pwrspec_1d[pwrspec_case]
sim_toread.pwrspec_2d[pwrspec_case] -= \
data_subtract.pwrspec_2d[pwrspec_case]
sim_toread.apply_2d_trans_by_treatment(transfer_dict)
sim_toread.convert_2d_to_1d(weights_2d=weights_2d)
agg1d = sim_toread.agg_stat_1d_pwrspec()
agg1d_from_2d = sim_toread.agg_stat_1d_pwrspec(from_2d=True)
agg2d = sim_toread.agg_stat_2d_pwrspec()
for treatment in sim_toread.treatment_cases:
result_dict[treatment]["pk_1d"][index, :] = \
agg1d[treatment]['mean']
result_dict[treatment]["pk_1d_from_2d"][index, :] = \
agg1d_from_2d[treatment]['mean']
result_dict[treatment]["pkstd_1d"][index, :] = \
agg1d[treatment]['std']
result_dict[treatment]["pkstd_1d_from_2d"][index, :] = \
agg1d_from_2d[treatment]['std']
result_dict[treatment]["pk_2d"][index, :, :] = \
agg2d[treatment]['mean']
result_dict[treatment]["counts_1d"][index, :] = \
agg1d[treatment]['counts']
result_dict[treatment]["counts_1d_from_2d"][index, :] = \
agg1d_from_2d[treatment]['counts']
result_dict[treatment]["counts_2d"][index, :, :] = \
agg2d[treatment]['counts']
# package all simulations of all treatments into a file
#out_dicttree = shelve.open(outfile, "n", protocol=-1)
out_dicttree = {}
out_dicttree["k_1d"] = k_1d
out_dicttree["k_1d_from_2d"] = k_1d_from_2d
out_dicttree["kx_2d"] = kx_2d
out_dicttree["ky_2d"] = ky_2d
out_dicttree["results"] = result_dict
file_tools.convert_numpytree_hdf5(out_dicttree, outfile)
def produce_summary(self, filelist, outfile, debug=False):
num_sim = len(filelist)
print "number of simulations to aggregate: %d" % num_sim
data_subtract = None
if self.params['subtract_pwrspec'] is not None:
print "agg WARNING: you are subtracting a power spectrum"
print "from file ", self.params['subtract_pwrspec']
data_subtract = ps.PowerSpectrum(self.params['subtract_pwrspec'])
# accumulate information about this set of files by loading just the
# first one:
sim_toread = ps.PowerSpectrum(filelist[0])
# run this here just to get the 2D->1D bin centers
sim_toread.convert_2d_to_1d()
k_1d = sim_toread.k_1d
k_1d_from_2d = sim_toread.k_1d_from_2d
num_k_1d = sim_toread.num_k_1d
num_k_1d_from_2d = sim_toread.num_k_1d_from_2d
(kx_2d, ky_2d) = (sim_toread.kx_2d, sim_toread.ky_2d)
(kx, ky) = (kx_2d['center'], ky_2d['center'])
(num_kx, num_ky) = (sim_toread.num_kx, sim_toread.num_ky)
print "k_1d bins: %d, %d, kx bins: %d, ky bins: %d" % \
(num_k_1d, num_k_1d_from_2d, num_kx, num_ky)
print "AggregateSummary: treatment cases: ", sim_toread.treatment_cases
# mask any parts of k space (useful for making signal simulations)
if self.params["kpar_range"] is not None:
print "restricting k_par to ", self.params["kpar_range"]
restrict_par = np.where(
np.logical_or(ky < self.params["kpar_range"][0],
ky > self.params["kpar_range"][1]))
else:
restrict_par = None
if self.params["kperp_range"] is not None:
print "restricting k_perp to ", self.params["kperp_range"]
restrict_perp = np.where(
np.logical_or(kx < self.params["kperp_range"][0],
kx > self.params["kperp_range"][1]))
else:
restrict_perp = None
# load the transfer functions and 2d->1d noise weights
# if applying a mode correction at fixed treatment (for Gaussian noise
# model)
fixed_treatment = self.params["fix_weight_treatment"]
if fixed_treatment:
checklist = None
else:
checklist = sim_toread.treatment_cases
full_transfer_dict = pe.load_transferfunc(
self.params["apply_2d_beamtransfer"],
self.params["apply_2d_modetransfer"], checklist)
transfer_dict = None
if full_transfer_dict is not None:
transfer_dict = {}
for treatment in sim_toread.treatment_cases:
if fixed_treatment is not None:
print "fixing transfer for %s to value at %s" % \
(treatment, fixed_treatment)
transfer_dict[treatment] = full_transfer_dict[
fixed_treatment]
else:
transfer_dict[treatment] = full_transfer_dict[treatment]
weights_2d = None
if self.params["noiseweights_2dto1d"] is not None:
print "applying 2D noise weights: " + \
self.params["noiseweights_2dto1d"]
weightfile = h5py.File(self.params["noiseweights_2dto1d"], "r")
weights_2d = {}
for treatment in sim_toread.treatment_cases:
if fixed_treatment is not None:
print "fixing weight for %s to value at %s" % \
(treatment, fixed_treatment)
weights_2d[treatment] = weightfile[fixed_treatment].value
else:
weights_2d[treatment] = weightfile[treatment].value
if restrict_perp is not None:
weights_2d[treatment][restrict_perp, :] = 0.
if restrict_par is not None:
weights_2d[treatment][:, restrict_par] = 0.
#outplot_weight_file = "cool_noiseweight_2d_%s.png" % treatment
#plot_slice.simpleplot_2D(outplot_weight_file,
# np.abs(weights_2d[treatment]),
# np.log10(kx), np.log10(ky),
# ["logkx", "logky"], 1., "Log-weight",
# "log-weight", logscale=True)
weightfile.close()
result_dict = {}
for treatment in sim_toread.treatment_cases:
treatment_dict = {}
treatment_dict["pk_1d"] = np.zeros((num_sim, num_k_1d))
treatment_dict["pk_1d_from_2d"] = \
np.zeros((num_sim, num_k_1d_from_2d))
treatment_dict["pkstd_1d"] = np.zeros((num_sim, num_k_1d))
treatment_dict["pkstd_1d_from_2d"] = \
np.zeros((num_sim, num_k_1d_from_2d))
treatment_dict["pk_2d"] = np.zeros((num_sim, num_kx, num_ky))
treatment_dict["counts_1d"] = np.zeros((num_sim, num_k_1d))
treatment_dict["counts_1d_from_2d"] = \
np.zeros((num_sim, num_k_1d_from_2d))
treatment_dict["counts_2d"] = np.zeros((num_sim, num_kx, num_ky))
result_dict[treatment] = treatment_dict
for (simfile, index) in zip(filelist, range(num_sim)):
print "processing ", simfile, index
sim_toread = ps.PowerSpectrum(simfile)
# optionally subtract some reference spectrum
if data_subtract is not None:
print "agg WARNING: you are subtracting a power spectrum"
# assuming these both have the same treatments
for pwrspec_case in sim_toread.pwrspec_1d:
sim_toread.pwrspec_1d[pwrspec_case] -= \
data_subtract.pwrspec_1d[pwrspec_case]
sim_toread.pwrspec_2d[pwrspec_case] -= \
data_subtract.pwrspec_2d[pwrspec_case]
sim_toread.apply_2d_trans_by_treatment(transfer_dict)
sim_toread.convert_2d_to_1d(weights_2d=weights_2d)
agg1d = sim_toread.agg_stat_1d_pwrspec()
agg1d_from_2d = sim_toread.agg_stat_1d_pwrspec(from_2d=True)
agg2d = sim_toread.agg_stat_2d_pwrspec()
for treatment in sim_toread.treatment_cases:
result_dict[treatment]["pk_1d"][index, :] = \
agg1d[treatment]['mean']
result_dict[treatment]["pk_1d_from_2d"][index, :] = \
agg1d_from_2d[treatment]['mean']
result_dict[treatment]["pkstd_1d"][index, :] = \
agg1d[treatment]['std']
result_dict[treatment]["pkstd_1d_from_2d"][index, :] = \
agg1d_from_2d[treatment]['std']
result_dict[treatment]["pk_2d"][index, :, :] = \
agg2d[treatment]['mean']
result_dict[treatment]["counts_1d"][index, :] = \
agg1d[treatment]['counts']
result_dict[treatment]["counts_1d_from_2d"][index, :] = \
agg1d_from_2d[treatment]['counts']
result_dict[treatment]["counts_2d"][index, :, :] = \
agg2d[treatment]['counts']
# package all simulations of all treatments into a file
#out_dicttree = shelve.open(outfile, "n", protocol=-1)
out_dicttree = {}
out_dicttree["k_1d"] = k_1d
out_dicttree["k_1d_from_2d"] = k_1d_from_2d
out_dicttree["kx_2d"] = kx_2d
out_dicttree["ky_2d"] = ky_2d
out_dicttree["results"] = result_dict
file_tools.convert_numpytree_hdf5(out_dicttree, outfile)