def execute(self, processes): """produce a list of files to combine and run""" stat_results = {} for treatment in self.treatments: stat_summary = {} (stat_1d, error_stat_1d, counts_1d) = \ self.aggregate_1d_statistics(treatment, from2d=False) stat_summary["pk_1d_stat"] = stat_1d stat_summary["pk_1d_errstat"] = error_stat_1d stat_summary["pk_1d_counts"] = counts_1d (stat_1d, error_stat_1d, counts_1d) = \ self.aggregate_1d_statistics(treatment, from2d=True) stat_summary["pk_1d_from_2d_stat"] = stat_1d stat_summary["pk_1d_from_2d_errstat"] = error_stat_1d stat_summary["pk_1d_from_2d_counts"] = counts_1d (stat_2d, counts_2d) = self.aggregate_2d_statistics(treatment) stat_summary["pk_2d_stat"] = stat_2d stat_summary["pk_2d_counts"] = counts_2d stat_summary["k_1d"] = self.summary["k_1d"]["center"].value stat_summary["k_1d_from_2d"] = \ self.summary["k_1d_from_2d"]["center"].value stat_summary["kx_2d"] = self.summary["kx_2d"]["center"].value stat_summary["ky_2d"] = self.summary["ky_2d"]["center"].value stat_results[treatment] = copy.deepcopy(stat_summary) # in shelve file model #self.summary["stats"] = stat_results #print self.summary["stats"]["0modes"].keys() self.summary.close() file_tools.convert_numpytree_hdf5(stat_results, self.params["statfile_out"])
def execute(self, processes): """produce a list of files to combine and run""" stat_results = {} for treatment in self.treatments: stat_summary = {} (stat_1d, error_stat_1d, counts_1d) = \ self.aggregate_1d_statistics(treatment, from2d=False) stat_summary["pk_1d_stat"] = stat_1d stat_summary["pk_1d_errstat"] = error_stat_1d stat_summary["pk_1d_counts"] = counts_1d (stat_1d, error_stat_1d, counts_1d) = \ self.aggregate_1d_statistics(treatment, from2d=True) stat_summary["pk_1d_from_2d_stat"] = stat_1d stat_summary["pk_1d_from_2d_errstat"] = error_stat_1d stat_summary["pk_1d_from_2d_counts"] = counts_1d (stat_2d, counts_2d) = self.aggregate_2d_statistics(treatment) stat_summary["pk_2d_stat"] = stat_2d stat_summary["pk_2d_counts"] = counts_2d stat_summary["k_1d"] = self.summary["k_1d"]["center"].value stat_summary["k_1d_from_2d"] = \ self.summary["k_1d_from_2d"]["center"].value stat_summary["kx_2d"] = self.summary["kx_2d"]["center"].value stat_summary["ky_2d"] = self.summary["ky_2d"]["center"].value stat_results[treatment] = copy.deepcopy(stat_summary) # in shelve file model #self.summary["stats"] = stat_results #print self.summary["stats"]["0modes"].keys() self.summary.close() file_tools.convert_numpytree_hdf5(stat_results, self.params["statfile_out"])
def produce_summary(self, filelist, outfile, debug=False): num_sim = len(filelist) print "number of simulations to aggregate: %d" % num_sim data_subtract = None if self.params['subtract_pwrspec'] is not None: print "agg WARNING: you are subtracting a power spectrum" print "from file ", self.params['subtract_pwrspec'] data_subtract = ps.PowerSpectrum(self.params['subtract_pwrspec']) # accumulate information about this set of files by loading just the # first one: sim_toread = ps.PowerSpectrum(filelist[0]) # run this here just to get the 2D->1D bin centers sim_toread.convert_2d_to_1d() k_1d = sim_toread.k_1d k_1d_from_2d = sim_toread.k_1d_from_2d num_k_1d = sim_toread.num_k_1d num_k_1d_from_2d = sim_toread.num_k_1d_from_2d (kx_2d, ky_2d) = (sim_toread.kx_2d, sim_toread.ky_2d) (kx, ky) = (kx_2d['center'], ky_2d['center']) (num_kx, num_ky) = (sim_toread.num_kx, sim_toread.num_ky) print "k_1d bins: %d, %d, kx bins: %d, ky bins: %d" % \ (num_k_1d, num_k_1d_from_2d, num_kx, num_ky) print "AggregateSummary: treatment cases: ", sim_toread.treatment_cases # mask any parts of k space (useful for making signal simulations) if self.params["kpar_range"] is not None: print "restricting k_par to ", self.params["kpar_range"] restrict_par = np.where(np.logical_or( ky < self.params["kpar_range"][0], ky > self.params["kpar_range"][1])) else: restrict_par = None if self.params["kperp_range"] is not None: print "restricting k_perp to ", self.params["kperp_range"] restrict_perp = np.where(np.logical_or( kx < self.params["kperp_range"][0], kx > self.params["kperp_range"][1])) else: restrict_perp = None # load the transfer functions and 2d->1d noise weights # if applying a mode correction at fixed treatment (for Gaussian noise # model) fixed_treatment = self.params["fix_weight_treatment"] if fixed_treatment: checklist = None else: checklist = sim_toread.treatment_cases full_transfer_dict = pe.load_transferfunc( self.params["apply_2d_beamtransfer"], self.params["apply_2d_modetransfer"], checklist) transfer_dict = None if full_transfer_dict is not None: transfer_dict = {} for treatment in sim_toread.treatment_cases: if fixed_treatment is not None: print "fixing transfer for %s to value at %s" % \ (treatment, fixed_treatment) transfer_dict[treatment] = full_transfer_dict[fixed_treatment] else: transfer_dict[treatment] = full_transfer_dict[treatment] weights_2d = None if self.params["noiseweights_2dto1d"] is not None: print "applying 2D noise weights: " + \ self.params["noiseweights_2dto1d"] weightfile = h5py.File(self.params["noiseweights_2dto1d"], "r") weights_2d = {} for treatment in sim_toread.treatment_cases: if fixed_treatment is not None: print "fixing weight for %s to value at %s" % \ (treatment, fixed_treatment) weights_2d[treatment] = weightfile[fixed_treatment].value else: weights_2d[treatment] = weightfile[treatment].value if restrict_perp is not None: weights_2d[treatment][restrict_perp, :] = 0. if restrict_par is not None: weights_2d[treatment][:, restrict_par] = 0. #outplot_weight_file = "cool_noiseweight_2d_%s.png" % treatment #plot_slice.simpleplot_2D(outplot_weight_file, # np.abs(weights_2d[treatment]), # np.log10(kx), np.log10(ky), # ["logkx", "logky"], 1., "Log-weight", # "log-weight", logscale=True) weightfile.close() result_dict = {} for treatment in sim_toread.treatment_cases: treatment_dict = {} treatment_dict["pk_1d"] = np.zeros((num_sim, num_k_1d)) treatment_dict["pk_1d_from_2d"] = \ np.zeros((num_sim, num_k_1d_from_2d)) treatment_dict["pkstd_1d"] = np.zeros((num_sim, num_k_1d)) treatment_dict["pkstd_1d_from_2d"] = \ np.zeros((num_sim, num_k_1d_from_2d)) treatment_dict["pk_2d"] = np.zeros((num_sim, num_kx, num_ky)) treatment_dict["counts_1d"] = np.zeros((num_sim, num_k_1d)) treatment_dict["counts_1d_from_2d"] = \ np.zeros((num_sim, num_k_1d_from_2d)) treatment_dict["counts_2d"] = np.zeros((num_sim, num_kx, num_ky)) result_dict[treatment] = treatment_dict for (simfile, index) in zip(filelist, range(num_sim)): print "processing ", simfile, index sim_toread = ps.PowerSpectrum(simfile) # optionally subtract some reference spectrum if data_subtract is not None: print "agg WARNING: you are subtracting a power spectrum" # assuming these both have the same treatments for pwrspec_case in sim_toread.pwrspec_1d: sim_toread.pwrspec_1d[pwrspec_case] -= \ data_subtract.pwrspec_1d[pwrspec_case] sim_toread.pwrspec_2d[pwrspec_case] -= \ data_subtract.pwrspec_2d[pwrspec_case] sim_toread.apply_2d_trans_by_treatment(transfer_dict) sim_toread.convert_2d_to_1d(weights_2d=weights_2d) agg1d = sim_toread.agg_stat_1d_pwrspec() agg1d_from_2d = sim_toread.agg_stat_1d_pwrspec(from_2d=True) agg2d = sim_toread.agg_stat_2d_pwrspec() for treatment in sim_toread.treatment_cases: result_dict[treatment]["pk_1d"][index, :] = \ agg1d[treatment]['mean'] result_dict[treatment]["pk_1d_from_2d"][index, :] = \ agg1d_from_2d[treatment]['mean'] result_dict[treatment]["pkstd_1d"][index, :] = \ agg1d[treatment]['std'] result_dict[treatment]["pkstd_1d_from_2d"][index, :] = \ agg1d_from_2d[treatment]['std'] result_dict[treatment]["pk_2d"][index, :, :] = \ agg2d[treatment]['mean'] result_dict[treatment]["counts_1d"][index, :] = \ agg1d[treatment]['counts'] result_dict[treatment]["counts_1d_from_2d"][index, :] = \ agg1d_from_2d[treatment]['counts'] result_dict[treatment]["counts_2d"][index, :, :] = \ agg2d[treatment]['counts'] # package all simulations of all treatments into a file #out_dicttree = shelve.open(outfile, "n", protocol=-1) out_dicttree = {} out_dicttree["k_1d"] = k_1d out_dicttree["k_1d_from_2d"] = k_1d_from_2d out_dicttree["kx_2d"] = kx_2d out_dicttree["ky_2d"] = ky_2d out_dicttree["results"] = result_dict file_tools.convert_numpytree_hdf5(out_dicttree, outfile)
def produce_summary(self, filelist, outfile, debug=False): num_sim = len(filelist) print "number of simulations to aggregate: %d" % num_sim data_subtract = None if self.params['subtract_pwrspec'] is not None: print "agg WARNING: you are subtracting a power spectrum" print "from file ", self.params['subtract_pwrspec'] data_subtract = ps.PowerSpectrum(self.params['subtract_pwrspec']) # accumulate information about this set of files by loading just the # first one: sim_toread = ps.PowerSpectrum(filelist[0]) # run this here just to get the 2D->1D bin centers sim_toread.convert_2d_to_1d() k_1d = sim_toread.k_1d k_1d_from_2d = sim_toread.k_1d_from_2d num_k_1d = sim_toread.num_k_1d num_k_1d_from_2d = sim_toread.num_k_1d_from_2d (kx_2d, ky_2d) = (sim_toread.kx_2d, sim_toread.ky_2d) (kx, ky) = (kx_2d['center'], ky_2d['center']) (num_kx, num_ky) = (sim_toread.num_kx, sim_toread.num_ky) print "k_1d bins: %d, %d, kx bins: %d, ky bins: %d" % \ (num_k_1d, num_k_1d_from_2d, num_kx, num_ky) print "AggregateSummary: treatment cases: ", sim_toread.treatment_cases # mask any parts of k space (useful for making signal simulations) if self.params["kpar_range"] is not None: print "restricting k_par to ", self.params["kpar_range"] restrict_par = np.where( np.logical_or(ky < self.params["kpar_range"][0], ky > self.params["kpar_range"][1])) else: restrict_par = None if self.params["kperp_range"] is not None: print "restricting k_perp to ", self.params["kperp_range"] restrict_perp = np.where( np.logical_or(kx < self.params["kperp_range"][0], kx > self.params["kperp_range"][1])) else: restrict_perp = None # load the transfer functions and 2d->1d noise weights # if applying a mode correction at fixed treatment (for Gaussian noise # model) fixed_treatment = self.params["fix_weight_treatment"] if fixed_treatment: checklist = None else: checklist = sim_toread.treatment_cases full_transfer_dict = pe.load_transferfunc( self.params["apply_2d_beamtransfer"], self.params["apply_2d_modetransfer"], checklist) transfer_dict = None if full_transfer_dict is not None: transfer_dict = {} for treatment in sim_toread.treatment_cases: if fixed_treatment is not None: print "fixing transfer for %s to value at %s" % \ (treatment, fixed_treatment) transfer_dict[treatment] = full_transfer_dict[ fixed_treatment] else: transfer_dict[treatment] = full_transfer_dict[treatment] weights_2d = None if self.params["noiseweights_2dto1d"] is not None: print "applying 2D noise weights: " + \ self.params["noiseweights_2dto1d"] weightfile = h5py.File(self.params["noiseweights_2dto1d"], "r") weights_2d = {} for treatment in sim_toread.treatment_cases: if fixed_treatment is not None: print "fixing weight for %s to value at %s" % \ (treatment, fixed_treatment) weights_2d[treatment] = weightfile[fixed_treatment].value else: weights_2d[treatment] = weightfile[treatment].value if restrict_perp is not None: weights_2d[treatment][restrict_perp, :] = 0. if restrict_par is not None: weights_2d[treatment][:, restrict_par] = 0. #outplot_weight_file = "cool_noiseweight_2d_%s.png" % treatment #plot_slice.simpleplot_2D(outplot_weight_file, # np.abs(weights_2d[treatment]), # np.log10(kx), np.log10(ky), # ["logkx", "logky"], 1., "Log-weight", # "log-weight", logscale=True) weightfile.close() result_dict = {} for treatment in sim_toread.treatment_cases: treatment_dict = {} treatment_dict["pk_1d"] = np.zeros((num_sim, num_k_1d)) treatment_dict["pk_1d_from_2d"] = \ np.zeros((num_sim, num_k_1d_from_2d)) treatment_dict["pkstd_1d"] = np.zeros((num_sim, num_k_1d)) treatment_dict["pkstd_1d_from_2d"] = \ np.zeros((num_sim, num_k_1d_from_2d)) treatment_dict["pk_2d"] = np.zeros((num_sim, num_kx, num_ky)) treatment_dict["counts_1d"] = np.zeros((num_sim, num_k_1d)) treatment_dict["counts_1d_from_2d"] = \ np.zeros((num_sim, num_k_1d_from_2d)) treatment_dict["counts_2d"] = np.zeros((num_sim, num_kx, num_ky)) result_dict[treatment] = treatment_dict for (simfile, index) in zip(filelist, range(num_sim)): print "processing ", simfile, index sim_toread = ps.PowerSpectrum(simfile) # optionally subtract some reference spectrum if data_subtract is not None: print "agg WARNING: you are subtracting a power spectrum" # assuming these both have the same treatments for pwrspec_case in sim_toread.pwrspec_1d: sim_toread.pwrspec_1d[pwrspec_case] -= \ data_subtract.pwrspec_1d[pwrspec_case] sim_toread.pwrspec_2d[pwrspec_case] -= \ data_subtract.pwrspec_2d[pwrspec_case] sim_toread.apply_2d_trans_by_treatment(transfer_dict) sim_toread.convert_2d_to_1d(weights_2d=weights_2d) agg1d = sim_toread.agg_stat_1d_pwrspec() agg1d_from_2d = sim_toread.agg_stat_1d_pwrspec(from_2d=True) agg2d = sim_toread.agg_stat_2d_pwrspec() for treatment in sim_toread.treatment_cases: result_dict[treatment]["pk_1d"][index, :] = \ agg1d[treatment]['mean'] result_dict[treatment]["pk_1d_from_2d"][index, :] = \ agg1d_from_2d[treatment]['mean'] result_dict[treatment]["pkstd_1d"][index, :] = \ agg1d[treatment]['std'] result_dict[treatment]["pkstd_1d_from_2d"][index, :] = \ agg1d_from_2d[treatment]['std'] result_dict[treatment]["pk_2d"][index, :, :] = \ agg2d[treatment]['mean'] result_dict[treatment]["counts_1d"][index, :] = \ agg1d[treatment]['counts'] result_dict[treatment]["counts_1d_from_2d"][index, :] = \ agg1d_from_2d[treatment]['counts'] result_dict[treatment]["counts_2d"][index, :, :] = \ agg2d[treatment]['counts'] # package all simulations of all treatments into a file #out_dicttree = shelve.open(outfile, "n", protocol=-1) out_dicttree = {} out_dicttree["k_1d"] = k_1d out_dicttree["k_1d_from_2d"] = k_1d_from_2d out_dicttree["kx_2d"] = kx_2d out_dicttree["ky_2d"] = ky_2d out_dicttree["results"] = result_dict file_tools.convert_numpytree_hdf5(out_dicttree, outfile)