def execute(self, processes):
        """produce a list of files to combine and run"""
        stat_results = {}
        for treatment in self.treatments:
            stat_summary = {}
            (stat_1d, error_stat_1d, counts_1d) = \
                    self.aggregate_1d_statistics(treatment, from2d=False)

            stat_summary["pk_1d_stat"] = stat_1d
            stat_summary["pk_1d_errstat"] = error_stat_1d
            stat_summary["pk_1d_counts"] = counts_1d

            (stat_1d, error_stat_1d, counts_1d) = \
                    self.aggregate_1d_statistics(treatment, from2d=True)

            stat_summary["pk_1d_from_2d_stat"] = stat_1d
            stat_summary["pk_1d_from_2d_errstat"] = error_stat_1d
            stat_summary["pk_1d_from_2d_counts"] = counts_1d

            (stat_2d, counts_2d) = self.aggregate_2d_statistics(treatment)

            stat_summary["pk_2d_stat"] = stat_2d
            stat_summary["pk_2d_counts"] = counts_2d

            stat_summary["k_1d"] = self.summary["k_1d"]["center"].value
            stat_summary["k_1d_from_2d"] = \
                    self.summary["k_1d_from_2d"]["center"].value

            stat_summary["kx_2d"] = self.summary["kx_2d"]["center"].value
            stat_summary["ky_2d"] = self.summary["ky_2d"]["center"].value

            stat_results[treatment] = copy.deepcopy(stat_summary)

        # in shelve file model
        #self.summary["stats"] = stat_results
        #print self.summary["stats"]["0modes"].keys()

        self.summary.close()
        file_tools.convert_numpytree_hdf5(stat_results,
                                          self.params["statfile_out"])
Beispiel #2
0
    def execute(self, processes):
        """produce a list of files to combine and run"""
        stat_results = {}
        for treatment in self.treatments:
            stat_summary = {}
            (stat_1d, error_stat_1d, counts_1d) = \
                    self.aggregate_1d_statistics(treatment, from2d=False)

            stat_summary["pk_1d_stat"] = stat_1d
            stat_summary["pk_1d_errstat"] = error_stat_1d
            stat_summary["pk_1d_counts"] = counts_1d

            (stat_1d, error_stat_1d, counts_1d) = \
                    self.aggregate_1d_statistics(treatment, from2d=True)

            stat_summary["pk_1d_from_2d_stat"] = stat_1d
            stat_summary["pk_1d_from_2d_errstat"] = error_stat_1d
            stat_summary["pk_1d_from_2d_counts"] = counts_1d

            (stat_2d, counts_2d) = self.aggregate_2d_statistics(treatment)

            stat_summary["pk_2d_stat"] = stat_2d
            stat_summary["pk_2d_counts"] = counts_2d

            stat_summary["k_1d"] = self.summary["k_1d"]["center"].value
            stat_summary["k_1d_from_2d"] = \
                    self.summary["k_1d_from_2d"]["center"].value

            stat_summary["kx_2d"] = self.summary["kx_2d"]["center"].value
            stat_summary["ky_2d"] = self.summary["ky_2d"]["center"].value

            stat_results[treatment] = copy.deepcopy(stat_summary)

        # in shelve file model
        #self.summary["stats"] = stat_results
        #print self.summary["stats"]["0modes"].keys()

        self.summary.close()
        file_tools.convert_numpytree_hdf5(stat_results,
                                          self.params["statfile_out"])
    def produce_summary(self, filelist, outfile, debug=False):
        num_sim = len(filelist)
        print "number of simulations to aggregate: %d" % num_sim

        data_subtract = None
        if self.params['subtract_pwrspec'] is not None:
            print "agg WARNING: you are subtracting a power spectrum"
            print "from file ", self.params['subtract_pwrspec']
            data_subtract = ps.PowerSpectrum(self.params['subtract_pwrspec'])

        # accumulate information about this set of files by loading just the
        # first one:
        sim_toread = ps.PowerSpectrum(filelist[0])
        # run this here just to get the 2D->1D bin centers
        sim_toread.convert_2d_to_1d()

        k_1d = sim_toread.k_1d
        k_1d_from_2d = sim_toread.k_1d_from_2d
        num_k_1d = sim_toread.num_k_1d
        num_k_1d_from_2d = sim_toread.num_k_1d_from_2d

        (kx_2d, ky_2d) = (sim_toread.kx_2d, sim_toread.ky_2d)
        (kx, ky) = (kx_2d['center'], ky_2d['center'])
        (num_kx, num_ky) = (sim_toread.num_kx, sim_toread.num_ky)

        print "k_1d bins: %d, %d, kx bins: %d, ky bins: %d" % \
               (num_k_1d, num_k_1d_from_2d, num_kx, num_ky)

        print "AggregateSummary: treatment cases: ", sim_toread.treatment_cases

        # mask any parts of k space (useful for making signal simulations)
        if self.params["kpar_range"] is not None:
            print "restricting k_par to ", self.params["kpar_range"]
            restrict_par = np.where(np.logical_or(
                               ky < self.params["kpar_range"][0],
                               ky > self.params["kpar_range"][1]))
        else:
            restrict_par = None

        if self.params["kperp_range"] is not None:
            print "restricting k_perp to ", self.params["kperp_range"]
            restrict_perp = np.where(np.logical_or(
                               kx < self.params["kperp_range"][0],
                               kx > self.params["kperp_range"][1]))
        else:
            restrict_perp = None

        # load the transfer functions and 2d->1d noise weights
        # if applying a mode correction at fixed treatment (for Gaussian noise
        # model)
        fixed_treatment = self.params["fix_weight_treatment"]
        if fixed_treatment:
            checklist = None
        else:
            checklist = sim_toread.treatment_cases

        full_transfer_dict = pe.load_transferfunc(
                                    self.params["apply_2d_beamtransfer"],
                                    self.params["apply_2d_modetransfer"],
                                    checklist)

        transfer_dict = None
        if full_transfer_dict is not None:
            transfer_dict = {}
            for treatment in sim_toread.treatment_cases:
                if fixed_treatment is not None:
                    print "fixing transfer for %s to value at %s" % \
                            (treatment, fixed_treatment)
                    transfer_dict[treatment] = full_transfer_dict[fixed_treatment]
                else:
                    transfer_dict[treatment] = full_transfer_dict[treatment]

        weights_2d = None
        if self.params["noiseweights_2dto1d"] is not None:
            print "applying 2D noise weights: " + \
                self.params["noiseweights_2dto1d"]

            weightfile = h5py.File(self.params["noiseweights_2dto1d"], "r")
            weights_2d = {}
            for treatment in sim_toread.treatment_cases:
                if fixed_treatment is not None:
                    print "fixing weight for %s to value at %s" % \
                            (treatment, fixed_treatment)
                    weights_2d[treatment] = weightfile[fixed_treatment].value
                else:
                    weights_2d[treatment] = weightfile[treatment].value

                if restrict_perp is not None:
                    weights_2d[treatment][restrict_perp, :] = 0.

                if restrict_par is not None:
                    weights_2d[treatment][:, restrict_par] = 0.

                #outplot_weight_file = "cool_noiseweight_2d_%s.png" % treatment
                #plot_slice.simpleplot_2D(outplot_weight_file,
                #                     np.abs(weights_2d[treatment]),
                #                     np.log10(kx), np.log10(ky),
                #                     ["logkx", "logky"], 1., "Log-weight",
                #                     "log-weight", logscale=True)

            weightfile.close()

        result_dict = {}
        for treatment in sim_toread.treatment_cases:
            treatment_dict = {}
            treatment_dict["pk_1d"] = np.zeros((num_sim, num_k_1d))
            treatment_dict["pk_1d_from_2d"] = \
                        np.zeros((num_sim, num_k_1d_from_2d))

            treatment_dict["pkstd_1d"] = np.zeros((num_sim, num_k_1d))
            treatment_dict["pkstd_1d_from_2d"] = \
                        np.zeros((num_sim, num_k_1d_from_2d))

            treatment_dict["pk_2d"] = np.zeros((num_sim, num_kx, num_ky))

            treatment_dict["counts_1d"] = np.zeros((num_sim, num_k_1d))
            treatment_dict["counts_1d_from_2d"] = \
                        np.zeros((num_sim, num_k_1d_from_2d))

            treatment_dict["counts_2d"] = np.zeros((num_sim, num_kx, num_ky))
            result_dict[treatment] = treatment_dict

        for (simfile, index) in zip(filelist, range(num_sim)):
            print "processing ", simfile, index
            sim_toread = ps.PowerSpectrum(simfile)

            # optionally subtract some reference spectrum
            if data_subtract is not None:
                print "agg WARNING: you are subtracting a power spectrum"
                # assuming these both have the same treatments
                for pwrspec_case in sim_toread.pwrspec_1d:
                    sim_toread.pwrspec_1d[pwrspec_case] -= \
                        data_subtract.pwrspec_1d[pwrspec_case]

                    sim_toread.pwrspec_2d[pwrspec_case] -= \
                        data_subtract.pwrspec_2d[pwrspec_case]

            sim_toread.apply_2d_trans_by_treatment(transfer_dict)
            sim_toread.convert_2d_to_1d(weights_2d=weights_2d)

            agg1d = sim_toread.agg_stat_1d_pwrspec()
            agg1d_from_2d = sim_toread.agg_stat_1d_pwrspec(from_2d=True)
            agg2d = sim_toread.agg_stat_2d_pwrspec()

            for treatment in sim_toread.treatment_cases:
                result_dict[treatment]["pk_1d"][index, :] = \
                                            agg1d[treatment]['mean']

                result_dict[treatment]["pk_1d_from_2d"][index, :] = \
                                            agg1d_from_2d[treatment]['mean']

                result_dict[treatment]["pkstd_1d"][index, :] = \
                                            agg1d[treatment]['std']

                result_dict[treatment]["pkstd_1d_from_2d"][index, :] = \
                                            agg1d_from_2d[treatment]['std']

                result_dict[treatment]["pk_2d"][index, :, :] = \
                                            agg2d[treatment]['mean']

                result_dict[treatment]["counts_1d"][index, :] = \
                                            agg1d[treatment]['counts']

                result_dict[treatment]["counts_1d_from_2d"][index, :] = \
                                            agg1d_from_2d[treatment]['counts']

                result_dict[treatment]["counts_2d"][index, :, :] = \
                                            agg2d[treatment]['counts']

        # package all simulations of all treatments into a file
        #out_dicttree = shelve.open(outfile, "n", protocol=-1)
        out_dicttree = {}
        out_dicttree["k_1d"] = k_1d
        out_dicttree["k_1d_from_2d"] = k_1d_from_2d
        out_dicttree["kx_2d"] = kx_2d
        out_dicttree["ky_2d"] = ky_2d
        out_dicttree["results"] = result_dict
        file_tools.convert_numpytree_hdf5(out_dicttree, outfile)
Beispiel #4
0
    def produce_summary(self, filelist, outfile, debug=False):
        num_sim = len(filelist)
        print "number of simulations to aggregate: %d" % num_sim

        data_subtract = None
        if self.params['subtract_pwrspec'] is not None:
            print "agg WARNING: you are subtracting a power spectrum"
            print "from file ", self.params['subtract_pwrspec']
            data_subtract = ps.PowerSpectrum(self.params['subtract_pwrspec'])

        # accumulate information about this set of files by loading just the
        # first one:
        sim_toread = ps.PowerSpectrum(filelist[0])
        # run this here just to get the 2D->1D bin centers
        sim_toread.convert_2d_to_1d()

        k_1d = sim_toread.k_1d
        k_1d_from_2d = sim_toread.k_1d_from_2d
        num_k_1d = sim_toread.num_k_1d
        num_k_1d_from_2d = sim_toread.num_k_1d_from_2d

        (kx_2d, ky_2d) = (sim_toread.kx_2d, sim_toread.ky_2d)
        (kx, ky) = (kx_2d['center'], ky_2d['center'])
        (num_kx, num_ky) = (sim_toread.num_kx, sim_toread.num_ky)

        print "k_1d bins: %d, %d, kx bins: %d, ky bins: %d" % \
               (num_k_1d, num_k_1d_from_2d, num_kx, num_ky)

        print "AggregateSummary: treatment cases: ", sim_toread.treatment_cases

        # mask any parts of k space (useful for making signal simulations)
        if self.params["kpar_range"] is not None:
            print "restricting k_par to ", self.params["kpar_range"]
            restrict_par = np.where(
                np.logical_or(ky < self.params["kpar_range"][0],
                              ky > self.params["kpar_range"][1]))
        else:
            restrict_par = None

        if self.params["kperp_range"] is not None:
            print "restricting k_perp to ", self.params["kperp_range"]
            restrict_perp = np.where(
                np.logical_or(kx < self.params["kperp_range"][0],
                              kx > self.params["kperp_range"][1]))
        else:
            restrict_perp = None

        # load the transfer functions and 2d->1d noise weights
        # if applying a mode correction at fixed treatment (for Gaussian noise
        # model)
        fixed_treatment = self.params["fix_weight_treatment"]
        if fixed_treatment:
            checklist = None
        else:
            checklist = sim_toread.treatment_cases

        full_transfer_dict = pe.load_transferfunc(
            self.params["apply_2d_beamtransfer"],
            self.params["apply_2d_modetransfer"], checklist)

        transfer_dict = None
        if full_transfer_dict is not None:
            transfer_dict = {}
            for treatment in sim_toread.treatment_cases:
                if fixed_treatment is not None:
                    print "fixing transfer for %s to value at %s" % \
                            (treatment, fixed_treatment)
                    transfer_dict[treatment] = full_transfer_dict[
                        fixed_treatment]
                else:
                    transfer_dict[treatment] = full_transfer_dict[treatment]

        weights_2d = None
        if self.params["noiseweights_2dto1d"] is not None:
            print "applying 2D noise weights: " + \
                self.params["noiseweights_2dto1d"]

            weightfile = h5py.File(self.params["noiseweights_2dto1d"], "r")
            weights_2d = {}
            for treatment in sim_toread.treatment_cases:
                if fixed_treatment is not None:
                    print "fixing weight for %s to value at %s" % \
                            (treatment, fixed_treatment)
                    weights_2d[treatment] = weightfile[fixed_treatment].value
                else:
                    weights_2d[treatment] = weightfile[treatment].value

                if restrict_perp is not None:
                    weights_2d[treatment][restrict_perp, :] = 0.

                if restrict_par is not None:
                    weights_2d[treatment][:, restrict_par] = 0.

                #outplot_weight_file = "cool_noiseweight_2d_%s.png" % treatment
                #plot_slice.simpleplot_2D(outplot_weight_file,
                #                     np.abs(weights_2d[treatment]),
                #                     np.log10(kx), np.log10(ky),
                #                     ["logkx", "logky"], 1., "Log-weight",
                #                     "log-weight", logscale=True)

            weightfile.close()

        result_dict = {}
        for treatment in sim_toread.treatment_cases:
            treatment_dict = {}
            treatment_dict["pk_1d"] = np.zeros((num_sim, num_k_1d))
            treatment_dict["pk_1d_from_2d"] = \
                        np.zeros((num_sim, num_k_1d_from_2d))

            treatment_dict["pkstd_1d"] = np.zeros((num_sim, num_k_1d))
            treatment_dict["pkstd_1d_from_2d"] = \
                        np.zeros((num_sim, num_k_1d_from_2d))

            treatment_dict["pk_2d"] = np.zeros((num_sim, num_kx, num_ky))

            treatment_dict["counts_1d"] = np.zeros((num_sim, num_k_1d))
            treatment_dict["counts_1d_from_2d"] = \
                        np.zeros((num_sim, num_k_1d_from_2d))

            treatment_dict["counts_2d"] = np.zeros((num_sim, num_kx, num_ky))
            result_dict[treatment] = treatment_dict

        for (simfile, index) in zip(filelist, range(num_sim)):
            print "processing ", simfile, index
            sim_toread = ps.PowerSpectrum(simfile)

            # optionally subtract some reference spectrum
            if data_subtract is not None:
                print "agg WARNING: you are subtracting a power spectrum"
                # assuming these both have the same treatments
                for pwrspec_case in sim_toread.pwrspec_1d:
                    sim_toread.pwrspec_1d[pwrspec_case] -= \
                        data_subtract.pwrspec_1d[pwrspec_case]

                    sim_toread.pwrspec_2d[pwrspec_case] -= \
                        data_subtract.pwrspec_2d[pwrspec_case]

            sim_toread.apply_2d_trans_by_treatment(transfer_dict)
            sim_toread.convert_2d_to_1d(weights_2d=weights_2d)

            agg1d = sim_toread.agg_stat_1d_pwrspec()
            agg1d_from_2d = sim_toread.agg_stat_1d_pwrspec(from_2d=True)
            agg2d = sim_toread.agg_stat_2d_pwrspec()

            for treatment in sim_toread.treatment_cases:
                result_dict[treatment]["pk_1d"][index, :] = \
                                            agg1d[treatment]['mean']

                result_dict[treatment]["pk_1d_from_2d"][index, :] = \
                                            agg1d_from_2d[treatment]['mean']

                result_dict[treatment]["pkstd_1d"][index, :] = \
                                            agg1d[treatment]['std']

                result_dict[treatment]["pkstd_1d_from_2d"][index, :] = \
                                            agg1d_from_2d[treatment]['std']

                result_dict[treatment]["pk_2d"][index, :, :] = \
                                            agg2d[treatment]['mean']

                result_dict[treatment]["counts_1d"][index, :] = \
                                            agg1d[treatment]['counts']

                result_dict[treatment]["counts_1d_from_2d"][index, :] = \
                                            agg1d_from_2d[treatment]['counts']

                result_dict[treatment]["counts_2d"][index, :, :] = \
                                            agg2d[treatment]['counts']

        # package all simulations of all treatments into a file
        #out_dicttree = shelve.open(outfile, "n", protocol=-1)
        out_dicttree = {}
        out_dicttree["k_1d"] = k_1d
        out_dicttree["k_1d_from_2d"] = k_1d_from_2d
        out_dicttree["kx_2d"] = kx_2d
        out_dicttree["ky_2d"] = ky_2d
        out_dicttree["results"] = result_dict
        file_tools.convert_numpytree_hdf5(out_dicttree, outfile)