def generate_difference_table(config1, config2, sbins=16, rbins=16, y1v2=None): hoopoe1, weights1, y1v2 = ct.setup(True, True, config1, y1v2=y1v2) hoopoe2, weights2, y1v2 = ct.setup(True, False, config2, y1v2=y1v2) subset = np.in1d(hoopoe1.res["coadd_objects_id"], hoopoe2.res["coadd_objects_id"]) nbc = cal.nbc() nbc_disc = cal.nbc() nbc_bulge = cal.nbc() wt1, wt2 = nbc.get_split_data(hoopoe1, weights=weights1, method="random") nbc_disc.load_from_cat(nbc, name="all") nbc_bulge.load_from_cat(nbc, name="all") import pdb pdb.set_trace() nbc_disc.diff_table(weights=weights1, compare_with_subset=subset, fit="disc", table_name="diff_table_disc.fits", sbins=sbins, rbins=rbins, binning="equal_number", rlim=(1.13, 3), slim=(12, 200)) nbc_bulge.diff_table(weights=weights2, compare_with_subset=subset, fit="bulge", table_name="diff_table_bulge.fits", sbins=sbins, rbins=rbins, binning="equal_number", rlim=(1.13, 3), slim=(12, 200))
def calibrate(data, method="grid", config=None, smoothing=3, sbins=16, rbins=16, nthread=1, rank=0): rbf = (method.lower()=="rbf") # Set up the wrappers nbc = cal.nbc() nbc_disc = cal.nbc() nbc.res = data.res del(data.res) gc.collect() nbc_bulge = cal.nbc() nbc_bulge.res=nbc.res[nbc.res["is_bulge"].astype(bool)] nbc_disc.res=nbc.res[np.invert(nbc.res["is_bulge"].astype(bool))] # Fit or interpolate if method is "polynomial": nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins)) nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins)) nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins)) nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins)) elif method is "rbf": nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing) nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing) elif method is "grid": nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read() nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read() # Apply to get a bias correction for each galaxy in the data nbc_disc.apply(split_half=0, scheme=method) print "Done disc calibration" nbc_bulge.apply(split_half=0, scheme=method) print "Done bulge calibration" print "Merging bulge/disc results...", nbc.res = arr.add_col(nbc.res, "m") nbc.res = arr.add_col(nbc.res, "c1") nbc.res = arr.add_col(nbc.res, "c2") nbc.res = arr.add_col(nbc.res, "a") nbc.res["m"][np.invert(nbc.res["is_bulge"].astype(bool))] = nbc_disc.res["m"] nbc.res["c1"][np.invert(nbc.res["is_bulge"].astype(bool))] = nbc_disc.res["c1"] nbc.res["c2"][np.invert(nbc.res["is_bulge"].astype(bool))] = nbc_disc.res["c2"] nbc.res["a"][np.invert(nbc.res["is_bulge"].astype(bool))] = nbc_disc.res["a"] nbc.res["m"][nbc.res["is_bulge"].astype(bool)] = nbc_bulge.res["m"] nbc.res["c1"][nbc.res["is_bulge"].astype(bool)] = nbc_bulge.res["c1"] nbc.res["c2"][nbc.res["is_bulge"].astype(bool)] = nbc_bulge.res["c2"] nbc.res["a"][nbc.res["is_bulge"].astype(bool)] = nbc_bulge.res["a"] print "done" if not os.path.exists(os.path.dirname(config["output"]["dir"])): os.system("mkdir -p %s"%os.path.dirname(config["output"]["dir"])) print "Saving calibrated catalogue to %s"%config["output"]["filename"] nbc.export(filename="%s/%s"%(config["output"]["dir"],config["output"]["filename"]))
def calibrate(data, method="rbf"): rbf = (method.lower()=="rbf") # Set up the wrappers nbc = cal.nbc() nbc_disc = cal.nbc() nbc_disc.res = data.res nbc_bulge = cal.nbc() # Fit or interpolate if method is "polynomial": nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"]) nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"]) nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"]) nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"]) elif method is "rbf": nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"]) nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"]) # Apply to get a bias correction for each galaxy in the data nbc_disc.apply(split_half=0, use_rbf=rbf) nbc.res = nbc_disc.res del(nbc_disc.res) print "Done bulge calibration" nbc_bulge.res = data.res nbc_bulge.apply(split_half=0, use_rbf=rbf) nbc.res[nbc.res["is_bulge"].astype(bool)] = nbc_bulge.res[nbc.res["is_bulge"].astype(bool)] del(nbc_bulge.res) print "Done disc calibration" if not os.path.exists(os.path.dirname(config["output_dir"])): os.system("mkdir -p %s"%os.path.dirname(config["output_dir"])) print "Saving calibrated catalogue to %s"%config["output_catalogue"] nbc.export(filename="%s/%s"%(config["output_dir"],config["output_catalogue"]))
def calculate(y1v2, hoopoe, split_method="none", weights=None, outputs=[], method="grid", smoothing=3, config=None, names=["m", "a"], sbins=16, rbins=16, nthread=1, rank=0, task_allocation={0 : ["disc_table", "bulge_table", "disc_htable", "bulge_htable"]} ): if not os.path.exists(config["output"]["dir"]): print "Warning - output plots directory does not exist" # First make some diagnostic distributions of the observable parameters relevant for the NBC if ("histograms" in outputs): pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output"]["dir"], weights=weights) pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs"%config["output"]["dir"], weights=weights) if ("alpha" in outputs): # Global fit of alpha to compare with previous runs plt.close() b = bias=di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22" ], xlim=(-0.03,0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True) plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1) plt.title("Y1 v2 Sim, PSF v02") os.system("mkdir -p %s/release/alpha"%config["output"]["dir"]) plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png"%config["output"]["dir"]) plt.close() nbc = cal.nbc() nbc_disc = cal.nbc() nbc_bulge = cal.nbc() # split the data into two subsets wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method) nbc_disc.load_from_cat(nbc, name="all") nbc_bulge.load_from_cat(nbc, name="all") #### Process disc then bulge runs # Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges edges_bulge = "equal_number" edges_disc = "equal_number" if ("half_tables" in outputs): if ("disc_htable" in task_allocation[rank]): nbc_disc.compute(split_half=1, fit="disc", weights=wt1, use_catalogue_weights=config["selection"]["weights"], reweight_per_bin=False, resample_per_bin=False, refdata=y1v2, binning=edges_disc, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method, sbins,rbins)) if ("bulge_htable" in task_allocation[rank]): nbc_bulge.compute(split_half=1, fit="bulge", weights=wt1, use_catalogue_weights=config["selection"]["weights"], reweight_per_bin=False, resample_per_bin=False, refdata=y1v2, binning=edges_bulge, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method, sbins,rbins)) if ("tables" in outputs): if ("disc_table" in task_allocation[rank]): nbc_disc.compute(split_half=0, fit="disc", weights=weights, use_catalogue_weights=config["selection"]["weights"], reweight_per_bin=False, resample_per_bin=False, refdata=y1v2, binning=edges_disc, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], sbins, rbins)) if ("bulge_table" in task_allocation[rank]): nbc_bulge.compute(split_half=0, fit="bulge", weights=weights, use_catalogue_weights=config["selection"]["weights"], reweight_per_bin=False, resample_per_bin=False, refdata=y1v2, binning=edges_bulge, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], sbins,rbins)) sub_dir = method.lower() # Now plot out the points and the resulting smoothing fit if ("snr" in outputs): if (method.lower()=="rbf") or (method.lower()=="grid"): nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing) nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing) elif (method.lower()=="polynomial"): nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins)) nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins)) nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins)) nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins)) nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-disc-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+sub_dir, smoothing, sbins, rbins), use_rbf=rbf) nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-disc-v1-1.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf) plt.close() nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-disc-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf) nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-disc-v1-2.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf) plt.close() nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-bulge-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf) nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-bulge-v1-1.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf) plt.close() nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-bulge-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf) nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-bulge-v1-2.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf) plt.close() if (split_method!="none"): # Derive a calibration from the grid in whichever way is specified in the config file if (method=="rbf"): nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins), smoothing=smoothing) nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins), smoothing=smoothing) elif (method=="grid"): nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))[1].read() nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))[1].read() elif (method=="polynomial"): nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins)) nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins)) nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins)) nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins)) # Apply it to the other half nbc_disc.apply(split_half=2, scheme=method, names=names) nbc_bulge.apply(split_half=2, scheme=method, names=names) nbc.combine_bd(nbc_disc,nbc_bulge, split_half=2, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) ) # Finally save some diagnostic plots in tomographic bins if ("redshift" in outputs): zbins=[ 0.2, 0.43, 0.63, 0.9, 1.3] tophat = config["calibration"]["ztype"]=="tophat" # Only do the redshift plots with the split catalogues if they were recalculated this time plt.close() if "m" in names: bias0=nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat, separate_components=False) bias=nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^",">"], apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat, separate_components=False) plt.ylabel("Multiplicative Bias $m$") plt.legend(loc="center right") plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat))) plt.close() if "a" in names: nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat) nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=4, apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat) plt.ylabel(r"PSF Leakage $\alpha$") plt.legend(loc="upper left") plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat))) plt.close() np.savetxt("%s/data/m-vs-z-uncalibrated-%s-halfcat-unblinded%s.txt"%(config["output"]["dir"],split_method, ""+"-tophat"*int(tophat)), np.array(bias0)) np.savetxt("%s/data/m-vs-z-%s_nbc-%s-halfcat-unblinded%s.txt"%(config["output"]["dir"],method,split_method, ""+"-tophat"*int(tophat)),np.array(bias)) # Finally redo the fits with the full catalogue nbc = cal.nbc() nbc_disc = cal.nbc() nbc_bulge = cal.nbc() wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights) nbc_disc.load_from_cat(nbc, name="all") nbc_bulge.load_from_cat(nbc, name="all") if (method.lower()=="rbf"): nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing) nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing) elif (method.lower()=="grid"): nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read() nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read() elif (method.lower()=="polynomial"): nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins)) nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins)) nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins)) nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins)) nbc_disc.apply(split_half=0, scheme=method, names=names) nbc_bulge.apply(split_half=0, scheme=method, names=names) nbc.combine_bd(nbc_disc,nbc_bulge, split_half=0, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) ) nbc.preserve_calibration_col("%s/data/hoopoe-%s-nbc_column.fits"%(config["output"]["dir"],method)) plt.close() if "m" in names: bias0 = nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False) bias = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^",">"], apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False) plt.ylabel("Multiplicative Bias $m$") plt.legend(loc="center right") plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"],sub_dir, smoothing, sbins, rbins, int(tophat))) plt.close() np.savetxt("%s/data/m-vs-z-uncalibrated-fullcat-unblinded%s.txt"%(config["output"]["dir"], ""+"-tophat"*int(tophat)), np.array(bias0)) np.savetxt("%s/data/m-vs-z-%s_nbc-fullcat-unblinded%s.txt"%(config["output"]["dir"], method,""+"-tophat"*int(tophat)), np.array(bias)) if "a" in names: nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat) nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=4, apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat) plt.ylabel(r"PSF Leakage $\alpha$") plt.legend(loc="upper left") plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"],sub_dir, smoothing, sbins, rbins, int(tophat))) plt.close() print "Done" return 0
def calibrate_in_bins(nbc, halfcats=False, split_method="", scheme="rbf", config=None, sbins=12, rbins=9, smoothing=3, names=["m", "a"]): discs = cal.nbc() bulges = cal.nbc() if halfcats: split_half = 2 prefix = "%s-halfcat" % split_method else: split_half = 0 prefix = "fullcat" zbin_numbers = np.unique(nbc.res["des_bin"]) print "Calibrating in %d redshift bins" % zbin_numbers.size for k in zbin_numbers: print "-- calibrating galaxies in bin %d" % k if halfcats: select_bin1 = nbc.res1["des_bin"] == k select_bin = nbc.res2["des_bin"] == k discs.res1 = copy.deepcopy(nbc.res1[select_bin1]) discs.truth1 = copy.deepcopy(nbc.truth1[select_bin1]) bulges.res1 = copy.deepcopy(nbc.res1[select_bin1]) bulges.truth1 = copy.deepcopy(nbc.truth1[select_bin1]) discs.res2 = copy.deepcopy(nbc.res2[select_bin]) discs.truth2 = copy.deepcopy(nbc.truth2[select_bin]) bulges.res2 = copy.deepcopy(nbc.res2[select_bin]) bulges.truth2 = copy.deepcopy(nbc.truth2[select_bin]) else: select_bin = nbc.res["des_bin"] == k discs.res = copy.deepcopy(nbc.res[select_bin]) discs.truth = copy.deepcopy(nbc.truth[select_bin]) bulges.res = copy.deepcopy(nbc.res[select_bin]) bulges.truth = copy.deepcopy(nbc.truth[select_bin]) if (scheme == "rbf"): discs.fit_rbf( table= "%s/nbc_data/bias_table_hoopoe-v1-%s-disc-%dsbins-%drbins-zbin%d.fits" % (config["output_dir"], prefix, sbins, rbins, k), smoothing=smoothing) bulges.fit_rbf( table= "%s/nbc_data/bias_table_hoopoe-v1-%s-bulge-%dsbins-%drbins-zbin%d.fits" % (config["output_dir"], prefix, sbins, rbins, k), smoothing=smoothing) elif (scheme == "grid"): discs.bias_grid = fi.FITS( "%s/nbc_data/bias_table_hoopoe-v1-%s-disc-%dsbins-%drbins-zbin%d.fits" % (config["output_dir"], prefix, sbins, rbins, k))[1].read() bulges.bias_grid = fi.FITS( "%s/nbc_data/bias_table_hoopoe-v1-%s-bulge-%dsbins-%drbins-zbin%d.fits" % (config["output_dir"], prefix, sbins, rbins, k))[1].read() elif (scheme == "polynomial"): discs.fit( "m", table= "%s/nbc_data/bias_table_hoopoe-v1-%s-disc-%dsbins-%drbins-zbin%d.fits" % (config["output_dir"], prefix, sbins, rbins, k)) discs.fit( "a", table= "%s/nbc_data/bias_table_hoopoe-v1-%s-disc-%dsbins-%drbins-zbin%d.fits" % (config["output_dir"], prefix, sbins, rbins, k)) bulges.fit( "m", table= "%s/nbc_data/bias_table_hoopoe-v1-%s-bulge-%dsbins-%drbins-zbin%d.fits" % (config["output_dir"], prefix, sbins, rbins, k)) bulges.fit( "a", table= "%s/nbc_data/bias_table_hoopoe-v1-%s-bulge-%dsbins-%drbins-zbin%d.fits" % (config["output_dir"], prefix, sbins, rbins, k)) discs.apply(split_half=split_half, scheme=scheme, names=names) bulges.apply(split_half=split_half, scheme=scheme, names=names) nbc.combine_bd(discs, bulges, split_half=split_half, mask=select_bin, names=["m"] * ("m" in names) + ["c1", "c2"] * ("a" in names)) return nbc
def diagnostics(y1v2, hoopoe, tomographic_calibration=False, histograms=True, split_method="random", weights=None, alpha=True, table=True, half_tables=True, vssnr=True, vsredshift=True, rbf=True, simple_grid=False, smoothing=3, config=None, names=["m", "a"], sbins=16, rbins=11): if rbf: sub_dir = "rbf" elif simple_grid: sub_dir = "grid" else: sub_dir = "polynomial" if not os.path.exists(config["output_dir"]): print "Warning - output plots directory does not exist" # First make some diagnostic distributions of the observable parameters relevant for the NBC if histograms: pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs" % config["output_dir"], weights=weights) pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs" % config["output_dir"], weights=weights) if alpha: # Global fit of alpha to compare with previous runs plt.close() b = bias = di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22"], xlim=(-0.03, 0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True) plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1) plt.title("Y1 v2 Sim, PSF v02") os.system("mkdir -p %s/release/alpha" % config["output_dir"]) plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png" % config["output_dir"]) plt.close() #Exit here if no calibration diagnostics are needed if (not table) and (not vssnr) and (not vsredshift): return 0 # Now compute the calibration using a random half of the simulation if vssnr or half_tables: nbc = cal.nbc() nbc_disc = cal.nbc() nbc_bulge = cal.nbc() wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method) nbc_disc.load_from_cat(nbc, name="all") nbc_bulge.load_from_cat(nbc, name="all") #### Process disc then bulge runs # Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges import pdb pdb.set_trace() if table: if ("match_gridpoints" in config.keys()): redges_bulge, sedges_bulge = bin_edges_from_table( config["match_gridpoints"], type="bulge") redges_disc, sedges_disc = bin_edges_from_table( config["match_gridpoints"], type="disc") edges_disc = (redges_disc, sedges_disc) edges_bulge = (redges_bulge, sedges_bulge) else: edges_bulge = "equal_number" edges_disc = "equal_number" if tomographic_calibration: zbin_numbers = np.unique(nbc.res["des_bin"]) if half_tables: print "Calculating half tables in %d redshift bins" % zbin_numbers.size for k in zbin_numbers: print "bin %d" % k nbc_disc.compute( split_half=1, fit="disc", weights=wt1, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_disc, redshift_bin=k, rbins=rbins, sbins=sbins, rlim=(1.13, 3.0), slim=(12, 200), table_name= "%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins-zbin%d.fits" % (config["output_dir"], split_method, sbins, rbins, k)) nbc_bulge.compute( split_half=1, fit="bulge", weights=wt1, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_bulge, redshift_bin=k, rbins=rbins, sbins=sbins, rlim=(1.13, 3.0), slim=(12, 200), table_name= "%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins-zbin%d.fits" % (config["output_dir"], split_method, sbins, rbins, k)) if table: print "Calculating full tables in %d redshift bins" % zbin_numbers.size for k in zbin_numbers: print "bin %d" % k nbc_disc.compute( split_half=0, fit="disc", weights=weights, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_disc, redshift_bin=k, rbins=rbins, sbins=sbins, rlim=(1.13, 3.0), slim=(12, 200), table_name= "%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins-zbin%d.fits" % (config["output_dir"], sbins, rbins, k)) nbc_bulge.compute( split_half=0, fit="bulge", weights=weights, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_bulge, redshift_bin=k, rbins=rbins, sbins=sbins, rlim=(1.13, 3.0), slim=(12, 200), table_name= "%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins-zbin%d.fits" % (config["output_dir"], sbins, rbins, k)) if rbf: scheme = "rbf" elif simple_grid: scheme = "grid" else: scheme = "polynomial" if half_tables: nbc = calibrate_in_bins(nbc, halfcats=True, split_method=split_method, scheme=scheme, config=config, sbins=sbins, rbins=rbins, smoothing=smoothing, names=names) # Finally save some diagnostic plots in tomographic bins if vsredshift: zbins = [0.2, 0.43, 0.63, 0.9, 1.3] tophat = config["tophat_binning"] if half_tables: plt.close() if "m" in names: nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o", "D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat, separate_components=False) nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^", ">"], apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat, separate_components=False) plt.ylabel("Multiplicative Bias $m$") plt.legend(loc="center right") plt.savefig( "%s/release/%s/m-bias-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png" % (config["output_dir"], sub_dir, split_method, smoothing, sbins, rbins, int(tophat))) plt.close() if "a" in names: nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated", ls="none", nbins=3, fmt=["o", "D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat) nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none", fmt=["^", ">"], nbins=3, apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat) plt.ylabel(r"PSF Leakage $\alpha$") plt.legend(loc="upper left") plt.savefig( "%s/release/%s/alpha-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png" % (config["output_dir"], sub_dir, split_method, smoothing, sbins, rbins, int(tophat))) plt.close() # Finally redo the fits with the full catalogue gc.collect() nbc = cal.nbc() wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method) nbc = calibrate_in_bins(nbc, halfcats=False, split_method=None, scheme=scheme, config=config, sbins=sbins, rbins=rbins, smoothing=smoothing, names=names) import pdb pdb.set_trace() plt.close() if "m" in names: nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o", "D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False) bias = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^", ">"], apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False) plt.ylabel("Multiplicative Bias $m$") plt.legend(loc="center right") plt.savefig( "%s/release/%s/m-bias-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png" % (config["output_dir"], sub_dir, smoothing, sbins, rbins, int(tophat))) plt.close() if "a" in names: nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated", ls="none", nbins=3, fmt=["o", "D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat) nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none", fmt=["^", ">"], nbins=3, apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat) plt.ylabel(r"PSF Leakage $\alpha$") plt.legend(loc="upper left") plt.savefig( "%s/release/%s/alpha-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png" % (config["output_dir"], sub_dir, smoothing, sbins, rbins, int(tophat))) plt.close()
pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs"%config["output"]["dir"], weights=weights) if ("alpha" in outputs): # Global fit of alpha to compare with previous runs plt.close() b = bias=di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22" ], xlim=(-0.03,0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True) plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1) plt.title("Y1 v2 Sim, PSF v02") os.system("mkdir -p %s/release/alpha"%config["output_dir"]) plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png"%config["output_dir"]) plt.close() # split the data into two subsets if ("snr" in outputs) or ("half_tables" in outputs): nbc = cal.nbc() nbc_disc = cal.nbc() nbc_bulge = cal.nbc() wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method) nbc_disc.load_from_cat(nbc, name="all") nbc_bulge.load_from_cat(nbc, name="all") #### Process disc then bulge runs # Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges edges_bulge = "equal_number" edges_disc = "equal_number" if ("half_tables" in outputs): nbc_disc.compute(split_half=1,
def diagnostics(y1v2, hoopoe, histograms=True, alpha=True, table=True, vssnr=True, vsredshift=True, rbf=True): if not os.path.exists(config["output_dir"]): print "Warning - output plots directory does not exist" # First make some diagnostic distributions of the observable parameters relevant for the NBC if histograms: pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output_dir"]) pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, data2=y1v2.res, outdir="%s/release/outputs"%config["output_dir"]) if alpha: # Global fit of alpha to compare with previous runs plt.close() b = di.get_alpha(hoopoe.res, hoopoe.res, nbins=20, xlim=(-0.015,0.025), binning="equal_number", use_weights=False, visual=True) plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1) plt.title("Y1 v2 Sim, PSF v02") os.system("mkdir -p %s/release/alpha"%config["output_dir"]) plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit.png"%config["output_dir"]) plt.close() #Exit here if no calibration diagnostics are needed if (not table) and (not vssnr) and (not vsredshift): return 0 # Now compute the calibration using a random half of the simulation nbc = cal.nbc() nbc_disc = cal.nbc() nbc_bulge = cal.nbc() nbc.get_split_data(hoopoe) nbc_disc.load_from_cat(nbc, name="all") nbc_bulge.load_from_cat(nbc, name="all") #### Process disc then bulge runs # Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges if table: nbc_disc.compute(split_half=1, fit="disc", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"]) nbc_bulge.compute(split_half=1, fit="bulge", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"]) nbc_disc.compute(split_half=0, fit="disc", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"]) nbc_bulge.compute(split_half=0, fit="bulge", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"]) # Now plot out the points and the resulting smooth fit if vssnr: if rbf: nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"]) nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"]) else: nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"]) nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"]) nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"]) nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"]) nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="m", do_half=1, output="%s/m-vs-snr-disc-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf) nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="a", do_half=1, output="%s/alpha-vs-snr-disc-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf) plt.close() nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="m", do_half=2, output="%s/m-vs-snr-disc-v1-2.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf) nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="a", do_half=2, output="%s/alpha-vs-snr-disc-v1-2.png"%(config["output_dir"]+"release"+"/rbf"*rbf), use_rbf=rbf) plt.close() nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="m", do_half=1, output="%s/m-vs-snr-bulge-v1-2.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf) nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="a", do_half=1, output="%s/alpha-vs-snr-bulge-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf) plt.close() nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="m", do_half=2, output="%s/m-vs-snr-bulge-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf) nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="a", do_half=2, output="%s/alpha-vs-snr-bulge-v1-2.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf) plt.close() if rbf: nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"]) nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"]) else: nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"]) nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"]) nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"]) nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"]) # Apply to the other half nbc_disc.apply(split_half=2, use_rbf=rbf) nbc_bulge.apply(split_half=2, use_rbf=rbf) nbc.get_combined_calibration(nbc_disc,nbc_bulge, split_half=2) # Finally save some diagnostic plots in tomographic bins if vsredshift: nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o","D"], colour="steelblue") nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple") plt.ylabel("Multiplicative Bias $m$") plt.legend(loc="lower left") plt.savefig("%s/release/%s/m-vs-redshift-diagnostic-v1.png"%(config["output_dir"],rbf*"/rbf")) plt.close() nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=3, fmt=["o","D"], colour="steelblue", error_type="std") nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=3, apply_calibration=True, colour="purple", error_type="std") plt.ylabel(r"PSF Leakage $\alpha$") plt.legend(loc="upper left") plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1.png"%(config["output_dir"],rbf*"/rbf")) plt.close()
def diagnostics(y1v2, hoopoe, histograms=True, split_method="random", weights=None, alpha=True, table=True, half_tables=True, vssnr=True, vsredshift=True, rbf=True, simple_grid=False, smoothing=3, config=None, names=["m", "a"], sbins=16, rbins=11): if rbf: sub_dir="rbf" elif simple_grid: sub_dir="grid" else: sub_dir="polynomial" if not os.path.exists(config["output_dir"]): print "Warning - output plots directory does not exist" # First make some diagnostic distributions of the observable parameters relevant for the NBC if histograms: pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output_dir"], weights=weights) pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs"%config["output_dir"], weights=weights) if alpha: # Global fit of alpha to compare with previous runs plt.close() b = bias=di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22" ], xlim=(-0.03,0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True) plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1) plt.title("Y1 v2 Sim, PSF v02") os.system("mkdir -p %s/release/alpha"%config["output_dir"]) plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png"%config["output_dir"]) plt.close() #Exit here if no calibration diagnostics are needed if (not table) and (not vssnr) and (not vsredshift): return 0 # Now compute the calibration using a random half of the simulation if vssnr or half_tables: nbc = cal.nbc() nbc_disc = cal.nbc() nbc_bulge = cal.nbc() wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method) nbc_disc.load_from_cat(nbc, name="all") nbc_bulge.load_from_cat(nbc, name="all") #### Process disc then bulge runs # Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges if table: if ("match_gridpoints" in config.keys()): redges_bulge, sedges_bulge = bin_edges_from_table(config["match_gridpoints"], type="bulge") redges_disc, sedges_disc = bin_edges_from_table(config["match_gridpoints"], type="disc") edges_disc = (redges_disc, sedges_disc) edges_bulge = (redges_bulge, sedges_bulge) else: edges_bulge = "equal_number" edges_disc = "equal_number" if half_tables: nbc_disc.compute(split_half=1, fit="disc", weights=wt1, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_disc, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method, sbins,rbins)) nbc_bulge.compute(split_half=1, fit="bulge", weights=wt1, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_bulge, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method, sbins,rbins)) if config["full_catalogue"]: nbc_disc.compute(split_half=0, fit="disc", weights=weights, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_disc, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], sbins, rbins)) nbc_bulge.compute(split_half=0, fit="bulge", weights=weights, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_bulge, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], sbins,rbins)) # Now plot out the points and the resulting smoothing fit if vssnr: if rbf: nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing) nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing) else: nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins)) nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins)) nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins)) nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins)) nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-disc-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+sub_dir, smoothing, sbins, rbins), use_rbf=rbf) nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-disc-v1-1.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf) plt.close() nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-disc-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf) nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-disc-v1-2.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf) plt.close() nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-bulge-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf) nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-bulge-v1-1.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf) plt.close() nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-bulge-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf) nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-bulge-v1-2.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf) plt.close() if half_tables and rbf: nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins), smoothing=smoothing) nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins), smoothing=smoothing) elif half_tables and simple_grid: nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))[1].read() nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))[1].read() elif half_tables and (not rbf) and (not simple_grid): nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins)) nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins)) nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins)) nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins)) # Apply to the other half if rbf: scheme="rbf" elif simple_grid: scheme="grid" else: scheme="polynomial" if half_tables: nbc_disc.apply(split_half=2, scheme=scheme, names=names) nbc_bulge.apply(split_half=2, scheme=scheme, names=names) nbc.combine_bd(nbc_disc,nbc_bulge, split_half=2, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) ) # Finally save some diagnostic plots in tomographic bins if vsredshift: zbins=[ 0.2, 0.43, 0.63, 0.9, 1.3] tophat = config["tophat_binning"] import pdb ; pdb.set_trace() if half_tables: plt.close() if "m" in names: bias0=nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat, separate_components=False) bias=nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^",">"], apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat, separate_components=False) plt.ylabel("Multiplicative Bias $m$") plt.legend(loc="center right") plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat))) plt.close() if "a" in names: nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat) nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=3, apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat) plt.ylabel(r"PSF Leakage $\alpha$") plt.legend(loc="upper left") plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat))) plt.close() # Finally redo the fits with the full catalogue nbc = cal.nbc() nbc_disc = cal.nbc() nbc_bulge = cal.nbc() wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights) nbc_disc.load_from_cat(nbc, name="all") nbc_bulge.load_from_cat(nbc, name="all") if rbf: nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing) nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing) elif simple_grid: nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))[1].read() nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))[1].read() else: nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins)) nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins)) nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins)) nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins)) nbc_disc.apply(split_half=0, scheme=scheme, names=names) nbc_bulge.apply(split_half=0, scheme=scheme, names=names) nbc.combine_bd(nbc_disc,nbc_bulge, split_half=0, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) ) plt.close() if "m" in names: bias0 = nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False) bias = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False) plt.ylabel("Multiplicative Bias $m$") plt.legend(loc="center right") plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"],sub_dir, smoothing, sbins, rbins, int(tophat))) plt.close() #Save the calibration data so we can reporoduce the paper plots quickly without rerunning all of the above out = fi.FITS("hoopoe-v2-nbc-%s.fits"%scheme, "rw") dat = np.empty(nbc.res.size, dtype=[("coadd_objects_id", int), ("m", float), ("c1", float), ("c2", float) ]) for name in ["coadd_objects_id", "m", "c1", "c2"]: dat[name]=nbc.res[name] out.write(dat) out[-1].write_key("EXTNAME", "nbc_col") out[-1].write_key("METHOD", scheme) out.close() np.savetxt("m-vs-z-dvec-fullcat-uncalibrated-tophatbins.txt", np.array(bias0).T, header="z m1 em1 m2 em2 m em") np.savetxt("shear_pipeline/plot_dump/datavecs/m-vs-z-dvec-fullcat-%s-nbc-tophatbins.txt"%scheme, np.array(bias).T, header="z m1 em1 m2 em2 m em") rewt=di.get_weights_to_match(y1v2.res["mean_rgpp_rp"], hoopoe.res["mean_rgpp_rp"],nbins=25) bmask = (hoopoe.res["is_bulge"]==1) mask = (hoopoe.res["bulge_flux"]<10) & (hoopoe.res["disc_flux"]<10) rewtb=di.get_weights_to_match(y1v2.res["bulge_flux"][y1v2.res["bulge_flux"]!=0], hoopoe.res["bulge_flux"][bmask & mask],nbins=25, xlim=(0.0,10.)) rewtd=di.get_weights_to_match(y1v2.res["disc_flux"][y1v2.res["bulge_flux"]==0], hoopoe.res["disc_flux"][np.invert(bmask) & mask],nbins=25, xlim=(0.0,10.)) wts_flux = np.zeros(hoopoe.res.size) wts_flux[bmask & mask] = rewtb wts_flux[np.invert(bmask) & mask] = rewtd wts_flux = wts_flux[mask] bias_size_wtd = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple", weights=rewt, split_half=0, bins=zbins, tophat=tophat, separate_components=False) bias_flux_wtd = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple", weights=wts_flux, split_half=0, bins=zbins, tophat=tophat, separate_components=False) out=np.vstack((bias_size_wtd[0], bias_size_wtd[-2], bias_size_wtd[-1], bias[-2], bias[-1])) np.savetxt("m-vs-z_sizewtd-%s.txt"%scheme, out.T) out=np.vstack((bias_flux_wtd[0], bias_flux_wtd[-2], bias_flux_wtd[-1], bias[-2], bias[-1])) np.savetxt("m-vs-z_fluxwtd-%s.txt"%scheme, out.T) if "a" in names: nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat) nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=3, apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat) plt.ylabel(r"PSF Leakage $\alpha$") plt.legend(loc="upper left") plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"],sub_dir, smoothing, sbins, rbins, int(tophat))) plt.close()
def do_halfcat_tests(catalogue, weights, params, scheme="grid", split_type="cosmos", sbins=16, rbins=16, smoothing=3): # Generate calibrator objects, then split the data nbc = cal.nbc() nbc_disc = cal.nbc() nbc_bulge = cal.nbc() wt1, wt2 = nbc.get_split_data(catalogue, weights=weights, method=split_type) nbc_disc.load_from_cat(nbc, name="all") nbc_bulge.load_from_cat(nbc, name="all") gc.collect() import pdb pdb.set_trace() # Do any setting up that needs doing prior to applying the calibration to the catalogue nbc_disc, nbc_bulge = setup(nbc_disc, nbc_bulge, split_type, scheme, params["output_dir"], params["sbins"], params["rbins"], smoothing) # Apply the calibration from one half of the catalogue to the second part nbc_disc.apply(split_half=2, scheme=scheme, names=["m", "a"]) nbc_bulge.apply(split_half=2, scheme=scheme, names=["m", "a"]) nbc.combine_bd(nbc_disc, nbc_bulge, split_half=2, names=["m", "c1", "c2"]) # Finally test for residual bias zbins = [0.2, 0.43, 0.63, 0.9, 1.3] bias0 = nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=4, fmt=["o", "D"], colour="steelblue", weights=wt2, bins=zbins, tophat=False, separate_components=False) bias = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^", ">"], apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=False, separate_components=False) # Save the residual bias numbers to a text file np.savetxt( "/home/samuroff/shear_pipeline/plot_dump/datavecs/m-vs-z-dvec-%s-halfcat-uncalibrated.txt" % (split_type), np.array(bias0).T, header="z m1 em1 m2 em2 m em") np.savetxt( "/home/samuroff/shear_pipeline/plot_dump/datavecs/m-vs-z-dvec-%s-halfcat-%s-nbc-sbins%d-rbins%d.txt" % (split_type, scheme, params["sbins"], params["rbins"]), np.array(bias).T, header="z m1 em1 m2 em2 m em") print "Done" return 0