def generate_difference_table(config1, config2, sbins=16, rbins=16, y1v2=None):
hoopoe1, weights1, y1v2 = ct.setup(True, True, config1, y1v2=y1v2)
hoopoe2, weights2, y1v2 = ct.setup(True, False, config2, y1v2=y1v2)
subset = np.in1d(hoopoe1.res["coadd_objects_id"],
hoopoe2.res["coadd_objects_id"])
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
wt1, wt2 = nbc.get_split_data(hoopoe1, weights=weights1, method="random")
nbc_disc.load_from_cat(nbc, name="all")
nbc_bulge.load_from_cat(nbc, name="all")
import pdb
pdb.set_trace()
nbc_disc.diff_table(weights=weights1,
compare_with_subset=subset,
fit="disc",
table_name="diff_table_disc.fits",
sbins=sbins,
rbins=rbins,
binning="equal_number",
rlim=(1.13, 3),
slim=(12, 200))
nbc_bulge.diff_table(weights=weights2,
compare_with_subset=subset,
fit="bulge",
table_name="diff_table_bulge.fits",
sbins=sbins,
rbins=rbins,
binning="equal_number",
rlim=(1.13, 3),
slim=(12, 200))
def calibrate(data, method="grid", config=None, smoothing=3, sbins=16, rbins=16, nthread=1, rank=0):
rbf = (method.lower()=="rbf")
# Set up the wrappers
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc.res = data.res
del(data.res)
gc.collect()
nbc_bulge = cal.nbc()
nbc_bulge.res=nbc.res[nbc.res["is_bulge"].astype(bool)]
nbc_disc.res=nbc.res[np.invert(nbc.res["is_bulge"].astype(bool))]
# Fit or interpolate
if method is "polynomial":
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
elif method is "rbf":
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
elif method is "grid":
nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read()
nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read()
# Apply to get a bias correction for each galaxy in the data
nbc_disc.apply(split_half=0, scheme=method)
print "Done disc calibration"
nbc_bulge.apply(split_half=0, scheme=method)
print "Done bulge calibration"
print "Merging bulge/disc results...",
nbc.res = arr.add_col(nbc.res, "m")
nbc.res = arr.add_col(nbc.res, "c1")
nbc.res = arr.add_col(nbc.res, "c2")
nbc.res = arr.add_col(nbc.res, "a")
nbc.res["m"][np.invert(nbc.res["is_bulge"].astype(bool))] = nbc_disc.res["m"]
nbc.res["c1"][np.invert(nbc.res["is_bulge"].astype(bool))] = nbc_disc.res["c1"]
nbc.res["c2"][np.invert(nbc.res["is_bulge"].astype(bool))] = nbc_disc.res["c2"]
nbc.res["a"][np.invert(nbc.res["is_bulge"].astype(bool))] = nbc_disc.res["a"]
nbc.res["m"][nbc.res["is_bulge"].astype(bool)] = nbc_bulge.res["m"]
nbc.res["c1"][nbc.res["is_bulge"].astype(bool)] = nbc_bulge.res["c1"]
nbc.res["c2"][nbc.res["is_bulge"].astype(bool)] = nbc_bulge.res["c2"]
nbc.res["a"][nbc.res["is_bulge"].astype(bool)] = nbc_bulge.res["a"]
print "done"
if not os.path.exists(os.path.dirname(config["output"]["dir"])):
os.system("mkdir -p %s"%os.path.dirname(config["output"]["dir"]))
print "Saving calibrated catalogue to %s"%config["output"]["filename"]
nbc.export(filename="%s/%s"%(config["output"]["dir"],config["output"]["filename"]))
def calibrate(data, method="rbf"):
rbf = (method.lower()=="rbf")
# Set up the wrappers
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_disc.res = data.res
nbc_bulge = cal.nbc()
# Fit or interpolate
if method is "polynomial":
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
elif method is "rbf":
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
# Apply to get a bias correction for each galaxy in the data
nbc_disc.apply(split_half=0, use_rbf=rbf)
nbc.res = nbc_disc.res
del(nbc_disc.res)
print "Done bulge calibration"
nbc_bulge.res = data.res
nbc_bulge.apply(split_half=0, use_rbf=rbf)
nbc.res[nbc.res["is_bulge"].astype(bool)] = nbc_bulge.res[nbc.res["is_bulge"].astype(bool)]
del(nbc_bulge.res)
print "Done disc calibration"
if not os.path.exists(os.path.dirname(config["output_dir"])):
os.system("mkdir -p %s"%os.path.dirname(config["output_dir"]))
print "Saving calibrated catalogue to %s"%config["output_catalogue"]
nbc.export(filename="%s/%s"%(config["output_dir"],config["output_catalogue"]))
def calculate(y1v2, hoopoe, split_method="none", weights=None, outputs=[], method="grid", smoothing=3, config=None, names=["m", "a"], sbins=16, rbins=16, nthread=1, rank=0, task_allocation={0 : ["disc_table", "bulge_table", "disc_htable", "bulge_htable"]} ):
if not os.path.exists(config["output"]["dir"]):
print "Warning - output plots directory does not exist"
# First make some diagnostic distributions of the observable parameters relevant for the NBC
if ("histograms" in outputs):
pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output"]["dir"], weights=weights)
pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs"%config["output"]["dir"], weights=weights)
if ("alpha" in outputs):
# Global fit of alpha to compare with previous runs
plt.close()
b = bias=di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22" ], xlim=(-0.03,0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True)
plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
plt.title("Y1 v2 Sim, PSF v02")
os.system("mkdir -p %s/release/alpha"%config["output"]["dir"])
plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png"%config["output"]["dir"])
plt.close()
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
# split the data into two subsets
wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method)
nbc_disc.load_from_cat(nbc, name="all")
nbc_bulge.load_from_cat(nbc, name="all")
#### Process disc then bulge runs
# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
edges_bulge = "equal_number"
edges_disc = "equal_number"
if ("half_tables" in outputs):
if ("disc_htable" in task_allocation[rank]):
nbc_disc.compute(split_half=1,
fit="disc", weights=wt1,
use_catalogue_weights=config["selection"]["weights"],
reweight_per_bin=False, resample_per_bin=False,
refdata=y1v2, binning=edges_disc,
rbins=rbins, sbins=sbins,
rlim=(1.13,3.0), slim=(12,200),
table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method, sbins,rbins))
if ("bulge_htable" in task_allocation[rank]):
nbc_bulge.compute(split_half=1,
fit="bulge", weights=wt1,
use_catalogue_weights=config["selection"]["weights"],
reweight_per_bin=False, resample_per_bin=False,
refdata=y1v2, binning=edges_bulge,
rbins=rbins, sbins=sbins,
rlim=(1.13,3.0), slim=(12,200),
table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method, sbins,rbins))
if ("tables" in outputs):
if ("disc_table" in task_allocation[rank]):
nbc_disc.compute(split_half=0,
fit="disc", weights=weights,
use_catalogue_weights=config["selection"]["weights"],
reweight_per_bin=False, resample_per_bin=False,
refdata=y1v2, binning=edges_disc,
rbins=rbins, sbins=sbins,
rlim=(1.13,3.0), slim=(12,200),
table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], sbins, rbins))
if ("bulge_table" in task_allocation[rank]):
nbc_bulge.compute(split_half=0,
fit="bulge", weights=weights,
use_catalogue_weights=config["selection"]["weights"],
reweight_per_bin=False, resample_per_bin=False,
refdata=y1v2, binning=edges_bulge,
rbins=rbins, sbins=sbins,
rlim=(1.13,3.0), slim=(12,200),
table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], sbins,rbins))
sub_dir = method.lower()
# Now plot out the points and the resulting smoothing fit
if ("snr" in outputs):
if (method.lower()=="rbf") or (method.lower()=="grid"):
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
elif (method.lower()=="polynomial"):
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-disc-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-disc-v1-1.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-disc-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-disc-v1-2.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-bulge-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-bulge-v1-1.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-bulge-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-bulge-v1-2.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
if (split_method!="none"):
# Derive a calibration from the grid in whichever way is specified in the config file
if (method=="rbf"):
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins), smoothing=smoothing)
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins), smoothing=smoothing)
elif (method=="grid"):
nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))[1].read()
nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))[1].read()
elif (method=="polynomial"):
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))
# Apply it to the other half
nbc_disc.apply(split_half=2, scheme=method, names=names)
nbc_bulge.apply(split_half=2, scheme=method, names=names)
nbc.combine_bd(nbc_disc,nbc_bulge, split_half=2, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )
# Finally save some diagnostic plots in tomographic bins
if ("redshift" in outputs):
zbins=[ 0.2, 0.43, 0.63, 0.9, 1.3]
tophat = config["calibration"]["ztype"]=="tophat"
# Only do the redshift plots with the split catalogues if they were recalculated this time
plt.close()
if "m" in names:
bias0=nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
bias=nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^",">"], apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
plt.ylabel("Multiplicative Bias $m$")
plt.legend(loc="center right")
plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
plt.close()
if "a" in names:
nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat)
nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=4, apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat)
plt.ylabel(r"PSF Leakage $\alpha$")
plt.legend(loc="upper left")
plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
plt.close()
np.savetxt("%s/data/m-vs-z-uncalibrated-%s-halfcat-unblinded%s.txt"%(config["output"]["dir"],split_method, ""+"-tophat"*int(tophat)), np.array(bias0))
np.savetxt("%s/data/m-vs-z-%s_nbc-%s-halfcat-unblinded%s.txt"%(config["output"]["dir"],method,split_method, ""+"-tophat"*int(tophat)),np.array(bias))
# Finally redo the fits with the full catalogue
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights)
nbc_disc.load_from_cat(nbc, name="all")
nbc_bulge.load_from_cat(nbc, name="all")
if (method.lower()=="rbf"):
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
elif (method.lower()=="grid"):
nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read()
nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read()
elif (method.lower()=="polynomial"):
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_disc.apply(split_half=0, scheme=method, names=names)
nbc_bulge.apply(split_half=0, scheme=method, names=names)
nbc.combine_bd(nbc_disc,nbc_bulge, split_half=0, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )
nbc.preserve_calibration_col("%s/data/hoopoe-%s-nbc_column.fits"%(config["output"]["dir"],method))
plt.close()
if "m" in names:
bias0 = nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
bias = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^",">"], apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
plt.ylabel("Multiplicative Bias $m$")
plt.legend(loc="center right")
plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
plt.close()
np.savetxt("%s/data/m-vs-z-uncalibrated-fullcat-unblinded%s.txt"%(config["output"]["dir"], ""+"-tophat"*int(tophat)), np.array(bias0))
np.savetxt("%s/data/m-vs-z-%s_nbc-fullcat-unblinded%s.txt"%(config["output"]["dir"], method,""+"-tophat"*int(tophat)), np.array(bias))
if "a" in names:
nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat)
nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=4, apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat)
plt.ylabel(r"PSF Leakage $\alpha$")
plt.legend(loc="upper left")
plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
plt.close()
print "Done"
return 0
def calibrate_in_bins(nbc,
halfcats=False,
split_method="",
scheme="rbf",
config=None,
sbins=12,
rbins=9,
smoothing=3,
names=["m", "a"]):
discs = cal.nbc()
bulges = cal.nbc()
if halfcats:
split_half = 2
prefix = "%s-halfcat" % split_method
else:
split_half = 0
prefix = "fullcat"
zbin_numbers = np.unique(nbc.res["des_bin"])
print "Calibrating in %d redshift bins" % zbin_numbers.size
for k in zbin_numbers:
print "-- calibrating galaxies in bin %d" % k
if halfcats:
select_bin1 = nbc.res1["des_bin"] == k
select_bin = nbc.res2["des_bin"] == k
discs.res1 = copy.deepcopy(nbc.res1[select_bin1])
discs.truth1 = copy.deepcopy(nbc.truth1[select_bin1])
bulges.res1 = copy.deepcopy(nbc.res1[select_bin1])
bulges.truth1 = copy.deepcopy(nbc.truth1[select_bin1])
discs.res2 = copy.deepcopy(nbc.res2[select_bin])
discs.truth2 = copy.deepcopy(nbc.truth2[select_bin])
bulges.res2 = copy.deepcopy(nbc.res2[select_bin])
bulges.truth2 = copy.deepcopy(nbc.truth2[select_bin])
else:
select_bin = nbc.res["des_bin"] == k
discs.res = copy.deepcopy(nbc.res[select_bin])
discs.truth = copy.deepcopy(nbc.truth[select_bin])
bulges.res = copy.deepcopy(nbc.res[select_bin])
bulges.truth = copy.deepcopy(nbc.truth[select_bin])
if (scheme == "rbf"):
discs.fit_rbf(
table=
"%s/nbc_data/bias_table_hoopoe-v1-%s-disc-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], prefix, sbins, rbins, k),
smoothing=smoothing)
bulges.fit_rbf(
table=
"%s/nbc_data/bias_table_hoopoe-v1-%s-bulge-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], prefix, sbins, rbins, k),
smoothing=smoothing)
elif (scheme == "grid"):
discs.bias_grid = fi.FITS(
"%s/nbc_data/bias_table_hoopoe-v1-%s-disc-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], prefix, sbins, rbins, k))[1].read()
bulges.bias_grid = fi.FITS(
"%s/nbc_data/bias_table_hoopoe-v1-%s-bulge-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], prefix, sbins, rbins, k))[1].read()
elif (scheme == "polynomial"):
discs.fit(
"m",
table=
"%s/nbc_data/bias_table_hoopoe-v1-%s-disc-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], prefix, sbins, rbins, k))
discs.fit(
"a",
table=
"%s/nbc_data/bias_table_hoopoe-v1-%s-disc-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], prefix, sbins, rbins, k))
bulges.fit(
"m",
table=
"%s/nbc_data/bias_table_hoopoe-v1-%s-bulge-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], prefix, sbins, rbins, k))
bulges.fit(
"a",
table=
"%s/nbc_data/bias_table_hoopoe-v1-%s-bulge-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], prefix, sbins, rbins, k))
discs.apply(split_half=split_half, scheme=scheme, names=names)
bulges.apply(split_half=split_half, scheme=scheme, names=names)
nbc.combine_bd(discs,
bulges,
split_half=split_half,
mask=select_bin,
names=["m"] * ("m" in names) + ["c1", "c2"] *
("a" in names))
return nbc
def diagnostics(y1v2,
hoopoe,
tomographic_calibration=False,
histograms=True,
split_method="random",
weights=None,
alpha=True,
table=True,
half_tables=True,
vssnr=True,
vsredshift=True,
rbf=True,
simple_grid=False,
smoothing=3,
config=None,
names=["m", "a"],
sbins=16,
rbins=11):
if rbf:
sub_dir = "rbf"
elif simple_grid:
sub_dir = "grid"
else:
sub_dir = "polynomial"
if not os.path.exists(config["output_dir"]):
print "Warning - output plots directory does not exist"
# First make some diagnostic distributions of the observable parameters relevant for the NBC
if histograms:
pl.histograms_vs_input(["e", "size", "flux"],
hoopoe.truth,
data2=y1v2.res,
outdir="%s/release/inputs" %
config["output_dir"],
weights=weights)
pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"],
hoopoe.res,
kl=True,
data2=y1v2.res,
outdir="%s/release/outputs" % config["output_dir"],
weights=weights)
if alpha:
# Global fit of alpha to compare with previous runs
plt.close()
b = bias = di.get_alpha(hoopoe.res,
hoopoe.res,
nbins=15,
names=["alpha11", "alpha22"],
xlim=(-0.03, 0.02),
xdata_name="mean_hsm_psf_e%d_sky",
weights=weights,
visual=True)
plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
plt.title("Y1 v2 Sim, PSF v02")
os.system("mkdir -p %s/release/alpha" % config["output_dir"])
plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png" %
config["output_dir"])
plt.close()
#Exit here if no calibration diagnostics are needed
if (not table) and (not vssnr) and (not vsredshift):
return 0
# Now compute the calibration using a random half of the simulation
if vssnr or half_tables:
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
wt1, wt2 = nbc.get_split_data(hoopoe,
weights=weights,
method=split_method)
nbc_disc.load_from_cat(nbc, name="all")
nbc_bulge.load_from_cat(nbc, name="all")
#### Process disc then bulge runs
# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
import pdb
pdb.set_trace()
if table:
if ("match_gridpoints" in config.keys()):
redges_bulge, sedges_bulge = bin_edges_from_table(
config["match_gridpoints"], type="bulge")
redges_disc, sedges_disc = bin_edges_from_table(
config["match_gridpoints"], type="disc")
edges_disc = (redges_disc, sedges_disc)
edges_bulge = (redges_bulge, sedges_bulge)
else:
edges_bulge = "equal_number"
edges_disc = "equal_number"
if tomographic_calibration:
zbin_numbers = np.unique(nbc.res["des_bin"])
if half_tables:
print "Calculating half tables in %d redshift bins" % zbin_numbers.size
for k in zbin_numbers:
print "bin %d" % k
nbc_disc.compute(
split_half=1,
fit="disc",
weights=wt1,
reweight_per_bin=config["reweight_perbin"],
resample_per_bin=config["resample_perbin"],
refdata=y1v2,
binning=edges_disc,
redshift_bin=k,
rbins=rbins,
sbins=sbins,
rlim=(1.13, 3.0),
slim=(12, 200),
table_name=
"%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], split_method, sbins, rbins, k))
nbc_bulge.compute(
split_half=1,
fit="bulge",
weights=wt1,
reweight_per_bin=config["reweight_perbin"],
resample_per_bin=config["resample_perbin"],
refdata=y1v2,
binning=edges_bulge,
redshift_bin=k,
rbins=rbins,
sbins=sbins,
rlim=(1.13, 3.0),
slim=(12, 200),
table_name=
"%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], split_method, sbins, rbins, k))
if table:
print "Calculating full tables in %d redshift bins" % zbin_numbers.size
for k in zbin_numbers:
print "bin %d" % k
nbc_disc.compute(
split_half=0,
fit="disc",
weights=weights,
reweight_per_bin=config["reweight_perbin"],
resample_per_bin=config["resample_perbin"],
refdata=y1v2,
binning=edges_disc,
redshift_bin=k,
rbins=rbins,
sbins=sbins,
rlim=(1.13, 3.0),
slim=(12, 200),
table_name=
"%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], sbins, rbins, k))
nbc_bulge.compute(
split_half=0,
fit="bulge",
weights=weights,
reweight_per_bin=config["reweight_perbin"],
resample_per_bin=config["resample_perbin"],
refdata=y1v2,
binning=edges_bulge,
redshift_bin=k,
rbins=rbins,
sbins=sbins,
rlim=(1.13, 3.0),
slim=(12, 200),
table_name=
"%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], sbins, rbins, k))
if rbf:
scheme = "rbf"
elif simple_grid:
scheme = "grid"
else:
scheme = "polynomial"
if half_tables:
nbc = calibrate_in_bins(nbc,
halfcats=True,
split_method=split_method,
scheme=scheme,
config=config,
sbins=sbins,
rbins=rbins,
smoothing=smoothing,
names=names)
# Finally save some diagnostic plots in tomographic bins
if vsredshift:
zbins = [0.2, 0.43, 0.63, 0.9, 1.3]
tophat = config["tophat_binning"]
if half_tables:
plt.close()
if "m" in names:
nbc.redshift_diagnostic(bias="m",
label="Uncalibrated",
ls="none",
nbins=3,
fmt=["o", "D"],
colour="steelblue",
weights=wt2,
bins=zbins,
tophat=tophat,
separate_components=False)
nbc.redshift_diagnostic(bias="m",
label="Calibrated",
ls="none",
nbins=3,
fmt=["^", ">"],
apply_calibration=True,
colour="purple",
weights=wt2,
bins=zbins,
tophat=tophat,
separate_components=False)
plt.ylabel("Multiplicative Bias $m$")
plt.legend(loc="center right")
plt.savefig(
"%s/release/%s/m-bias-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
% (config["output_dir"], sub_dir, split_method, smoothing,
sbins, rbins, int(tophat)))
plt.close()
if "a" in names:
nbc.redshift_diagnostic(bias="alpha",
label="Uncalibrated",
ls="none",
nbins=3,
fmt=["o", "D"],
colour="steelblue",
weights=wt2,
bins=zbins,
tophat=tophat)
nbc.redshift_diagnostic(bias="alpha",
label="Calibrated",
ls="none",
fmt=["^", ">"],
nbins=3,
apply_calibration=True,
colour="purple",
weights=wt2,
bins=zbins,
tophat=tophat)
plt.ylabel(r"PSF Leakage $\alpha$")
plt.legend(loc="upper left")
plt.savefig(
"%s/release/%s/alpha-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
% (config["output_dir"], sub_dir, split_method, smoothing,
sbins, rbins, int(tophat)))
plt.close()
# Finally redo the fits with the full catalogue
gc.collect()
nbc = cal.nbc()
wt1, wt2 = nbc.get_split_data(hoopoe,
weights=weights,
method=split_method)
nbc = calibrate_in_bins(nbc,
halfcats=False,
split_method=None,
scheme=scheme,
config=config,
sbins=sbins,
rbins=rbins,
smoothing=smoothing,
names=names)
import pdb
pdb.set_trace()
plt.close()
if "m" in names:
nbc.redshift_diagnostic(bias="m",
label="Uncalibrated",
ls="none",
nbins=3,
fmt=["o", "D"],
colour="steelblue",
weights=weights,
split_half=0,
bins=zbins,
tophat=tophat,
separate_components=False)
bias = nbc.redshift_diagnostic(bias="m",
label="Calibrated",
ls="none",
nbins=3,
fmt=["^", ">"],
apply_calibration=True,
colour="purple",
weights=weights,
split_half=0,
bins=zbins,
tophat=tophat,
separate_components=False)
plt.ylabel("Multiplicative Bias $m$")
plt.legend(loc="center right")
plt.savefig(
"%s/release/%s/m-bias-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
% (config["output_dir"], sub_dir, smoothing, sbins, rbins,
int(tophat)))
plt.close()
if "a" in names:
nbc.redshift_diagnostic(bias="alpha",
label="Uncalibrated",
ls="none",
nbins=3,
fmt=["o", "D"],
colour="steelblue",
weights=weights,
split_half=0,
bins=zbins,
tophat=tophat)
nbc.redshift_diagnostic(bias="alpha",
label="Calibrated",
ls="none",
fmt=["^", ">"],
nbins=3,
apply_calibration=True,
colour="purple",
weights=weights,
split_half=0,
bins=zbins,
tophat=tophat)
plt.ylabel(r"PSF Leakage $\alpha$")
plt.legend(loc="upper left")
plt.savefig(
"%s/release/%s/alpha-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
% (config["output_dir"], sub_dir, smoothing, sbins, rbins,
int(tophat)))
plt.close()
pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs"%config["output"]["dir"], weights=weights)
if ("alpha" in outputs):
# Global fit of alpha to compare with previous runs
plt.close()
b = bias=di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22" ], xlim=(-0.03,0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True)
plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
plt.title("Y1 v2 Sim, PSF v02")
os.system("mkdir -p %s/release/alpha"%config["output_dir"])
plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png"%config["output_dir"])
plt.close()
# split the data into two subsets
if ("snr" in outputs) or ("half_tables" in outputs):
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method)
nbc_disc.load_from_cat(nbc, name="all")
nbc_bulge.load_from_cat(nbc, name="all")
#### Process disc then bulge runs
# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
edges_bulge = "equal_number"
edges_disc = "equal_number"
if ("half_tables" in outputs):
nbc_disc.compute(split_half=1,
def diagnostics(y1v2, hoopoe, histograms=True, alpha=True, table=True, vssnr=True, vsredshift=True, rbf=True):
if not os.path.exists(config["output_dir"]):
print "Warning - output plots directory does not exist"
# First make some diagnostic distributions of the observable parameters relevant for the NBC
if histograms:
pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output_dir"])
pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, data2=y1v2.res, outdir="%s/release/outputs"%config["output_dir"])
if alpha:
# Global fit of alpha to compare with previous runs
plt.close()
b = di.get_alpha(hoopoe.res, hoopoe.res, nbins=20, xlim=(-0.015,0.025), binning="equal_number", use_weights=False, visual=True)
plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
plt.title("Y1 v2 Sim, PSF v02")
os.system("mkdir -p %s/release/alpha"%config["output_dir"])
plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit.png"%config["output_dir"])
plt.close()
#Exit here if no calibration diagnostics are needed
if (not table) and (not vssnr) and (not vsredshift):
return 0
# Now compute the calibration using a random half of the simulation
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
nbc.get_split_data(hoopoe)
nbc_disc.load_from_cat(nbc, name="all")
nbc_bulge.load_from_cat(nbc, name="all")
#### Process disc then bulge runs
# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
if table:
nbc_disc.compute(split_half=1, fit="disc", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
nbc_bulge.compute(split_half=1, fit="bulge", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])
nbc_disc.compute(split_half=0, fit="disc", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
nbc_bulge.compute(split_half=0, fit="bulge", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
# Now plot out the points and the resulting smooth fit
if vssnr:
if rbf:
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
else:
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="m", do_half=1, output="%s/m-vs-snr-disc-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="a", do_half=1, output="%s/alpha-vs-snr-disc-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
plt.close()
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="m", do_half=2, output="%s/m-vs-snr-disc-v1-2.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="a", do_half=2, output="%s/alpha-vs-snr-disc-v1-2.png"%(config["output_dir"]+"release"+"/rbf"*rbf), use_rbf=rbf)
plt.close()
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="m", do_half=1, output="%s/m-vs-snr-bulge-v1-2.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="a", do_half=1, output="%s/alpha-vs-snr-bulge-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
plt.close()
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="m", do_half=2, output="%s/m-vs-snr-bulge-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="a", do_half=2, output="%s/alpha-vs-snr-bulge-v1-2.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
plt.close()
if rbf:
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])
else:
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])
# Apply to the other half
nbc_disc.apply(split_half=2, use_rbf=rbf)
nbc_bulge.apply(split_half=2, use_rbf=rbf)
nbc.get_combined_calibration(nbc_disc,nbc_bulge, split_half=2)
# Finally save some diagnostic plots in tomographic bins
if vsredshift:
nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o","D"], colour="steelblue")
nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple")
plt.ylabel("Multiplicative Bias $m$")
plt.legend(loc="lower left")
plt.savefig("%s/release/%s/m-vs-redshift-diagnostic-v1.png"%(config["output_dir"],rbf*"/rbf"))
plt.close()
nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=3, fmt=["o","D"], colour="steelblue", error_type="std")
nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=3, apply_calibration=True, colour="purple", error_type="std")
plt.ylabel(r"PSF Leakage $\alpha$")
plt.legend(loc="upper left")
plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1.png"%(config["output_dir"],rbf*"/rbf"))
plt.close()
def diagnostics(y1v2, hoopoe, histograms=True, split_method="random", weights=None, alpha=True, table=True, half_tables=True, vssnr=True, vsredshift=True, rbf=True, simple_grid=False, smoothing=3, config=None, names=["m", "a"], sbins=16, rbins=11):
if rbf:
sub_dir="rbf"
elif simple_grid:
sub_dir="grid"
else:
sub_dir="polynomial"
if not os.path.exists(config["output_dir"]):
print "Warning - output plots directory does not exist"
# First make some diagnostic distributions of the observable parameters relevant for the NBC
if histograms:
pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output_dir"], weights=weights)
pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs"%config["output_dir"], weights=weights)
if alpha:
# Global fit of alpha to compare with previous runs
plt.close()
b = bias=di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22" ], xlim=(-0.03,0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True)
plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
plt.title("Y1 v2 Sim, PSF v02")
os.system("mkdir -p %s/release/alpha"%config["output_dir"])
plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png"%config["output_dir"])
plt.close()
#Exit here if no calibration diagnostics are needed
if (not table) and (not vssnr) and (not vsredshift):
return 0
# Now compute the calibration using a random half of the simulation
if vssnr or half_tables:
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method)
nbc_disc.load_from_cat(nbc, name="all")
nbc_bulge.load_from_cat(nbc, name="all")
#### Process disc then bulge runs
# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
if table:
if ("match_gridpoints" in config.keys()):
redges_bulge, sedges_bulge = bin_edges_from_table(config["match_gridpoints"], type="bulge")
redges_disc, sedges_disc = bin_edges_from_table(config["match_gridpoints"], type="disc")
edges_disc = (redges_disc, sedges_disc)
edges_bulge = (redges_bulge, sedges_bulge)
else:
edges_bulge = "equal_number"
edges_disc = "equal_number"
if half_tables:
nbc_disc.compute(split_half=1, fit="disc", weights=wt1, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_disc, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method, sbins,rbins))
nbc_bulge.compute(split_half=1, fit="bulge", weights=wt1, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_bulge, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method, sbins,rbins))
if config["full_catalogue"]:
nbc_disc.compute(split_half=0, fit="disc", weights=weights, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_disc, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], sbins, rbins))
nbc_bulge.compute(split_half=0, fit="bulge", weights=weights, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_bulge, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], sbins,rbins))
# Now plot out the points and the resulting smoothing fit
if vssnr:
if rbf:
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
else:
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-disc-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-disc-v1-1.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-disc-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-disc-v1-2.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-bulge-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-bulge-v1-1.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-bulge-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-bulge-v1-2.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
if half_tables and rbf:
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins), smoothing=smoothing)
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins), smoothing=smoothing)
elif half_tables and simple_grid:
nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))[1].read()
nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))[1].read()
elif half_tables and (not rbf) and (not simple_grid):
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))
# Apply to the other half
if rbf:
scheme="rbf"
elif simple_grid:
scheme="grid"
else:
scheme="polynomial"
if half_tables:
nbc_disc.apply(split_half=2, scheme=scheme, names=names)
nbc_bulge.apply(split_half=2, scheme=scheme, names=names)
nbc.combine_bd(nbc_disc,nbc_bulge, split_half=2, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )
# Finally save some diagnostic plots in tomographic bins
if vsredshift:
zbins=[ 0.2, 0.43, 0.63, 0.9, 1.3]
tophat = config["tophat_binning"]
import pdb ; pdb.set_trace()
if half_tables:
plt.close()
if "m" in names:
bias0=nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
bias=nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^",">"], apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
plt.ylabel("Multiplicative Bias $m$")
plt.legend(loc="center right")
plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
plt.close()
if "a" in names:
nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat)
nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=3, apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat)
plt.ylabel(r"PSF Leakage $\alpha$")
plt.legend(loc="upper left")
plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
plt.close()
# Finally redo the fits with the full catalogue
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights)
nbc_disc.load_from_cat(nbc, name="all")
nbc_bulge.load_from_cat(nbc, name="all")
if rbf:
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
elif simple_grid:
nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))[1].read()
nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))[1].read()
else:
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_disc.apply(split_half=0, scheme=scheme, names=names)
nbc_bulge.apply(split_half=0, scheme=scheme, names=names)
nbc.combine_bd(nbc_disc,nbc_bulge, split_half=0, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )
plt.close()
if "m" in names:
bias0 = nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
bias = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
plt.ylabel("Multiplicative Bias $m$")
plt.legend(loc="center right")
plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
plt.close()
#Save the calibration data so we can reporoduce the paper plots quickly without rerunning all of the above
out = fi.FITS("hoopoe-v2-nbc-%s.fits"%scheme, "rw")
dat = np.empty(nbc.res.size, dtype=[("coadd_objects_id", int), ("m", float), ("c1", float), ("c2", float) ])
for name in ["coadd_objects_id", "m", "c1", "c2"]: dat[name]=nbc.res[name]
out.write(dat)
out[-1].write_key("EXTNAME", "nbc_col")
out[-1].write_key("METHOD", scheme)
out.close()
np.savetxt("m-vs-z-dvec-fullcat-uncalibrated-tophatbins.txt", np.array(bias0).T, header="z m1 em1 m2 em2 m em")
np.savetxt("shear_pipeline/plot_dump/datavecs/m-vs-z-dvec-fullcat-%s-nbc-tophatbins.txt"%scheme, np.array(bias).T, header="z m1 em1 m2 em2 m em")
rewt=di.get_weights_to_match(y1v2.res["mean_rgpp_rp"], hoopoe.res["mean_rgpp_rp"],nbins=25)
bmask = (hoopoe.res["is_bulge"]==1)
mask = (hoopoe.res["bulge_flux"]<10) & (hoopoe.res["disc_flux"]<10)
rewtb=di.get_weights_to_match(y1v2.res["bulge_flux"][y1v2.res["bulge_flux"]!=0], hoopoe.res["bulge_flux"][bmask & mask],nbins=25, xlim=(0.0,10.))
rewtd=di.get_weights_to_match(y1v2.res["disc_flux"][y1v2.res["bulge_flux"]==0], hoopoe.res["disc_flux"][np.invert(bmask) & mask],nbins=25, xlim=(0.0,10.))
wts_flux = np.zeros(hoopoe.res.size)
wts_flux[bmask & mask] = rewtb
wts_flux[np.invert(bmask) & mask] = rewtd
wts_flux = wts_flux[mask]
bias_size_wtd = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple", weights=rewt, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
bias_flux_wtd = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple", weights=wts_flux, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
out=np.vstack((bias_size_wtd[0], bias_size_wtd[-2], bias_size_wtd[-1], bias[-2], bias[-1]))
np.savetxt("m-vs-z_sizewtd-%s.txt"%scheme, out.T)
out=np.vstack((bias_flux_wtd[0], bias_flux_wtd[-2], bias_flux_wtd[-1], bias[-2], bias[-1]))
np.savetxt("m-vs-z_fluxwtd-%s.txt"%scheme, out.T)
if "a" in names:
nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat)
nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=3, apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat)
plt.ylabel(r"PSF Leakage $\alpha$")
plt.legend(loc="upper left")
plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
plt.close()
def do_halfcat_tests(catalogue,
weights,
params,
scheme="grid",
split_type="cosmos",
sbins=16,
rbins=16,
smoothing=3):
# Generate calibrator objects, then split the data
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
wt1, wt2 = nbc.get_split_data(catalogue,
weights=weights,
method=split_type)
nbc_disc.load_from_cat(nbc, name="all")
nbc_bulge.load_from_cat(nbc, name="all")
gc.collect()
import pdb
pdb.set_trace()
# Do any setting up that needs doing prior to applying the calibration to the catalogue
nbc_disc, nbc_bulge = setup(nbc_disc, nbc_bulge, split_type, scheme,
params["output_dir"], params["sbins"],
params["rbins"], smoothing)
# Apply the calibration from one half of the catalogue to the second part
nbc_disc.apply(split_half=2, scheme=scheme, names=["m", "a"])
nbc_bulge.apply(split_half=2, scheme=scheme, names=["m", "a"])
nbc.combine_bd(nbc_disc, nbc_bulge, split_half=2, names=["m", "c1", "c2"])
# Finally test for residual bias
zbins = [0.2, 0.43, 0.63, 0.9, 1.3]
bias0 = nbc.redshift_diagnostic(bias="m",
label="Uncalibrated",
ls="none",
nbins=4,
fmt=["o", "D"],
colour="steelblue",
weights=wt2,
bins=zbins,
tophat=False,
separate_components=False)
bias = nbc.redshift_diagnostic(bias="m",
label="Calibrated",
ls="none",
nbins=4,
fmt=["^", ">"],
apply_calibration=True,
colour="purple",
weights=wt2,
bins=zbins,
tophat=False,
separate_components=False)
# Save the residual bias numbers to a text file
np.savetxt(
"/home/samuroff/shear_pipeline/plot_dump/datavecs/m-vs-z-dvec-%s-halfcat-uncalibrated.txt"
% (split_type),
np.array(bias0).T,
header="z m1 em1 m2 em2 m em")
np.savetxt(
"/home/samuroff/shear_pipeline/plot_dump/datavecs/m-vs-z-dvec-%s-halfcat-%s-nbc-sbins%d-rbins%d.txt"
% (split_type, scheme, params["sbins"], params["rbins"]),
np.array(bias).T,
header="z m1 em1 m2 em2 m em")
print "Done"
return 0
