def calculate(y1v2, hoopoe, split_method="none", weights=None, outputs=[], method="grid", smoothing=3, config=None, names=["m", "a"], sbins=16, rbins=16, nthread=1, rank=0, task_allocation={0 : ["disc_table", "bulge_table", "disc_htable", "bulge_htable"]} ):
if not os.path.exists(config["output"]["dir"]):
print "Warning - output plots directory does not exist"
# First make some diagnostic distributions of the observable parameters relevant for the NBC
if ("histograms" in outputs):
pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output"]["dir"], weights=weights)
pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs"%config["output"]["dir"], weights=weights)
if ("alpha" in outputs):
# Global fit of alpha to compare with previous runs
plt.close()
b = bias=di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22" ], xlim=(-0.03,0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True)
plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
plt.title("Y1 v2 Sim, PSF v02")
os.system("mkdir -p %s/release/alpha"%config["output"]["dir"])
plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png"%config["output"]["dir"])
plt.close()
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
# split the data into two subsets
wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method)
nbc_disc.load_from_cat(nbc, name="all")
nbc_bulge.load_from_cat(nbc, name="all")
#### Process disc then bulge runs
# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
edges_bulge = "equal_number"
edges_disc = "equal_number"
if ("half_tables" in outputs):
if ("disc_htable" in task_allocation[rank]):
nbc_disc.compute(split_half=1,
fit="disc", weights=wt1,
use_catalogue_weights=config["selection"]["weights"],
reweight_per_bin=False, resample_per_bin=False,
refdata=y1v2, binning=edges_disc,
rbins=rbins, sbins=sbins,
rlim=(1.13,3.0), slim=(12,200),
table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method, sbins,rbins))
if ("bulge_htable" in task_allocation[rank]):
nbc_bulge.compute(split_half=1,
fit="bulge", weights=wt1,
use_catalogue_weights=config["selection"]["weights"],
reweight_per_bin=False, resample_per_bin=False,
refdata=y1v2, binning=edges_bulge,
rbins=rbins, sbins=sbins,
rlim=(1.13,3.0), slim=(12,200),
table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method, sbins,rbins))
if ("tables" in outputs):
if ("disc_table" in task_allocation[rank]):
nbc_disc.compute(split_half=0,
fit="disc", weights=weights,
use_catalogue_weights=config["selection"]["weights"],
reweight_per_bin=False, resample_per_bin=False,
refdata=y1v2, binning=edges_disc,
rbins=rbins, sbins=sbins,
rlim=(1.13,3.0), slim=(12,200),
table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], sbins, rbins))
if ("bulge_table" in task_allocation[rank]):
nbc_bulge.compute(split_half=0,
fit="bulge", weights=weights,
use_catalogue_weights=config["selection"]["weights"],
reweight_per_bin=False, resample_per_bin=False,
refdata=y1v2, binning=edges_bulge,
rbins=rbins, sbins=sbins,
rlim=(1.13,3.0), slim=(12,200),
table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], sbins,rbins))
sub_dir = method.lower()
# Now plot out the points and the resulting smoothing fit
if ("snr" in outputs):
if (method.lower()=="rbf") or (method.lower()=="grid"):
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
elif (method.lower()=="polynomial"):
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-disc-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-disc-v1-1.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-disc-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-disc-v1-2.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-bulge-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-bulge-v1-1.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-bulge-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-bulge-v1-2.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
if (split_method!="none"):
# Derive a calibration from the grid in whichever way is specified in the config file
if (method=="rbf"):
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins), smoothing=smoothing)
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins), smoothing=smoothing)
elif (method=="grid"):
nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))[1].read()
nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))[1].read()
elif (method=="polynomial"):
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))
# Apply it to the other half
nbc_disc.apply(split_half=2, scheme=method, names=names)
nbc_bulge.apply(split_half=2, scheme=method, names=names)
nbc.combine_bd(nbc_disc,nbc_bulge, split_half=2, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )
# Finally save some diagnostic plots in tomographic bins
if ("redshift" in outputs):
zbins=[ 0.2, 0.43, 0.63, 0.9, 1.3]
tophat = config["calibration"]["ztype"]=="tophat"
# Only do the redshift plots with the split catalogues if they were recalculated this time
plt.close()
if "m" in names:
bias0=nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
bias=nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^",">"], apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
plt.ylabel("Multiplicative Bias $m$")
plt.legend(loc="center right")
plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
plt.close()
if "a" in names:
nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat)
nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=4, apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat)
plt.ylabel(r"PSF Leakage $\alpha$")
plt.legend(loc="upper left")
plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
plt.close()
np.savetxt("%s/data/m-vs-z-uncalibrated-%s-halfcat-unblinded%s.txt"%(config["output"]["dir"],split_method, ""+"-tophat"*int(tophat)), np.array(bias0))
np.savetxt("%s/data/m-vs-z-%s_nbc-%s-halfcat-unblinded%s.txt"%(config["output"]["dir"],method,split_method, ""+"-tophat"*int(tophat)),np.array(bias))
# Finally redo the fits with the full catalogue
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights)
nbc_disc.load_from_cat(nbc, name="all")
nbc_bulge.load_from_cat(nbc, name="all")
if (method.lower()=="rbf"):
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
elif (method.lower()=="grid"):
nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read()
nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read()
elif (method.lower()=="polynomial"):
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
nbc_disc.apply(split_half=0, scheme=method, names=names)
nbc_bulge.apply(split_half=0, scheme=method, names=names)
nbc.combine_bd(nbc_disc,nbc_bulge, split_half=0, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )
nbc.preserve_calibration_col("%s/data/hoopoe-%s-nbc_column.fits"%(config["output"]["dir"],method))
plt.close()
if "m" in names:
bias0 = nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
bias = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^",">"], apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
plt.ylabel("Multiplicative Bias $m$")
plt.legend(loc="center right")
plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
plt.close()
np.savetxt("%s/data/m-vs-z-uncalibrated-fullcat-unblinded%s.txt"%(config["output"]["dir"], ""+"-tophat"*int(tophat)), np.array(bias0))
np.savetxt("%s/data/m-vs-z-%s_nbc-fullcat-unblinded%s.txt"%(config["output"]["dir"], method,""+"-tophat"*int(tophat)), np.array(bias))
if "a" in names:
nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat)
nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=4, apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat)
plt.ylabel(r"PSF Leakage $\alpha$")
plt.legend(loc="upper left")
plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
plt.close()
print "Done"
return 0
def compute(self, split_half=0, fit="bord", apply_calibration=False, table_name=None, ellipticity_name="e", sbins=10, rbins=5, binning="equal_number",rlim=(1,3), slim=(10,1000)):
print 'measuring bias'
if split_half>0:
exec "data = self.res%d"%split_half
print "using half %d of the catalogue (%d objects)"%(split_half,data.size)
if hasattr(self, "truth"):
exec "tr = self.truth%d"%split_half
else:
data = self.res
print "using the full catalogue (%d objects)"%(data.size)
if hasattr(self, "truth"):
tr = self.truth
if fit.lower()!="bord":
print "Using %s only galaxies"%fit
val = int(fit.lower()=="bulge")
sel_fit = data["is_bulge"]==val
else:
sel_fit = np.ones_like(data).astype(bool)
data = data[sel_fit]
tr = tr[sel_fit]
sel_lim = (data["snr"]>slim[0]) & (data["snr"]<slim[1]) & (data["mean_rgpp_rp"]>rlim[0]) & (data["mean_rgpp_rp"]<rlim[1])
data = data[sel_lim]
tr = tr[sel_lim]
if isinstance(binning,str) :
if binning.lower()=="uniform":
snr_edges = np.logspace(np.log10(slim[0]),np.log10(slim[1]),sbins+1)
rgp_edges = np.linspace(rlim[0],rlim[1],rbins+1)
elif binning.lower()=="equal_number":
snr_edges = di.find_bin_edges(np.log10(data["snr"]), sbins)
rgp_edges = di.find_bin_edges(np.log10(data["mean_rgpp_rp"]), rbins)
snr_centres = (snr_edges[1:]+snr_edges[:-1])/2.0
rgp_centres = (rgp_edges[1:]+rgp_edges[:-1])/2.0
list_bias = []
bias_grid=[]
b = di.get_bias(tr, data, nbins=5, apply_calibration=apply_calibration, ellipticity_name=ellipticity_name, binning="equal_number", names=["m","c","m11","m22","c11","c22"], silent=True)
print "Global biases:"
print "m11 : ", b["m11"]
print "m22 : ", b["m22"]
print "m : ", b["m"]
print "c11 : ", b["c11"]
print "c22 : ", b["c22"]
print "c : ", b["c"]
print "Will do dynamic binning in SNR"
for i in xrange(len(rgp_edges)-1):
snr_samp = data["snr"][(np.log10(data['mean_rgpp_rp']) > rgp_edges[i]) & (np.log10(data['mean_rgpp_rp']) < rgp_edges[i+1])]
snr_edges=di.find_bin_edges(np.log10(snr_samp), sbins)
for j in xrange(len(snr_edges)-1):
empty=False
print "bin %d %d snr = [%2.3f-%2.3f] rgpp/rp = [%2.3f-%2.3f]"%(j, i, 10**snr_edges[j], 10**snr_edges[j+1], 10**rgp_edges[i], 10**rgp_edges[i+1] )
# Select in bins of snr and size
select = (np.log10(data['snr']) > snr_edges[j]) & (np.log10(data['snr']) < snr_edges[j+1]) & (np.log10(data['mean_rgpp_rp']) > rgp_edges[i]) & (np.log10(data['mean_rgpp_rp']) < rgp_edges[i+1])
ngal = np.nonzero(select.astype(int))[0].size
# Raise an error if there are too few simulated galaxies in a given bin
if ngal < 60:
print "Warning: <100 galaxies in bin %d, %d (ngal=%d)"%(i,j, ngal)
empty=False
if ngal==0:
print "Warning: no galaxies in bin %d, %d "%(i,j)
empty=True
vrgp_mid = rgp_centres[i]
vsnr_mid = snr_centres[j]
vrgp_min = rgp_edges[i]
vsnr_min = snr_edges[j]
vrgp_max = rgp_edges[i+1]
vsnr_max = snr_edges[j+1]
if ngal==0:
print "Warning: no galaxies in bin %d, %d"%(i,j)
list_bias.append([j, i, ngal, 0, vrgp_min, vrgp_max, vsnr_min, vsnr_max, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. , 0., 0.])
continue
filename_str = 'snr%2.2f.rgpp%2.2f' % (10**vsnr_mid,10**vrgp_mid)
b = di.get_bias(tr[select], data[select], apply_calibration=apply_calibration, nbins=5, ellipticity_name=ellipticity_name, binning="equal_number", names=["m","c","m11","m22","c11","c22"], silent=True)
a = di.get_alpha(data[select], data[select], nbins=5, xlim=(-0.015, 0.02), binning="equal_number", names=["alpha", "alpha11", "alpha22"], silent=True, use_weights=False)
list_bias.append([j, i, ngal, 10**vrgp_min, 10**vrgp_max, 10**vsnr_min, 10**vsnr_max, b["m"][0], b["m"][1], b["c"][0], b["c"][1], b["m11"][0], b["m11"][1], b["m22"][0], b["m22"][1], b["c11"][0], b["c11"][1], b["c22"][0], b["c22"][1], a["alpha"][0], a["alpha"][1], a["alpha11"][0], a["alpha11"][1], a["alpha22"][0], a["alpha22"][1] ])
lab=["j","i","ngal","rgp_lower","rgp_upper","snr_lower","snr_upper","m","err_m","c","err_c","m1","err_m1","m2","err_m2","c1","err_c1","c2", "err_c2","alpha","err_alpha","alpha11","err_alpha11","alpha22","err_alpha22"]
dt = {'names': lab, 'formats': ['i4']*3 + ['f8']*22 }
arr_bias = np.core.records.fromarrays(np.array(list_bias).transpose(), dtype=dt)
if table_name is None:
filename_table_bias = 'bias_table-%s-selection_section%d%s.fits'%(fit, split_half,"_calibrated"*apply_calibration)
else:
filename_table_bias = table_name
import pyfits
pyfits.writeto(filename_table_bias,arr_bias,clobber=True)
print 'saved %s'%filename_table_bias
def diagnostics(y1v2,
hoopoe,
tomographic_calibration=False,
histograms=True,
split_method="random",
weights=None,
alpha=True,
table=True,
half_tables=True,
vssnr=True,
vsredshift=True,
rbf=True,
simple_grid=False,
smoothing=3,
config=None,
names=["m", "a"],
sbins=16,
rbins=11):
if rbf:
sub_dir = "rbf"
elif simple_grid:
sub_dir = "grid"
else:
sub_dir = "polynomial"
if not os.path.exists(config["output_dir"]):
print "Warning - output plots directory does not exist"
# First make some diagnostic distributions of the observable parameters relevant for the NBC
if histograms:
pl.histograms_vs_input(["e", "size", "flux"],
hoopoe.truth,
data2=y1v2.res,
outdir="%s/release/inputs" %
config["output_dir"],
weights=weights)
pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"],
hoopoe.res,
kl=True,
data2=y1v2.res,
outdir="%s/release/outputs" % config["output_dir"],
weights=weights)
if alpha:
# Global fit of alpha to compare with previous runs
plt.close()
b = bias = di.get_alpha(hoopoe.res,
hoopoe.res,
nbins=15,
names=["alpha11", "alpha22"],
xlim=(-0.03, 0.02),
xdata_name="mean_hsm_psf_e%d_sky",
weights=weights,
visual=True)
plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
plt.title("Y1 v2 Sim, PSF v02")
os.system("mkdir -p %s/release/alpha" % config["output_dir"])
plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png" %
config["output_dir"])
plt.close()
#Exit here if no calibration diagnostics are needed
if (not table) and (not vssnr) and (not vsredshift):
return 0
# Now compute the calibration using a random half of the simulation
if vssnr or half_tables:
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
wt1, wt2 = nbc.get_split_data(hoopoe,
weights=weights,
method=split_method)
nbc_disc.load_from_cat(nbc, name="all")
nbc_bulge.load_from_cat(nbc, name="all")
#### Process disc then bulge runs
# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
import pdb
pdb.set_trace()
if table:
if ("match_gridpoints" in config.keys()):
redges_bulge, sedges_bulge = bin_edges_from_table(
config["match_gridpoints"], type="bulge")
redges_disc, sedges_disc = bin_edges_from_table(
config["match_gridpoints"], type="disc")
edges_disc = (redges_disc, sedges_disc)
edges_bulge = (redges_bulge, sedges_bulge)
else:
edges_bulge = "equal_number"
edges_disc = "equal_number"
if tomographic_calibration:
zbin_numbers = np.unique(nbc.res["des_bin"])
if half_tables:
print "Calculating half tables in %d redshift bins" % zbin_numbers.size
for k in zbin_numbers:
print "bin %d" % k
nbc_disc.compute(
split_half=1,
fit="disc",
weights=wt1,
reweight_per_bin=config["reweight_perbin"],
resample_per_bin=config["resample_perbin"],
refdata=y1v2,
binning=edges_disc,
redshift_bin=k,
rbins=rbins,
sbins=sbins,
rlim=(1.13, 3.0),
slim=(12, 200),
table_name=
"%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], split_method, sbins, rbins, k))
nbc_bulge.compute(
split_half=1,
fit="bulge",
weights=wt1,
reweight_per_bin=config["reweight_perbin"],
resample_per_bin=config["resample_perbin"],
refdata=y1v2,
binning=edges_bulge,
redshift_bin=k,
rbins=rbins,
sbins=sbins,
rlim=(1.13, 3.0),
slim=(12, 200),
table_name=
"%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], split_method, sbins, rbins, k))
if table:
print "Calculating full tables in %d redshift bins" % zbin_numbers.size
for k in zbin_numbers:
print "bin %d" % k
nbc_disc.compute(
split_half=0,
fit="disc",
weights=weights,
reweight_per_bin=config["reweight_perbin"],
resample_per_bin=config["resample_perbin"],
refdata=y1v2,
binning=edges_disc,
redshift_bin=k,
rbins=rbins,
sbins=sbins,
rlim=(1.13, 3.0),
slim=(12, 200),
table_name=
"%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], sbins, rbins, k))
nbc_bulge.compute(
split_half=0,
fit="bulge",
weights=weights,
reweight_per_bin=config["reweight_perbin"],
resample_per_bin=config["resample_perbin"],
refdata=y1v2,
binning=edges_bulge,
redshift_bin=k,
rbins=rbins,
sbins=sbins,
rlim=(1.13, 3.0),
slim=(12, 200),
table_name=
"%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins-zbin%d.fits"
% (config["output_dir"], sbins, rbins, k))
if rbf:
scheme = "rbf"
elif simple_grid:
scheme = "grid"
else:
scheme = "polynomial"
if half_tables:
nbc = calibrate_in_bins(nbc,
halfcats=True,
split_method=split_method,
scheme=scheme,
config=config,
sbins=sbins,
rbins=rbins,
smoothing=smoothing,
names=names)
# Finally save some diagnostic plots in tomographic bins
if vsredshift:
zbins = [0.2, 0.43, 0.63, 0.9, 1.3]
tophat = config["tophat_binning"]
if half_tables:
plt.close()
if "m" in names:
nbc.redshift_diagnostic(bias="m",
label="Uncalibrated",
ls="none",
nbins=3,
fmt=["o", "D"],
colour="steelblue",
weights=wt2,
bins=zbins,
tophat=tophat,
separate_components=False)
nbc.redshift_diagnostic(bias="m",
label="Calibrated",
ls="none",
nbins=3,
fmt=["^", ">"],
apply_calibration=True,
colour="purple",
weights=wt2,
bins=zbins,
tophat=tophat,
separate_components=False)
plt.ylabel("Multiplicative Bias $m$")
plt.legend(loc="center right")
plt.savefig(
"%s/release/%s/m-bias-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
% (config["output_dir"], sub_dir, split_method, smoothing,
sbins, rbins, int(tophat)))
plt.close()
if "a" in names:
nbc.redshift_diagnostic(bias="alpha",
label="Uncalibrated",
ls="none",
nbins=3,
fmt=["o", "D"],
colour="steelblue",
weights=wt2,
bins=zbins,
tophat=tophat)
nbc.redshift_diagnostic(bias="alpha",
label="Calibrated",
ls="none",
fmt=["^", ">"],
nbins=3,
apply_calibration=True,
colour="purple",
weights=wt2,
bins=zbins,
tophat=tophat)
plt.ylabel(r"PSF Leakage $\alpha$")
plt.legend(loc="upper left")
plt.savefig(
"%s/release/%s/alpha-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
% (config["output_dir"], sub_dir, split_method, smoothing,
sbins, rbins, int(tophat)))
plt.close()
# Finally redo the fits with the full catalogue
gc.collect()
nbc = cal.nbc()
wt1, wt2 = nbc.get_split_data(hoopoe,
weights=weights,
method=split_method)
nbc = calibrate_in_bins(nbc,
halfcats=False,
split_method=None,
scheme=scheme,
config=config,
sbins=sbins,
rbins=rbins,
smoothing=smoothing,
names=names)
import pdb
pdb.set_trace()
plt.close()
if "m" in names:
nbc.redshift_diagnostic(bias="m",
label="Uncalibrated",
ls="none",
nbins=3,
fmt=["o", "D"],
colour="steelblue",
weights=weights,
split_half=0,
bins=zbins,
tophat=tophat,
separate_components=False)
bias = nbc.redshift_diagnostic(bias="m",
label="Calibrated",
ls="none",
nbins=3,
fmt=["^", ">"],
apply_calibration=True,
colour="purple",
weights=weights,
split_half=0,
bins=zbins,
tophat=tophat,
separate_components=False)
plt.ylabel("Multiplicative Bias $m$")
plt.legend(loc="center right")
plt.savefig(
"%s/release/%s/m-bias-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
% (config["output_dir"], sub_dir, smoothing, sbins, rbins,
int(tophat)))
plt.close()
if "a" in names:
nbc.redshift_diagnostic(bias="alpha",
label="Uncalibrated",
ls="none",
nbins=3,
fmt=["o", "D"],
colour="steelblue",
weights=weights,
split_half=0,
bins=zbins,
tophat=tophat)
nbc.redshift_diagnostic(bias="alpha",
label="Calibrated",
ls="none",
fmt=["^", ">"],
nbins=3,
apply_calibration=True,
colour="purple",
weights=weights,
split_half=0,
bins=zbins,
tophat=tophat)
plt.ylabel(r"PSF Leakage $\alpha$")
plt.legend(loc="upper left")
plt.savefig(
"%s/release/%s/alpha-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
% (config["output_dir"], sub_dir, smoothing, sbins, rbins,
int(tophat)))
plt.close()
def calculate(y1v2, hoopoe, split_method="none", weights=None, outputs=[] method="grid", smoothing=3, config=None, names=["m", "a"], sbins=16, rbins=16):
if not os.path.exists(config["output_dir"]):
print "Warning - output plots directory does not exist"
# First make some diagnostic distributions of the observable parameters relevant for the NBC
if ("histograms" in outputs):
pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output"]["dir"], weights=weights)
pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs"%config["output"]["dir"], weights=weights)
if ("alpha" in outputs):
# Global fit of alpha to compare with previous runs
plt.close()
b = bias=di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22" ], xlim=(-0.03,0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True)
plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
plt.title("Y1 v2 Sim, PSF v02")
os.system("mkdir -p %s/release/alpha"%config["output_dir"])
plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png"%config["output_dir"])
plt.close()
# split the data into two subsets
if ("snr" in outputs) or ("half_tables" in outputs):
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method)
nbc_disc.load_from_cat(nbc, name="all")
def diagnostics(y1v2, hoopoe, histograms=True, alpha=True, table=True, vssnr=True, vsredshift=True, rbf=True):
if not os.path.exists(config["output_dir"]):
print "Warning - output plots directory does not exist"
# First make some diagnostic distributions of the observable parameters relevant for the NBC
if histograms:
pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output_dir"])
pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, data2=y1v2.res, outdir="%s/release/outputs"%config["output_dir"])
if alpha:
# Global fit of alpha to compare with previous runs
plt.close()
b = di.get_alpha(hoopoe.res, hoopoe.res, nbins=20, xlim=(-0.015,0.025), binning="equal_number", use_weights=False, visual=True)
plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
plt.title("Y1 v2 Sim, PSF v02")
os.system("mkdir -p %s/release/alpha"%config["output_dir"])
plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit.png"%config["output_dir"])
plt.close()
#Exit here if no calibration diagnostics are needed
if (not table) and (not vssnr) and (not vsredshift):
return 0
# Now compute the calibration using a random half of the simulation
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
nbc.get_split_data(hoopoe)
nbc_disc.load_from_cat(nbc, name="all")
nbc_bulge.load_from_cat(nbc, name="all")
#### Process disc then bulge runs
# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
if table:
nbc_disc.compute(split_half=1, fit="disc", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
nbc_bulge.compute(split_half=1, fit="bulge", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])
nbc_disc.compute(split_half=0, fit="disc", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
nbc_bulge.compute(split_half=0, fit="bulge", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
# Now plot out the points and the resulting smooth fit
if vssnr:
if rbf:
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
else:
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="m", do_half=1, output="%s/m-vs-snr-disc-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="a", do_half=1, output="%s/alpha-vs-snr-disc-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
plt.close()
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="m", do_half=2, output="%s/m-vs-snr-disc-v1-2.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="a", do_half=2, output="%s/alpha-vs-snr-disc-v1-2.png"%(config["output_dir"]+"release"+"/rbf"*rbf), use_rbf=rbf)
plt.close()
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="m", do_half=1, output="%s/m-vs-snr-bulge-v1-2.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="a", do_half=1, output="%s/alpha-vs-snr-bulge-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
plt.close()
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="m", do_half=2, output="%s/m-vs-snr-bulge-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="a", do_half=2, output="%s/alpha-vs-snr-bulge-v1-2.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
plt.close()
if rbf:
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])
else:
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])
# Apply to the other half
nbc_disc.apply(split_half=2, use_rbf=rbf)
nbc_bulge.apply(split_half=2, use_rbf=rbf)
nbc.get_combined_calibration(nbc_disc,nbc_bulge, split_half=2)
# Finally save some diagnostic plots in tomographic bins
if vsredshift:
nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o","D"], colour="steelblue")
nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple")
plt.ylabel("Multiplicative Bias $m$")
plt.legend(loc="lower left")
plt.savefig("%s/release/%s/m-vs-redshift-diagnostic-v1.png"%(config["output_dir"],rbf*"/rbf"))
plt.close()
nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=3, fmt=["o","D"], colour="steelblue", error_type="std")
nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=3, apply_calibration=True, colour="purple", error_type="std")
plt.ylabel(r"PSF Leakage $\alpha$")
plt.legend(loc="upper left")
plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1.png"%(config["output_dir"],rbf*"/rbf"))
plt.close()
def diagnostics(y1v2, hoopoe, histograms=True, split_method="random", weights=None, alpha=True, table=True, half_tables=True, vssnr=True, vsredshift=True, rbf=True, simple_grid=False, smoothing=3, config=None, names=["m", "a"], sbins=16, rbins=11):
if rbf:
sub_dir="rbf"
elif simple_grid:
sub_dir="grid"
else:
sub_dir="polynomial"
if not os.path.exists(config["output_dir"]):
print "Warning - output plots directory does not exist"
# First make some diagnostic distributions of the observable parameters relevant for the NBC
if histograms:
pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output_dir"], weights=weights)
pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs"%config["output_dir"], weights=weights)
if alpha:
# Global fit of alpha to compare with previous runs
plt.close()
b = bias=di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22" ], xlim=(-0.03,0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True)
plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
plt.title("Y1 v2 Sim, PSF v02")
os.system("mkdir -p %s/release/alpha"%config["output_dir"])
plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png"%config["output_dir"])
plt.close()
#Exit here if no calibration diagnostics are needed
if (not table) and (not vssnr) and (not vsredshift):
return 0
# Now compute the calibration using a random half of the simulation
if vssnr or half_tables:
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method)
nbc_disc.load_from_cat(nbc, name="all")
nbc_bulge.load_from_cat(nbc, name="all")
#### Process disc then bulge runs
# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
if table:
if ("match_gridpoints" in config.keys()):
redges_bulge, sedges_bulge = bin_edges_from_table(config["match_gridpoints"], type="bulge")
redges_disc, sedges_disc = bin_edges_from_table(config["match_gridpoints"], type="disc")
edges_disc = (redges_disc, sedges_disc)
edges_bulge = (redges_bulge, sedges_bulge)
else:
edges_bulge = "equal_number"
edges_disc = "equal_number"
if half_tables:
nbc_disc.compute(split_half=1, fit="disc", weights=wt1, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_disc, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method, sbins,rbins))
nbc_bulge.compute(split_half=1, fit="bulge", weights=wt1, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_bulge, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method, sbins,rbins))
if config["full_catalogue"]:
nbc_disc.compute(split_half=0, fit="disc", weights=weights, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_disc, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], sbins, rbins))
nbc_bulge.compute(split_half=0, fit="bulge", weights=weights, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_bulge, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], sbins,rbins))
# Now plot out the points and the resulting smoothing fit
if vssnr:
if rbf:
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
else:
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-disc-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-disc-v1-1.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-disc-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-disc-v1-2.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-bulge-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-bulge-v1-1.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-bulge-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-bulge-v1-2.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
plt.close()
if half_tables and rbf:
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins), smoothing=smoothing)
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins), smoothing=smoothing)
elif half_tables and simple_grid:
nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))[1].read()
nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))[1].read()
elif half_tables and (not rbf) and (not simple_grid):
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))
# Apply to the other half
if rbf:
scheme="rbf"
elif simple_grid:
scheme="grid"
else:
scheme="polynomial"
if half_tables:
nbc_disc.apply(split_half=2, scheme=scheme, names=names)
nbc_bulge.apply(split_half=2, scheme=scheme, names=names)
nbc.combine_bd(nbc_disc,nbc_bulge, split_half=2, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )
# Finally save some diagnostic plots in tomographic bins
if vsredshift:
zbins=[ 0.2, 0.43, 0.63, 0.9, 1.3]
tophat = config["tophat_binning"]
import pdb ; pdb.set_trace()
if half_tables:
plt.close()
if "m" in names:
bias0=nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
bias=nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^",">"], apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
plt.ylabel("Multiplicative Bias $m$")
plt.legend(loc="center right")
plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
plt.close()
if "a" in names:
nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat)
nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=3, apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat)
plt.ylabel(r"PSF Leakage $\alpha$")
plt.legend(loc="upper left")
plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
plt.close()
# Finally redo the fits with the full catalogue
nbc = cal.nbc()
nbc_disc = cal.nbc()
nbc_bulge = cal.nbc()
wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights)
nbc_disc.load_from_cat(nbc, name="all")
nbc_bulge.load_from_cat(nbc, name="all")
if rbf:
nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
elif simple_grid:
nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))[1].read()
nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))[1].read()
else:
nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
nbc_disc.apply(split_half=0, scheme=scheme, names=names)
nbc_bulge.apply(split_half=0, scheme=scheme, names=names)
nbc.combine_bd(nbc_disc,nbc_bulge, split_half=0, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )
plt.close()
if "m" in names:
bias0 = nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
bias = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
plt.ylabel("Multiplicative Bias $m$")
plt.legend(loc="center right")
plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
plt.close()
#Save the calibration data so we can reporoduce the paper plots quickly without rerunning all of the above
out = fi.FITS("hoopoe-v2-nbc-%s.fits"%scheme, "rw")
dat = np.empty(nbc.res.size, dtype=[("coadd_objects_id", int), ("m", float), ("c1", float), ("c2", float) ])
for name in ["coadd_objects_id", "m", "c1", "c2"]: dat[name]=nbc.res[name]
out.write(dat)
out[-1].write_key("EXTNAME", "nbc_col")
out[-1].write_key("METHOD", scheme)
out.close()
np.savetxt("m-vs-z-dvec-fullcat-uncalibrated-tophatbins.txt", np.array(bias0).T, header="z m1 em1 m2 em2 m em")
np.savetxt("shear_pipeline/plot_dump/datavecs/m-vs-z-dvec-fullcat-%s-nbc-tophatbins.txt"%scheme, np.array(bias).T, header="z m1 em1 m2 em2 m em")
rewt=di.get_weights_to_match(y1v2.res["mean_rgpp_rp"], hoopoe.res["mean_rgpp_rp"],nbins=25)
bmask = (hoopoe.res["is_bulge"]==1)
mask = (hoopoe.res["bulge_flux"]<10) & (hoopoe.res["disc_flux"]<10)
rewtb=di.get_weights_to_match(y1v2.res["bulge_flux"][y1v2.res["bulge_flux"]!=0], hoopoe.res["bulge_flux"][bmask & mask],nbins=25, xlim=(0.0,10.))
rewtd=di.get_weights_to_match(y1v2.res["disc_flux"][y1v2.res["bulge_flux"]==0], hoopoe.res["disc_flux"][np.invert(bmask) & mask],nbins=25, xlim=(0.0,10.))
wts_flux = np.zeros(hoopoe.res.size)
wts_flux[bmask & mask] = rewtb
wts_flux[np.invert(bmask) & mask] = rewtd
wts_flux = wts_flux[mask]
bias_size_wtd = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple", weights=rewt, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
bias_flux_wtd = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple", weights=wts_flux, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
out=np.vstack((bias_size_wtd[0], bias_size_wtd[-2], bias_size_wtd[-1], bias[-2], bias[-1]))
np.savetxt("m-vs-z_sizewtd-%s.txt"%scheme, out.T)
out=np.vstack((bias_flux_wtd[0], bias_flux_wtd[-2], bias_flux_wtd[-1], bias[-2], bias[-1]))
np.savetxt("m-vs-z_fluxwtd-%s.txt"%scheme, out.T)
if "a" in names:
nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat)
nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=3, apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat)
plt.ylabel(r"PSF Leakage $\alpha$")
plt.legend(loc="upper left")
plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
plt.close()