예제 #1
0
def generate_difference_table(config1, config2, sbins=16, rbins=16, y1v2=None):
    hoopoe1, weights1, y1v2 = ct.setup(True, True, config1, y1v2=y1v2)
    hoopoe2, weights2, y1v2 = ct.setup(True, False, config2, y1v2=y1v2)

    subset = np.in1d(hoopoe1.res["coadd_objects_id"],
                     hoopoe2.res["coadd_objects_id"])

    nbc = cal.nbc()
    nbc_disc = cal.nbc()
    nbc_bulge = cal.nbc()
    wt1, wt2 = nbc.get_split_data(hoopoe1, weights=weights1, method="random")
    nbc_disc.load_from_cat(nbc, name="all")
    nbc_bulge.load_from_cat(nbc, name="all")

    import pdb
    pdb.set_trace()

    nbc_disc.diff_table(weights=weights1,
                        compare_with_subset=subset,
                        fit="disc",
                        table_name="diff_table_disc.fits",
                        sbins=sbins,
                        rbins=rbins,
                        binning="equal_number",
                        rlim=(1.13, 3),
                        slim=(12, 200))
    nbc_bulge.diff_table(weights=weights2,
                         compare_with_subset=subset,
                         fit="bulge",
                         table_name="diff_table_bulge.fits",
                         sbins=sbins,
                         rbins=rbins,
                         binning="equal_number",
                         rlim=(1.13, 3),
                         slim=(12, 200))
예제 #2
0
def calibrate(data, method="grid", config=None, smoothing=3, sbins=16, rbins=16, nthread=1, rank=0):

	rbf = (method.lower()=="rbf")

	# Set up the wrappers
	nbc = cal.nbc()
	nbc_disc = cal.nbc()
	nbc.res = data.res
	del(data.res)
	gc.collect()
	nbc_bulge = cal.nbc()

	nbc_bulge.res=nbc.res[nbc.res["is_bulge"].astype(bool)]
	nbc_disc.res=nbc.res[np.invert(nbc.res["is_bulge"].astype(bool))]
	
	# Fit or interpolate
	if method is "polynomial":
		nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
		nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
		nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
		nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
	elif method is "rbf":
		nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
		nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
	elif method is "grid":
		nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read()
		nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read()

	# Apply to get a bias correction for each galaxy in the data
	nbc_disc.apply(split_half=0, scheme=method)
	print "Done disc calibration"

	nbc_bulge.apply(split_half=0, scheme=method)
	print "Done bulge calibration"

	print "Merging bulge/disc results...",
	nbc.res = arr.add_col(nbc.res, "m")
	nbc.res = arr.add_col(nbc.res, "c1")
	nbc.res = arr.add_col(nbc.res, "c2")
	nbc.res = arr.add_col(nbc.res, "a")

	nbc.res["m"][np.invert(nbc.res["is_bulge"].astype(bool))] = nbc_disc.res["m"]
	nbc.res["c1"][np.invert(nbc.res["is_bulge"].astype(bool))] = nbc_disc.res["c1"]
	nbc.res["c2"][np.invert(nbc.res["is_bulge"].astype(bool))] = nbc_disc.res["c2"]
	nbc.res["a"][np.invert(nbc.res["is_bulge"].astype(bool))] = nbc_disc.res["a"]
	nbc.res["m"][nbc.res["is_bulge"].astype(bool)] = nbc_bulge.res["m"]
	nbc.res["c1"][nbc.res["is_bulge"].astype(bool)] = nbc_bulge.res["c1"]
	nbc.res["c2"][nbc.res["is_bulge"].astype(bool)] = nbc_bulge.res["c2"]
	nbc.res["a"][nbc.res["is_bulge"].astype(bool)] = nbc_bulge.res["a"]
	print "done"


	if not os.path.exists(os.path.dirname(config["output"]["dir"])):
		os.system("mkdir -p %s"%os.path.dirname(config["output"]["dir"]))

	print "Saving calibrated catalogue to %s"%config["output"]["filename"]
	nbc.export(filename="%s/%s"%(config["output"]["dir"],config["output"]["filename"]))
예제 #3
0
def calibrate(data, method="rbf"):

	rbf = (method.lower()=="rbf")

	# Set up the wrappers
	nbc = cal.nbc()
	nbc_disc = cal.nbc()
	nbc_disc.res = data.res
	nbc_bulge = cal.nbc()
	
	# Fit or interpolate
	if method is "polynomial":
		nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
		nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
		nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
		nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
	elif method is "rbf":
		nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
		nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])

	# Apply to get a bias correction for each galaxy in the data
	nbc_disc.apply(split_half=0, use_rbf=rbf)
	nbc.res = nbc_disc.res
	del(nbc_disc.res)
	print "Done bulge calibration"

	nbc_bulge.res = data.res
	nbc_bulge.apply(split_half=0, use_rbf=rbf)
	nbc.res[nbc.res["is_bulge"].astype(bool)] = nbc_bulge.res[nbc.res["is_bulge"].astype(bool)]
	del(nbc_bulge.res)
	print "Done disc calibration"


	if not os.path.exists(os.path.dirname(config["output_dir"])):
		os.system("mkdir -p %s"%os.path.dirname(config["output_dir"]))

	print "Saving calibrated catalogue to %s"%config["output_catalogue"]
	nbc.export(filename="%s/%s"%(config["output_dir"],config["output_catalogue"]))
예제 #4
0
def calculate(y1v2, hoopoe, split_method="none", weights=None, outputs=[], method="grid", smoothing=3, config=None, names=["m", "a"], sbins=16, rbins=16, nthread=1, rank=0, task_allocation={0 : ["disc_table", "bulge_table", "disc_htable", "bulge_htable"]} ):

	if not os.path.exists(config["output"]["dir"]):
		print "Warning - output plots directory does not exist"

	# First make some diagnostic distributions of the observable parameters relevant for the NBC
	if ("histograms" in outputs):
		pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output"]["dir"], weights=weights)
		pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs"%config["output"]["dir"], weights=weights)

	if ("alpha" in outputs):
		# Global fit of alpha to compare with previous runs
		plt.close()
		b = bias=di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22" ], xlim=(-0.03,0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True)
		plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
		plt.title("Y1 v2 Sim, PSF v02")

		os.system("mkdir -p %s/release/alpha"%config["output"]["dir"])
		plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png"%config["output"]["dir"])
		plt.close()

	nbc = cal.nbc()
	nbc_disc = cal.nbc()
	nbc_bulge = cal.nbc()

	# split the data into two subsets
	wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method)

	nbc_disc.load_from_cat(nbc, name="all")
	nbc_bulge.load_from_cat(nbc, name="all")

	#### Process disc then bulge runs

	# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
	edges_bulge = "equal_number"
	edges_disc = "equal_number"

	if ("half_tables" in outputs):
		if ("disc_htable" in task_allocation[rank]):
			nbc_disc.compute(split_half=1, 
			    fit="disc", weights=wt1,
			    use_catalogue_weights=config["selection"]["weights"], 
				reweight_per_bin=False, resample_per_bin=False,
				refdata=y1v2, binning=edges_disc, 
				rbins=rbins, sbins=sbins, 
				rlim=(1.13,3.0), slim=(12,200),
				table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method, sbins,rbins))
		if ("bulge_htable" in task_allocation[rank]):
			nbc_bulge.compute(split_half=1, 
			    fit="bulge", weights=wt1,
			    use_catalogue_weights=config["selection"]["weights"],
				reweight_per_bin=False, resample_per_bin=False,
				refdata=y1v2, binning=edges_bulge,
				rbins=rbins, sbins=sbins,
				rlim=(1.13,3.0), slim=(12,200),
				table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method, sbins,rbins))

	if ("tables" in outputs):
		if ("disc_table" in task_allocation[rank]):
			nbc_disc.compute(split_half=0,
			    fit="disc", weights=weights,
			    use_catalogue_weights=config["selection"]["weights"],
				reweight_per_bin=False, resample_per_bin=False,
				refdata=y1v2, binning=edges_disc,
				rbins=rbins, sbins=sbins,
				rlim=(1.13,3.0), slim=(12,200),
				table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], sbins, rbins))
		if ("bulge_table" in task_allocation[rank]):
			nbc_bulge.compute(split_half=0,
			    fit="bulge", weights=weights,
			    use_catalogue_weights=config["selection"]["weights"],
				reweight_per_bin=False, resample_per_bin=False,
				refdata=y1v2, binning=edges_bulge,
				rbins=rbins, sbins=sbins,
				rlim=(1.13,3.0), slim=(12,200),
				table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], sbins,rbins))


	sub_dir = method.lower()
	# Now plot out the points and the resulting smoothing fit
	if ("snr" in outputs):
		if (method.lower()=="rbf") or (method.lower()=="grid"):
			nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
			nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
		elif (method.lower()=="polynomial"):
			nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
			nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
			nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
			nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))

		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-disc-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-disc-v1-1.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-disc-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-disc-v1-2.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-bulge-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-bulge-v1-1.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-bulge-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-bulge-v1-2.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()

	if (split_method!="none"):
		# Derive a calibration from the grid in whichever way is specified in the config file
		if (method=="rbf"):
			nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins), smoothing=smoothing)
			nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins), smoothing=smoothing)
		elif (method=="grid"):
			nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))[1].read()
			nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))[1].read()
		elif (method=="polynomial"):
			nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))
			nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))
			nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))
			nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))

		# Apply it to the other half
		nbc_disc.apply(split_half=2, scheme=method, names=names)
		nbc_bulge.apply(split_half=2, scheme=method, names=names)
		nbc.combine_bd(nbc_disc,nbc_bulge, split_half=2, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )

	# Finally save some diagnostic plots in tomographic bins
	if ("redshift" in outputs): 
		zbins=[ 0.2, 0.43, 0.63, 0.9, 1.3]
		tophat = config["calibration"]["ztype"]=="tophat"

		# Only do the redshift plots with the split catalogues if they were recalculated this time
		plt.close()
		if "m" in names:
			bias0=nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
			bias=nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^",">"], apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
			plt.ylabel("Multiplicative Bias $m$")
			plt.legend(loc="center right")
			plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
			plt.close()
		if "a" in names:
			nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat)
			nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=4, apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat)
			plt.ylabel(r"PSF Leakage $\alpha$")
			plt.legend(loc="upper left")
			plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
			plt.close()

		np.savetxt("%s/data/m-vs-z-uncalibrated-%s-halfcat-unblinded%s.txt"%(config["output"]["dir"],split_method, ""+"-tophat"*int(tophat)), np.array(bias0))
		np.savetxt("%s/data/m-vs-z-%s_nbc-%s-halfcat-unblinded%s.txt"%(config["output"]["dir"],method,split_method, ""+"-tophat"*int(tophat)),np.array(bias))

		# Finally redo the fits with the full catalogue
		nbc = cal.nbc()
		nbc_disc = cal.nbc()
		nbc_bulge = cal.nbc()
		wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights)

		nbc_disc.load_from_cat(nbc, name="all")
		nbc_bulge.load_from_cat(nbc, name="all")

		if (method.lower()=="rbf"):
			nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
			nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
		elif (method.lower()=="grid"):
			nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read()
			nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read()
		elif (method.lower()=="polynomial"):
			nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
			nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
			nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
			nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))


		nbc_disc.apply(split_half=0, scheme=method, names=names)
		nbc_bulge.apply(split_half=0, scheme=method, names=names)
		nbc.combine_bd(nbc_disc,nbc_bulge, split_half=0, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )

		nbc.preserve_calibration_col("%s/data/hoopoe-%s-nbc_column.fits"%(config["output"]["dir"],method))

		plt.close()
		if "m" in names:
			bias0 = nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
			bias = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^",">"], apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
			plt.ylabel("Multiplicative Bias $m$")
			plt.legend(loc="center right")
			plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
			plt.close()
			np.savetxt("%s/data/m-vs-z-uncalibrated-fullcat-unblinded%s.txt"%(config["output"]["dir"], ""+"-tophat"*int(tophat)), np.array(bias0))
			np.savetxt("%s/data/m-vs-z-%s_nbc-fullcat-unblinded%s.txt"%(config["output"]["dir"], method,""+"-tophat"*int(tophat)), np.array(bias))

		if "a" in names:
			nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat)
			nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=4, apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat)
			plt.ylabel(r"PSF Leakage $\alpha$")
			plt.legend(loc="upper left")
			plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
			plt.close()

		print "Done"
		return 0
예제 #5
0
def calibrate_in_bins(nbc,
                      halfcats=False,
                      split_method="",
                      scheme="rbf",
                      config=None,
                      sbins=12,
                      rbins=9,
                      smoothing=3,
                      names=["m", "a"]):

    discs = cal.nbc()
    bulges = cal.nbc()

    if halfcats:
        split_half = 2
        prefix = "%s-halfcat" % split_method
    else:
        split_half = 0
        prefix = "fullcat"

    zbin_numbers = np.unique(nbc.res["des_bin"])
    print "Calibrating in %d redshift bins" % zbin_numbers.size
    for k in zbin_numbers:
        print "-- calibrating galaxies in bin %d" % k

        if halfcats:
            select_bin1 = nbc.res1["des_bin"] == k
            select_bin = nbc.res2["des_bin"] == k
            discs.res1 = copy.deepcopy(nbc.res1[select_bin1])
            discs.truth1 = copy.deepcopy(nbc.truth1[select_bin1])
            bulges.res1 = copy.deepcopy(nbc.res1[select_bin1])
            bulges.truth1 = copy.deepcopy(nbc.truth1[select_bin1])
            discs.res2 = copy.deepcopy(nbc.res2[select_bin])
            discs.truth2 = copy.deepcopy(nbc.truth2[select_bin])
            bulges.res2 = copy.deepcopy(nbc.res2[select_bin])
            bulges.truth2 = copy.deepcopy(nbc.truth2[select_bin])

        else:
            select_bin = nbc.res["des_bin"] == k
            discs.res = copy.deepcopy(nbc.res[select_bin])
            discs.truth = copy.deepcopy(nbc.truth[select_bin])
            bulges.res = copy.deepcopy(nbc.res[select_bin])
            bulges.truth = copy.deepcopy(nbc.truth[select_bin])

        if (scheme == "rbf"):
            discs.fit_rbf(
                table=
                "%s/nbc_data/bias_table_hoopoe-v1-%s-disc-%dsbins-%drbins-zbin%d.fits"
                % (config["output_dir"], prefix, sbins, rbins, k),
                smoothing=smoothing)
            bulges.fit_rbf(
                table=
                "%s/nbc_data/bias_table_hoopoe-v1-%s-bulge-%dsbins-%drbins-zbin%d.fits"
                % (config["output_dir"], prefix, sbins, rbins, k),
                smoothing=smoothing)
        elif (scheme == "grid"):
            discs.bias_grid = fi.FITS(
                "%s/nbc_data/bias_table_hoopoe-v1-%s-disc-%dsbins-%drbins-zbin%d.fits"
                % (config["output_dir"], prefix, sbins, rbins, k))[1].read()
            bulges.bias_grid = fi.FITS(
                "%s/nbc_data/bias_table_hoopoe-v1-%s-bulge-%dsbins-%drbins-zbin%d.fits"
                % (config["output_dir"], prefix, sbins, rbins, k))[1].read()
        elif (scheme == "polynomial"):
            discs.fit(
                "m",
                table=
                "%s/nbc_data/bias_table_hoopoe-v1-%s-disc-%dsbins-%drbins-zbin%d.fits"
                % (config["output_dir"], prefix, sbins, rbins, k))
            discs.fit(
                "a",
                table=
                "%s/nbc_data/bias_table_hoopoe-v1-%s-disc-%dsbins-%drbins-zbin%d.fits"
                % (config["output_dir"], prefix, sbins, rbins, k))
            bulges.fit(
                "m",
                table=
                "%s/nbc_data/bias_table_hoopoe-v1-%s-bulge-%dsbins-%drbins-zbin%d.fits"
                % (config["output_dir"], prefix, sbins, rbins, k))
            bulges.fit(
                "a",
                table=
                "%s/nbc_data/bias_table_hoopoe-v1-%s-bulge-%dsbins-%drbins-zbin%d.fits"
                % (config["output_dir"], prefix, sbins, rbins, k))

        discs.apply(split_half=split_half, scheme=scheme, names=names)
        bulges.apply(split_half=split_half, scheme=scheme, names=names)
        nbc.combine_bd(discs,
                       bulges,
                       split_half=split_half,
                       mask=select_bin,
                       names=["m"] * ("m" in names) + ["c1", "c2"] *
                       ("a" in names))

    return nbc
예제 #6
0
def diagnostics(y1v2,
                hoopoe,
                tomographic_calibration=False,
                histograms=True,
                split_method="random",
                weights=None,
                alpha=True,
                table=True,
                half_tables=True,
                vssnr=True,
                vsredshift=True,
                rbf=True,
                simple_grid=False,
                smoothing=3,
                config=None,
                names=["m", "a"],
                sbins=16,
                rbins=11):

    if rbf:
        sub_dir = "rbf"
    elif simple_grid:
        sub_dir = "grid"
    else:
        sub_dir = "polynomial"

    if not os.path.exists(config["output_dir"]):
        print "Warning - output plots directory does not exist"

    # First make some diagnostic distributions of the observable parameters relevant for the NBC
    if histograms:
        pl.histograms_vs_input(["e", "size", "flux"],
                               hoopoe.truth,
                               data2=y1v2.res,
                               outdir="%s/release/inputs" %
                               config["output_dir"],
                               weights=weights)
        pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"],
                      hoopoe.res,
                      kl=True,
                      data2=y1v2.res,
                      outdir="%s/release/outputs" % config["output_dir"],
                      weights=weights)

    if alpha:
        # Global fit of alpha to compare with previous runs
        plt.close()
        b = bias = di.get_alpha(hoopoe.res,
                                hoopoe.res,
                                nbins=15,
                                names=["alpha11", "alpha22"],
                                xlim=(-0.03, 0.02),
                                xdata_name="mean_hsm_psf_e%d_sky",
                                weights=weights,
                                visual=True)
        plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
        plt.title("Y1 v2 Sim, PSF v02")

        os.system("mkdir -p %s/release/alpha" % config["output_dir"])
        plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png" %
                    config["output_dir"])
        plt.close()

    #Exit here if no calibration diagnostics are needed
    if (not table) and (not vssnr) and (not vsredshift):
        return 0

    # Now compute the calibration using a random half of the simulation
    if vssnr or half_tables:
        nbc = cal.nbc()
        nbc_disc = cal.nbc()
        nbc_bulge = cal.nbc()
        wt1, wt2 = nbc.get_split_data(hoopoe,
                                      weights=weights,
                                      method=split_method)

        nbc_disc.load_from_cat(nbc, name="all")
        nbc_bulge.load_from_cat(nbc, name="all")

    #### Process disc then bulge runs

    # Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
    import pdb
    pdb.set_trace()
    if table:
        if ("match_gridpoints" in config.keys()):
            redges_bulge, sedges_bulge = bin_edges_from_table(
                config["match_gridpoints"], type="bulge")
            redges_disc, sedges_disc = bin_edges_from_table(
                config["match_gridpoints"], type="disc")

            edges_disc = (redges_disc, sedges_disc)
            edges_bulge = (redges_bulge, sedges_bulge)

        else:
            edges_bulge = "equal_number"
            edges_disc = "equal_number"

        if tomographic_calibration:
            zbin_numbers = np.unique(nbc.res["des_bin"])

        if half_tables:
            print "Calculating half tables in %d redshift bins" % zbin_numbers.size
            for k in zbin_numbers:
                print "bin %d" % k
                nbc_disc.compute(
                    split_half=1,
                    fit="disc",
                    weights=wt1,
                    reweight_per_bin=config["reweight_perbin"],
                    resample_per_bin=config["resample_perbin"],
                    refdata=y1v2,
                    binning=edges_disc,
                    redshift_bin=k,
                    rbins=rbins,
                    sbins=sbins,
                    rlim=(1.13, 3.0),
                    slim=(12, 200),
                    table_name=
                    "%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins-zbin%d.fits"
                    % (config["output_dir"], split_method, sbins, rbins, k))
                nbc_bulge.compute(
                    split_half=1,
                    fit="bulge",
                    weights=wt1,
                    reweight_per_bin=config["reweight_perbin"],
                    resample_per_bin=config["resample_perbin"],
                    refdata=y1v2,
                    binning=edges_bulge,
                    redshift_bin=k,
                    rbins=rbins,
                    sbins=sbins,
                    rlim=(1.13, 3.0),
                    slim=(12, 200),
                    table_name=
                    "%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins-zbin%d.fits"
                    % (config["output_dir"], split_method, sbins, rbins, k))

        if table:
            print "Calculating full tables in %d redshift bins" % zbin_numbers.size
            for k in zbin_numbers:
                print "bin %d" % k
                nbc_disc.compute(
                    split_half=0,
                    fit="disc",
                    weights=weights,
                    reweight_per_bin=config["reweight_perbin"],
                    resample_per_bin=config["resample_perbin"],
                    refdata=y1v2,
                    binning=edges_disc,
                    redshift_bin=k,
                    rbins=rbins,
                    sbins=sbins,
                    rlim=(1.13, 3.0),
                    slim=(12, 200),
                    table_name=
                    "%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins-zbin%d.fits"
                    % (config["output_dir"], sbins, rbins, k))
                nbc_bulge.compute(
                    split_half=0,
                    fit="bulge",
                    weights=weights,
                    reweight_per_bin=config["reweight_perbin"],
                    resample_per_bin=config["resample_perbin"],
                    refdata=y1v2,
                    binning=edges_bulge,
                    redshift_bin=k,
                    rbins=rbins,
                    sbins=sbins,
                    rlim=(1.13, 3.0),
                    slim=(12, 200),
                    table_name=
                    "%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins-zbin%d.fits"
                    % (config["output_dir"], sbins, rbins, k))

    if rbf:
        scheme = "rbf"
    elif simple_grid:
        scheme = "grid"
    else:
        scheme = "polynomial"

    if half_tables:
        nbc = calibrate_in_bins(nbc,
                                halfcats=True,
                                split_method=split_method,
                                scheme=scheme,
                                config=config,
                                sbins=sbins,
                                rbins=rbins,
                                smoothing=smoothing,
                                names=names)

    # Finally save some diagnostic plots in tomographic bins
    if vsredshift:
        zbins = [0.2, 0.43, 0.63, 0.9, 1.3]
        tophat = config["tophat_binning"]

        if half_tables:
            plt.close()
            if "m" in names:
                nbc.redshift_diagnostic(bias="m",
                                        label="Uncalibrated",
                                        ls="none",
                                        nbins=3,
                                        fmt=["o", "D"],
                                        colour="steelblue",
                                        weights=wt2,
                                        bins=zbins,
                                        tophat=tophat,
                                        separate_components=False)
                nbc.redshift_diagnostic(bias="m",
                                        label="Calibrated",
                                        ls="none",
                                        nbins=3,
                                        fmt=["^", ">"],
                                        apply_calibration=True,
                                        colour="purple",
                                        weights=wt2,
                                        bins=zbins,
                                        tophat=tophat,
                                        separate_components=False)
                plt.ylabel("Multiplicative Bias $m$")
                plt.legend(loc="center right")
                plt.savefig(
                    "%s/release/%s/m-bias-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
                    % (config["output_dir"], sub_dir, split_method, smoothing,
                       sbins, rbins, int(tophat)))
                plt.close()
            if "a" in names:
                nbc.redshift_diagnostic(bias="alpha",
                                        label="Uncalibrated",
                                        ls="none",
                                        nbins=3,
                                        fmt=["o", "D"],
                                        colour="steelblue",
                                        weights=wt2,
                                        bins=zbins,
                                        tophat=tophat)
                nbc.redshift_diagnostic(bias="alpha",
                                        label="Calibrated",
                                        ls="none",
                                        fmt=["^", ">"],
                                        nbins=3,
                                        apply_calibration=True,
                                        colour="purple",
                                        weights=wt2,
                                        bins=zbins,
                                        tophat=tophat)
                plt.ylabel(r"PSF Leakage $\alpha$")
                plt.legend(loc="upper left")
                plt.savefig(
                    "%s/release/%s/alpha-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
                    % (config["output_dir"], sub_dir, split_method, smoothing,
                       sbins, rbins, int(tophat)))
                plt.close()

        # Finally redo the fits with the full catalogue
        gc.collect()
        nbc = cal.nbc()
        wt1, wt2 = nbc.get_split_data(hoopoe,
                                      weights=weights,
                                      method=split_method)
        nbc = calibrate_in_bins(nbc,
                                halfcats=False,
                                split_method=None,
                                scheme=scheme,
                                config=config,
                                sbins=sbins,
                                rbins=rbins,
                                smoothing=smoothing,
                                names=names)

        import pdb
        pdb.set_trace()

        plt.close()
        if "m" in names:
            nbc.redshift_diagnostic(bias="m",
                                    label="Uncalibrated",
                                    ls="none",
                                    nbins=3,
                                    fmt=["o", "D"],
                                    colour="steelblue",
                                    weights=weights,
                                    split_half=0,
                                    bins=zbins,
                                    tophat=tophat,
                                    separate_components=False)
            bias = nbc.redshift_diagnostic(bias="m",
                                           label="Calibrated",
                                           ls="none",
                                           nbins=3,
                                           fmt=["^", ">"],
                                           apply_calibration=True,
                                           colour="purple",
                                           weights=weights,
                                           split_half=0,
                                           bins=zbins,
                                           tophat=tophat,
                                           separate_components=False)
            plt.ylabel("Multiplicative Bias $m$")
            plt.legend(loc="center right")
            plt.savefig(
                "%s/release/%s/m-bias-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
                % (config["output_dir"], sub_dir, smoothing, sbins, rbins,
                   int(tophat)))
            plt.close()

        if "a" in names:
            nbc.redshift_diagnostic(bias="alpha",
                                    label="Uncalibrated",
                                    ls="none",
                                    nbins=3,
                                    fmt=["o", "D"],
                                    colour="steelblue",
                                    weights=weights,
                                    split_half=0,
                                    bins=zbins,
                                    tophat=tophat)
            nbc.redshift_diagnostic(bias="alpha",
                                    label="Calibrated",
                                    ls="none",
                                    fmt=["^", ">"],
                                    nbins=3,
                                    apply_calibration=True,
                                    colour="purple",
                                    weights=weights,
                                    split_half=0,
                                    bins=zbins,
                                    tophat=tophat)
            plt.ylabel(r"PSF Leakage $\alpha$")
            plt.legend(loc="upper left")
            plt.savefig(
                "%s/release/%s/alpha-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
                % (config["output_dir"], sub_dir, smoothing, sbins, rbins,
                   int(tophat)))
            plt.close()
예제 #7
0
		pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs"%config["output"]["dir"], weights=weights)

	if ("alpha" in outputs):
		# Global fit of alpha to compare with previous runs
		plt.close()
		b = bias=di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22" ], xlim=(-0.03,0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True)
		plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
		plt.title("Y1 v2 Sim, PSF v02")

		os.system("mkdir -p %s/release/alpha"%config["output_dir"])
		plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png"%config["output_dir"])
		plt.close()

	# split the data into two subsets
	if ("snr" in outputs) or ("half_tables" in outputs): 
		nbc = cal.nbc()
		nbc_disc = cal.nbc()
		nbc_bulge = cal.nbc()
		wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method)

		nbc_disc.load_from_cat(nbc, name="all")
		nbc_bulge.load_from_cat(nbc, name="all")

	#### Process disc then bulge runs

	# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
	edges_bulge = "equal_number"
	edges_disc = "equal_number"

	if ("half_tables" in outputs):
		nbc_disc.compute(split_half=1, 
예제 #8
0
def diagnostics(y1v2, hoopoe, histograms=True, alpha=True, table=True, vssnr=True, vsredshift=True, rbf=True):

	if not os.path.exists(config["output_dir"]):
		print "Warning - output plots directory does not exist"

	# First make some diagnostic distributions of the observable parameters relevant for the NBC
	if histograms:
		pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output_dir"])
		pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, data2=y1v2.res, outdir="%s/release/outputs"%config["output_dir"])

	if alpha:
		# Global fit of alpha to compare with previous runs
		plt.close()
		b = di.get_alpha(hoopoe.res, hoopoe.res, nbins=20, xlim=(-0.015,0.025), binning="equal_number", use_weights=False, visual=True)
		plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
		plt.title("Y1 v2 Sim, PSF v02")

		os.system("mkdir -p %s/release/alpha"%config["output_dir"])
		plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit.png"%config["output_dir"])
		plt.close()


	#Exit here if no calibration diagnostics are needed
	if (not table) and (not vssnr) and (not vsredshift):
		return 0

	# Now compute the calibration using a random half of the simulation
	nbc = cal.nbc()
	nbc_disc = cal.nbc()
	nbc_bulge = cal.nbc()
	nbc.get_split_data(hoopoe)

	nbc_disc.load_from_cat(nbc, name="all")
	nbc_bulge.load_from_cat(nbc, name="all")
	

	#### Process disc then bulge runs

	# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
	if table:
		nbc_disc.compute(split_half=1, fit="disc", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
		nbc_bulge.compute(split_half=1, fit="bulge", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])

		nbc_disc.compute(split_half=0, fit="disc", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
		nbc_bulge.compute(split_half=0, fit="bulge", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])

	# Now plot out the points and the resulting smooth fit
	if vssnr:
		if rbf:
			nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
			nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
		else:
			nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
			nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
			nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
			nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])

		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="m", do_half=1, output="%s/m-vs-snr-disc-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="a", do_half=1, output="%s/alpha-vs-snr-disc-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
		plt.close()
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="m", do_half=2, output="%s/m-vs-snr-disc-v1-2.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="a", do_half=2, output="%s/alpha-vs-snr-disc-v1-2.png"%(config["output_dir"]+"release"+"/rbf"*rbf), use_rbf=rbf)
		plt.close()
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="m", do_half=1, output="%s/m-vs-snr-bulge-v1-2.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="a", do_half=1, output="%s/alpha-vs-snr-bulge-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
		plt.close()
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="m", do_half=2, output="%s/m-vs-snr-bulge-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="a", do_half=2, output="%s/alpha-vs-snr-bulge-v1-2.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
		plt.close()

	if rbf:
		nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
		nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])
	else:
		nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
		nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
		nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])
		nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])

	# Apply to the other half
	nbc_disc.apply(split_half=2, use_rbf=rbf)
	nbc_bulge.apply(split_half=2, use_rbf=rbf)

	nbc.get_combined_calibration(nbc_disc,nbc_bulge, split_half=2)

	# Finally save some diagnostic plots in tomographic bins
	if vsredshift: 
		nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o","D"], colour="steelblue")
		nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple")
		plt.ylabel("Multiplicative Bias $m$")
		plt.legend(loc="lower left")
		plt.savefig("%s/release/%s/m-vs-redshift-diagnostic-v1.png"%(config["output_dir"],rbf*"/rbf"))
		plt.close()

		nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=3, fmt=["o","D"], colour="steelblue", error_type="std")
		nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=3, apply_calibration=True, colour="purple", error_type="std")
		plt.ylabel(r"PSF Leakage $\alpha$")
		plt.legend(loc="upper left")
		plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1.png"%(config["output_dir"],rbf*"/rbf"))
		plt.close()
예제 #9
0
def diagnostics(y1v2, hoopoe, histograms=True, split_method="random", weights=None, alpha=True, table=True, half_tables=True, vssnr=True, vsredshift=True, rbf=True, simple_grid=False, smoothing=3, config=None, names=["m", "a"], sbins=16, rbins=11):

	if rbf:
		sub_dir="rbf"
	elif simple_grid:
		sub_dir="grid"
	else:
		sub_dir="polynomial"

	if not os.path.exists(config["output_dir"]):
		print "Warning - output plots directory does not exist"

	# First make some diagnostic distributions of the observable parameters relevant for the NBC
	if histograms:
		pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output_dir"], weights=weights)
		pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs"%config["output_dir"], weights=weights)

	if alpha:
		# Global fit of alpha to compare with previous runs
		plt.close()
		b = bias=di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22" ], xlim=(-0.03,0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True)
		plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
		plt.title("Y1 v2 Sim, PSF v02")

		os.system("mkdir -p %s/release/alpha"%config["output_dir"])
		plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png"%config["output_dir"])
		plt.close()


	#Exit here if no calibration diagnostics are needed
	if (not table) and (not vssnr) and (not vsredshift):
		return 0

	# Now compute the calibration using a random half of the simulation
	if vssnr or half_tables: 
		nbc = cal.nbc()
		nbc_disc = cal.nbc()
		nbc_bulge = cal.nbc()
		wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method)

		nbc_disc.load_from_cat(nbc, name="all")
		nbc_bulge.load_from_cat(nbc, name="all")

	#### Process disc then bulge runs

	# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
	if table:
		if ("match_gridpoints" in config.keys()):
			redges_bulge, sedges_bulge = bin_edges_from_table(config["match_gridpoints"], type="bulge")
			redges_disc, sedges_disc = bin_edges_from_table(config["match_gridpoints"], type="disc")

			edges_disc = (redges_disc, sedges_disc)
			edges_bulge = (redges_bulge, sedges_bulge)

		else:
			edges_bulge = "equal_number"
			edges_disc = "equal_number"

		if half_tables:
			nbc_disc.compute(split_half=1, fit="disc", weights=wt1, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_disc, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method, sbins,rbins))
			nbc_bulge.compute(split_half=1, fit="bulge", weights=wt1, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_bulge, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method, sbins,rbins))

		if config["full_catalogue"]:
			nbc_disc.compute(split_half=0, fit="disc", weights=weights, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_disc, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], sbins, rbins))
			nbc_bulge.compute(split_half=0, fit="bulge", weights=weights, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_bulge, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], sbins,rbins))
		


	# Now plot out the points and the resulting smoothing fit
	if vssnr:
		if rbf:
			nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
			nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
		else:
			nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
			nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
			nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
			nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))

		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-disc-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-disc-v1-1.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-disc-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-disc-v1-2.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-bulge-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-bulge-v1-1.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-bulge-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-bulge-v1-2.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()

	if half_tables and rbf:
		nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins), smoothing=smoothing)
		nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins), smoothing=smoothing)
	elif half_tables and simple_grid:
		nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))[1].read()
		nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))[1].read()
	elif half_tables and (not rbf) and (not simple_grid):
		nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))
		nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))
		nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))
		nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))

	# Apply to the other half
	if rbf:
		scheme="rbf"
	elif simple_grid:
		scheme="grid"
	else:
		scheme="polynomial"

	if half_tables:
		nbc_disc.apply(split_half=2, scheme=scheme, names=names)
		nbc_bulge.apply(split_half=2, scheme=scheme, names=names)

		nbc.combine_bd(nbc_disc,nbc_bulge, split_half=2, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )

	# Finally save some diagnostic plots in tomographic bins

	if vsredshift: 
		zbins=[ 0.2, 0.43, 0.63, 0.9, 1.3]
		tophat = config["tophat_binning"]

		import pdb ; pdb.set_trace()

		if half_tables:
			plt.close()
			if "m" in names:
				bias0=nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
				bias=nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^",">"], apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
				plt.ylabel("Multiplicative Bias $m$")
				plt.legend(loc="center right")
				plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
				plt.close()
			if "a" in names:
				nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat)
				nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=3, apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat)
				plt.ylabel(r"PSF Leakage $\alpha$")
				plt.legend(loc="upper left")
				plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
				plt.close()

		# Finally redo the fits with the full catalogue
		nbc = cal.nbc()
		nbc_disc = cal.nbc()
		nbc_bulge = cal.nbc()
		wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights)

		nbc_disc.load_from_cat(nbc, name="all")
		nbc_bulge.load_from_cat(nbc, name="all")


		if rbf:
			nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
			nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
		elif simple_grid:
			nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))[1].read()
			nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))[1].read()
		else:
			nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
			nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
			nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
			nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))


		nbc_disc.apply(split_half=0, scheme=scheme, names=names)
		nbc_bulge.apply(split_half=0, scheme=scheme, names=names)
		nbc.combine_bd(nbc_disc,nbc_bulge, split_half=0, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )


		plt.close()
		if "m" in names:
			bias0 = nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
			bias = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
			plt.ylabel("Multiplicative Bias $m$")
			plt.legend(loc="center right")
			plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
			plt.close()

			#Save the calibration data so we can reporoduce the paper plots quickly without rerunning all of the above
			out = fi.FITS("hoopoe-v2-nbc-%s.fits"%scheme, "rw")
			dat = np.empty(nbc.res.size, dtype=[("coadd_objects_id", int), ("m", float), ("c1", float), ("c2", float) ])
			for name in ["coadd_objects_id", "m", "c1", "c2"]: dat[name]=nbc.res[name]
			out.write(dat)
			out[-1].write_key("EXTNAME", "nbc_col")
			out[-1].write_key("METHOD", scheme)
			out.close()


			np.savetxt("m-vs-z-dvec-fullcat-uncalibrated-tophatbins.txt", np.array(bias0).T, header="z m1 em1 m2 em2 m em")
			np.savetxt("shear_pipeline/plot_dump/datavecs/m-vs-z-dvec-fullcat-%s-nbc-tophatbins.txt"%scheme, np.array(bias).T, header="z m1 em1 m2 em2 m em")

			rewt=di.get_weights_to_match(y1v2.res["mean_rgpp_rp"], hoopoe.res["mean_rgpp_rp"],nbins=25)

			bmask = (hoopoe.res["is_bulge"]==1)
			mask = (hoopoe.res["bulge_flux"]<10) & (hoopoe.res["disc_flux"]<10)

			rewtb=di.get_weights_to_match(y1v2.res["bulge_flux"][y1v2.res["bulge_flux"]!=0], hoopoe.res["bulge_flux"][bmask & mask],nbins=25, xlim=(0.0,10.))
			rewtd=di.get_weights_to_match(y1v2.res["disc_flux"][y1v2.res["bulge_flux"]==0], hoopoe.res["disc_flux"][np.invert(bmask) & mask],nbins=25, xlim=(0.0,10.))
			wts_flux = np.zeros(hoopoe.res.size)
			wts_flux[bmask & mask] = rewtb
			wts_flux[np.invert(bmask) & mask] = rewtd
			wts_flux = wts_flux[mask]


			bias_size_wtd = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple", weights=rewt, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
			bias_flux_wtd = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple", weights=wts_flux, split_half=0, bins=zbins, tophat=tophat, separate_components=False)

			out=np.vstack((bias_size_wtd[0], bias_size_wtd[-2], bias_size_wtd[-1], bias[-2], bias[-1]))
			np.savetxt("m-vs-z_sizewtd-%s.txt"%scheme, out.T)

			out=np.vstack((bias_flux_wtd[0], bias_flux_wtd[-2], bias_flux_wtd[-1], bias[-2], bias[-1]))
			np.savetxt("m-vs-z_fluxwtd-%s.txt"%scheme, out.T)








		if "a" in names:
			nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat)
			nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=3, apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat)
			plt.ylabel(r"PSF Leakage $\alpha$")
			plt.legend(loc="upper left")
			plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
			plt.close()
예제 #10
0
def do_halfcat_tests(catalogue,
                     weights,
                     params,
                     scheme="grid",
                     split_type="cosmos",
                     sbins=16,
                     rbins=16,
                     smoothing=3):
    # Generate calibrator objects, then split the data
    nbc = cal.nbc()
    nbc_disc = cal.nbc()
    nbc_bulge = cal.nbc()

    wt1, wt2 = nbc.get_split_data(catalogue,
                                  weights=weights,
                                  method=split_type)
    nbc_disc.load_from_cat(nbc, name="all")
    nbc_bulge.load_from_cat(nbc, name="all")

    gc.collect()
    import pdb
    pdb.set_trace()

    # Do any setting up that needs doing prior to applying the calibration to the catalogue
    nbc_disc, nbc_bulge = setup(nbc_disc, nbc_bulge, split_type, scheme,
                                params["output_dir"], params["sbins"],
                                params["rbins"], smoothing)

    # Apply the calibration from one half of the catalogue to the second part
    nbc_disc.apply(split_half=2, scheme=scheme, names=["m", "a"])
    nbc_bulge.apply(split_half=2, scheme=scheme, names=["m", "a"])
    nbc.combine_bd(nbc_disc, nbc_bulge, split_half=2, names=["m", "c1", "c2"])

    # Finally test for residual bias
    zbins = [0.2, 0.43, 0.63, 0.9, 1.3]
    bias0 = nbc.redshift_diagnostic(bias="m",
                                    label="Uncalibrated",
                                    ls="none",
                                    nbins=4,
                                    fmt=["o", "D"],
                                    colour="steelblue",
                                    weights=wt2,
                                    bins=zbins,
                                    tophat=False,
                                    separate_components=False)
    bias = nbc.redshift_diagnostic(bias="m",
                                   label="Calibrated",
                                   ls="none",
                                   nbins=4,
                                   fmt=["^", ">"],
                                   apply_calibration=True,
                                   colour="purple",
                                   weights=wt2,
                                   bins=zbins,
                                   tophat=False,
                                   separate_components=False)

    # Save the residual bias numbers to a text file
    np.savetxt(
        "/home/samuroff/shear_pipeline/plot_dump/datavecs/m-vs-z-dvec-%s-halfcat-uncalibrated.txt"
        % (split_type),
        np.array(bias0).T,
        header="z m1 em1 m2 em2 m em")
    np.savetxt(
        "/home/samuroff/shear_pipeline/plot_dump/datavecs/m-vs-z-dvec-%s-halfcat-%s-nbc-sbins%d-rbins%d.txt"
        % (split_type, scheme, params["sbins"], params["rbins"]),
        np.array(bias).T,
        header="z m1 em1 m2 em2 m em")

    print "Done"
    return 0