Example #1
0
def calculate(y1v2, hoopoe, split_method="none", weights=None, outputs=[], method="grid", smoothing=3, config=None, names=["m", "a"], sbins=16, rbins=16, nthread=1, rank=0, task_allocation={0 : ["disc_table", "bulge_table", "disc_htable", "bulge_htable"]} ):

	if not os.path.exists(config["output"]["dir"]):
		print "Warning - output plots directory does not exist"

	# First make some diagnostic distributions of the observable parameters relevant for the NBC
	if ("histograms" in outputs):
		pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output"]["dir"], weights=weights)
		pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs"%config["output"]["dir"], weights=weights)

	if ("alpha" in outputs):
		# Global fit of alpha to compare with previous runs
		plt.close()
		b = bias=di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22" ], xlim=(-0.03,0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True)
		plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
		plt.title("Y1 v2 Sim, PSF v02")

		os.system("mkdir -p %s/release/alpha"%config["output"]["dir"])
		plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png"%config["output"]["dir"])
		plt.close()

	nbc = cal.nbc()
	nbc_disc = cal.nbc()
	nbc_bulge = cal.nbc()

	# split the data into two subsets
	wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method)

	nbc_disc.load_from_cat(nbc, name="all")
	nbc_bulge.load_from_cat(nbc, name="all")

	#### Process disc then bulge runs

	# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
	edges_bulge = "equal_number"
	edges_disc = "equal_number"

	if ("half_tables" in outputs):
		if ("disc_htable" in task_allocation[rank]):
			nbc_disc.compute(split_half=1, 
			    fit="disc", weights=wt1,
			    use_catalogue_weights=config["selection"]["weights"], 
				reweight_per_bin=False, resample_per_bin=False,
				refdata=y1v2, binning=edges_disc, 
				rbins=rbins, sbins=sbins, 
				rlim=(1.13,3.0), slim=(12,200),
				table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method, sbins,rbins))
		if ("bulge_htable" in task_allocation[rank]):
			nbc_bulge.compute(split_half=1, 
			    fit="bulge", weights=wt1,
			    use_catalogue_weights=config["selection"]["weights"],
				reweight_per_bin=False, resample_per_bin=False,
				refdata=y1v2, binning=edges_bulge,
				rbins=rbins, sbins=sbins,
				rlim=(1.13,3.0), slim=(12,200),
				table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method, sbins,rbins))

	if ("tables" in outputs):
		if ("disc_table" in task_allocation[rank]):
			nbc_disc.compute(split_half=0,
			    fit="disc", weights=weights,
			    use_catalogue_weights=config["selection"]["weights"],
				reweight_per_bin=False, resample_per_bin=False,
				refdata=y1v2, binning=edges_disc,
				rbins=rbins, sbins=sbins,
				rlim=(1.13,3.0), slim=(12,200),
				table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], sbins, rbins))
		if ("bulge_table" in task_allocation[rank]):
			nbc_bulge.compute(split_half=0,
			    fit="bulge", weights=weights,
			    use_catalogue_weights=config["selection"]["weights"],
				reweight_per_bin=False, resample_per_bin=False,
				refdata=y1v2, binning=edges_bulge,
				rbins=rbins, sbins=sbins,
				rlim=(1.13,3.0), slim=(12,200),
				table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], sbins,rbins))


	sub_dir = method.lower()
	# Now plot out the points and the resulting smoothing fit
	if ("snr" in outputs):
		if (method.lower()=="rbf") or (method.lower()=="grid"):
			nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
			nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
		elif (method.lower()=="polynomial"):
			nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
			nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
			nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
			nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))

		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-disc-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-disc-v1-1.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-disc-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-disc-v1-2.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-bulge-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-bulge-v1-1.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-bulge-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output"]["dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-bulge-v1-2.png"%(config["output"]["dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()

	if (split_method!="none"):
		# Derive a calibration from the grid in whichever way is specified in the config file
		if (method=="rbf"):
			nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins), smoothing=smoothing)
			nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins), smoothing=smoothing)
		elif (method=="grid"):
			nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))[1].read()
			nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))[1].read()
		elif (method=="polynomial"):
			nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))
			nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))
			nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))
			nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"], split_method,sbins,rbins))

		# Apply it to the other half
		nbc_disc.apply(split_half=2, scheme=method, names=names)
		nbc_bulge.apply(split_half=2, scheme=method, names=names)
		nbc.combine_bd(nbc_disc,nbc_bulge, split_half=2, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )

	# Finally save some diagnostic plots in tomographic bins
	if ("redshift" in outputs): 
		zbins=[ 0.2, 0.43, 0.63, 0.9, 1.3]
		tophat = config["calibration"]["ztype"]=="tophat"

		# Only do the redshift plots with the split catalogues if they were recalculated this time
		plt.close()
		if "m" in names:
			bias0=nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
			bias=nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^",">"], apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
			plt.ylabel("Multiplicative Bias $m$")
			plt.legend(loc="center right")
			plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
			plt.close()
		if "a" in names:
			nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat)
			nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=4, apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat)
			plt.ylabel(r"PSF Leakage $\alpha$")
			plt.legend(loc="upper left")
			plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
			plt.close()

		np.savetxt("%s/data/m-vs-z-uncalibrated-%s-halfcat-unblinded%s.txt"%(config["output"]["dir"],split_method, ""+"-tophat"*int(tophat)), np.array(bias0))
		np.savetxt("%s/data/m-vs-z-%s_nbc-%s-halfcat-unblinded%s.txt"%(config["output"]["dir"],method,split_method, ""+"-tophat"*int(tophat)),np.array(bias))

		# Finally redo the fits with the full catalogue
		nbc = cal.nbc()
		nbc_disc = cal.nbc()
		nbc_bulge = cal.nbc()
		wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights)

		nbc_disc.load_from_cat(nbc, name="all")
		nbc_bulge.load_from_cat(nbc, name="all")

		if (method.lower()=="rbf"):
			nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
			nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins), smoothing=smoothing)
		elif (method.lower()=="grid"):
			nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read()
			nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))[1].read()
		elif (method.lower()=="polynomial"):
			nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
			nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
			nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))
			nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output"]["dir"],sbins,rbins))


		nbc_disc.apply(split_half=0, scheme=method, names=names)
		nbc_bulge.apply(split_half=0, scheme=method, names=names)
		nbc.combine_bd(nbc_disc,nbc_bulge, split_half=0, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )

		nbc.preserve_calibration_col("%s/data/hoopoe-%s-nbc_column.fits"%(config["output"]["dir"],method))

		plt.close()
		if "m" in names:
			bias0 = nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
			bias = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^",">"], apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
			plt.ylabel("Multiplicative Bias $m$")
			plt.legend(loc="center right")
			plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
			plt.close()
			np.savetxt("%s/data/m-vs-z-uncalibrated-fullcat-unblinded%s.txt"%(config["output"]["dir"], ""+"-tophat"*int(tophat)), np.array(bias0))
			np.savetxt("%s/data/m-vs-z-%s_nbc-fullcat-unblinded%s.txt"%(config["output"]["dir"], method,""+"-tophat"*int(tophat)), np.array(bias))

		if "a" in names:
			nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=4, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat)
			nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=4, apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat)
			plt.ylabel(r"PSF Leakage $\alpha$")
			plt.legend(loc="upper left")
			plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output"]["dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
			plt.close()

		print "Done"
		return 0
Example #2
0
	def compute(self, split_half=0, fit="bord", apply_calibration=False, table_name=None, ellipticity_name="e", sbins=10, rbins=5, binning="equal_number",rlim=(1,3), slim=(10,1000)):
		print 'measuring bias'

		if split_half>0:
			exec "data = self.res%d"%split_half
			print "using half %d of the catalogue (%d objects)"%(split_half,data.size)
			if hasattr(self, "truth"):
				exec "tr = self.truth%d"%split_half
		else:
			data = self.res
			print "using the full catalogue (%d objects)"%(data.size)
			if hasattr(self, "truth"):
				tr = self.truth


		if fit.lower()!="bord":
			print "Using %s only galaxies"%fit
			val = int(fit.lower()=="bulge")
			sel_fit = data["is_bulge"]==val
		else:
			sel_fit = np.ones_like(data).astype(bool)


		data = data[sel_fit]
		tr = tr[sel_fit]

		sel_lim = (data["snr"]>slim[0]) & (data["snr"]<slim[1]) & (data["mean_rgpp_rp"]>rlim[0]) & (data["mean_rgpp_rp"]<rlim[1])
		data = data[sel_lim]
		tr = tr[sel_lim]

		if isinstance(binning,str) : 
			if binning.lower()=="uniform":
				snr_edges = np.logspace(np.log10(slim[0]),np.log10(slim[1]),sbins+1)
				rgp_edges = np.linspace(rlim[0],rlim[1],rbins+1)
			elif binning.lower()=="equal_number":
				snr_edges = di.find_bin_edges(np.log10(data["snr"]), sbins)
				rgp_edges = di.find_bin_edges(np.log10(data["mean_rgpp_rp"]), rbins)

		snr_centres = (snr_edges[1:]+snr_edges[:-1])/2.0
		rgp_centres = (rgp_edges[1:]+rgp_edges[:-1])/2.0

		list_bias = []
		bias_grid=[]

		b = di.get_bias(tr, data, nbins=5, apply_calibration=apply_calibration, ellipticity_name=ellipticity_name, binning="equal_number", names=["m","c","m11","m22","c11","c22"], silent=True)
		print "Global biases:"
		print "m11 : ", b["m11"]
		print "m22 : ", b["m22"]
		print "m : ", b["m"]
		print "c11 : ", b["c11"]
		print "c22 : ", b["c22"]
		print "c : ", b["c"]

		print "Will do dynamic binning in SNR"

		for i in xrange(len(rgp_edges)-1):
			snr_samp = data["snr"][(np.log10(data['mean_rgpp_rp']) > rgp_edges[i]) & (np.log10(data['mean_rgpp_rp']) < rgp_edges[i+1])]
			snr_edges=di.find_bin_edges(np.log10(snr_samp), sbins)
			for j in xrange(len(snr_edges)-1):
				empty=False
				print "bin %d %d  snr = [%2.3f-%2.3f] rgpp/rp = [%2.3f-%2.3f]"%(j, i, 10**snr_edges[j], 10**snr_edges[j+1], 10**rgp_edges[i], 10**rgp_edges[i+1] )

				# Select in bins of snr and size
				select = (np.log10(data['snr']) > snr_edges[j]) & (np.log10(data['snr']) < snr_edges[j+1]) & (np.log10(data['mean_rgpp_rp']) > rgp_edges[i]) & (np.log10(data['mean_rgpp_rp']) < rgp_edges[i+1])
				ngal = np.nonzero(select.astype(int))[0].size

				# Raise an error if there are too few simulated galaxies in a given bin
				if ngal < 60:
					print "Warning: <100 galaxies in bin %d, %d (ngal=%d)"%(i,j, ngal)
					empty=False
				if ngal==0:
					print "Warning: no galaxies in bin %d, %d "%(i,j)
					empty=True

				vrgp_mid = rgp_centres[i]
				vsnr_mid = snr_centres[j]
				vrgp_min = rgp_edges[i]
				vsnr_min = snr_edges[j]
				vrgp_max = rgp_edges[i+1]
				vsnr_max = snr_edges[j+1]

				if ngal==0:
					print "Warning: no galaxies in bin %d, %d"%(i,j)
					list_bias.append([j, i, ngal, 0, vrgp_min, vrgp_max, vsnr_min, vsnr_max, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.	, 0., 0.])
					continue

				

				filename_str = 'snr%2.2f.rgpp%2.2f' % (10**vsnr_mid,10**vrgp_mid)
				b = di.get_bias(tr[select], data[select], apply_calibration=apply_calibration, nbins=5, ellipticity_name=ellipticity_name, binning="equal_number", names=["m","c","m11","m22","c11","c22"], silent=True)
				a = di.get_alpha(data[select], data[select], nbins=5, xlim=(-0.015, 0.02), binning="equal_number", names=["alpha", "alpha11", "alpha22"], silent=True, use_weights=False)

				list_bias.append([j, i, ngal, 10**vrgp_min, 10**vrgp_max, 10**vsnr_min, 10**vsnr_max, b["m"][0], b["m"][1], b["c"][0], b["c"][1], b["m11"][0], b["m11"][1], b["m22"][0], b["m22"][1], b["c11"][0], b["c11"][1], b["c22"][0], b["c22"][1], a["alpha"][0], a["alpha"][1], a["alpha11"][0], a["alpha11"][1], a["alpha22"][0], a["alpha22"][1] ])

		lab=["j","i","ngal","rgp_lower","rgp_upper","snr_lower","snr_upper","m","err_m","c","err_c","m1","err_m1","m2","err_m2","c1","err_c1","c2",	"err_c2","alpha","err_alpha","alpha11","err_alpha11","alpha22","err_alpha22"]
		dt = {'names': lab, 'formats': ['i4']*3 + ['f8']*22 }
		arr_bias = np.core.records.fromarrays(np.array(list_bias).transpose(), dtype=dt)

		if table_name is None:
			filename_table_bias = 'bias_table-%s-selection_section%d%s.fits'%(fit, split_half,"_calibrated"*apply_calibration)
		else:
			filename_table_bias = table_name

		import pyfits
		pyfits.writeto(filename_table_bias,arr_bias,clobber=True)
		print 'saved %s'%filename_table_bias
Example #3
0
def diagnostics(y1v2,
                hoopoe,
                tomographic_calibration=False,
                histograms=True,
                split_method="random",
                weights=None,
                alpha=True,
                table=True,
                half_tables=True,
                vssnr=True,
                vsredshift=True,
                rbf=True,
                simple_grid=False,
                smoothing=3,
                config=None,
                names=["m", "a"],
                sbins=16,
                rbins=11):

    if rbf:
        sub_dir = "rbf"
    elif simple_grid:
        sub_dir = "grid"
    else:
        sub_dir = "polynomial"

    if not os.path.exists(config["output_dir"]):
        print "Warning - output plots directory does not exist"

    # First make some diagnostic distributions of the observable parameters relevant for the NBC
    if histograms:
        pl.histograms_vs_input(["e", "size", "flux"],
                               hoopoe.truth,
                               data2=y1v2.res,
                               outdir="%s/release/inputs" %
                               config["output_dir"],
                               weights=weights)
        pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"],
                      hoopoe.res,
                      kl=True,
                      data2=y1v2.res,
                      outdir="%s/release/outputs" % config["output_dir"],
                      weights=weights)

    if alpha:
        # Global fit of alpha to compare with previous runs
        plt.close()
        b = bias = di.get_alpha(hoopoe.res,
                                hoopoe.res,
                                nbins=15,
                                names=["alpha11", "alpha22"],
                                xlim=(-0.03, 0.02),
                                xdata_name="mean_hsm_psf_e%d_sky",
                                weights=weights,
                                visual=True)
        plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
        plt.title("Y1 v2 Sim, PSF v02")

        os.system("mkdir -p %s/release/alpha" % config["output_dir"])
        plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png" %
                    config["output_dir"])
        plt.close()

    #Exit here if no calibration diagnostics are needed
    if (not table) and (not vssnr) and (not vsredshift):
        return 0

    # Now compute the calibration using a random half of the simulation
    if vssnr or half_tables:
        nbc = cal.nbc()
        nbc_disc = cal.nbc()
        nbc_bulge = cal.nbc()
        wt1, wt2 = nbc.get_split_data(hoopoe,
                                      weights=weights,
                                      method=split_method)

        nbc_disc.load_from_cat(nbc, name="all")
        nbc_bulge.load_from_cat(nbc, name="all")

    #### Process disc then bulge runs

    # Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
    import pdb
    pdb.set_trace()
    if table:
        if ("match_gridpoints" in config.keys()):
            redges_bulge, sedges_bulge = bin_edges_from_table(
                config["match_gridpoints"], type="bulge")
            redges_disc, sedges_disc = bin_edges_from_table(
                config["match_gridpoints"], type="disc")

            edges_disc = (redges_disc, sedges_disc)
            edges_bulge = (redges_bulge, sedges_bulge)

        else:
            edges_bulge = "equal_number"
            edges_disc = "equal_number"

        if tomographic_calibration:
            zbin_numbers = np.unique(nbc.res["des_bin"])

        if half_tables:
            print "Calculating half tables in %d redshift bins" % zbin_numbers.size
            for k in zbin_numbers:
                print "bin %d" % k
                nbc_disc.compute(
                    split_half=1,
                    fit="disc",
                    weights=wt1,
                    reweight_per_bin=config["reweight_perbin"],
                    resample_per_bin=config["resample_perbin"],
                    refdata=y1v2,
                    binning=edges_disc,
                    redshift_bin=k,
                    rbins=rbins,
                    sbins=sbins,
                    rlim=(1.13, 3.0),
                    slim=(12, 200),
                    table_name=
                    "%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins-zbin%d.fits"
                    % (config["output_dir"], split_method, sbins, rbins, k))
                nbc_bulge.compute(
                    split_half=1,
                    fit="bulge",
                    weights=wt1,
                    reweight_per_bin=config["reweight_perbin"],
                    resample_per_bin=config["resample_perbin"],
                    refdata=y1v2,
                    binning=edges_bulge,
                    redshift_bin=k,
                    rbins=rbins,
                    sbins=sbins,
                    rlim=(1.13, 3.0),
                    slim=(12, 200),
                    table_name=
                    "%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins-zbin%d.fits"
                    % (config["output_dir"], split_method, sbins, rbins, k))

        if table:
            print "Calculating full tables in %d redshift bins" % zbin_numbers.size
            for k in zbin_numbers:
                print "bin %d" % k
                nbc_disc.compute(
                    split_half=0,
                    fit="disc",
                    weights=weights,
                    reweight_per_bin=config["reweight_perbin"],
                    resample_per_bin=config["resample_perbin"],
                    refdata=y1v2,
                    binning=edges_disc,
                    redshift_bin=k,
                    rbins=rbins,
                    sbins=sbins,
                    rlim=(1.13, 3.0),
                    slim=(12, 200),
                    table_name=
                    "%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins-zbin%d.fits"
                    % (config["output_dir"], sbins, rbins, k))
                nbc_bulge.compute(
                    split_half=0,
                    fit="bulge",
                    weights=weights,
                    reweight_per_bin=config["reweight_perbin"],
                    resample_per_bin=config["resample_perbin"],
                    refdata=y1v2,
                    binning=edges_bulge,
                    redshift_bin=k,
                    rbins=rbins,
                    sbins=sbins,
                    rlim=(1.13, 3.0),
                    slim=(12, 200),
                    table_name=
                    "%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins-zbin%d.fits"
                    % (config["output_dir"], sbins, rbins, k))

    if rbf:
        scheme = "rbf"
    elif simple_grid:
        scheme = "grid"
    else:
        scheme = "polynomial"

    if half_tables:
        nbc = calibrate_in_bins(nbc,
                                halfcats=True,
                                split_method=split_method,
                                scheme=scheme,
                                config=config,
                                sbins=sbins,
                                rbins=rbins,
                                smoothing=smoothing,
                                names=names)

    # Finally save some diagnostic plots in tomographic bins
    if vsredshift:
        zbins = [0.2, 0.43, 0.63, 0.9, 1.3]
        tophat = config["tophat_binning"]

        if half_tables:
            plt.close()
            if "m" in names:
                nbc.redshift_diagnostic(bias="m",
                                        label="Uncalibrated",
                                        ls="none",
                                        nbins=3,
                                        fmt=["o", "D"],
                                        colour="steelblue",
                                        weights=wt2,
                                        bins=zbins,
                                        tophat=tophat,
                                        separate_components=False)
                nbc.redshift_diagnostic(bias="m",
                                        label="Calibrated",
                                        ls="none",
                                        nbins=3,
                                        fmt=["^", ">"],
                                        apply_calibration=True,
                                        colour="purple",
                                        weights=wt2,
                                        bins=zbins,
                                        tophat=tophat,
                                        separate_components=False)
                plt.ylabel("Multiplicative Bias $m$")
                plt.legend(loc="center right")
                plt.savefig(
                    "%s/release/%s/m-bias-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
                    % (config["output_dir"], sub_dir, split_method, smoothing,
                       sbins, rbins, int(tophat)))
                plt.close()
            if "a" in names:
                nbc.redshift_diagnostic(bias="alpha",
                                        label="Uncalibrated",
                                        ls="none",
                                        nbins=3,
                                        fmt=["o", "D"],
                                        colour="steelblue",
                                        weights=wt2,
                                        bins=zbins,
                                        tophat=tophat)
                nbc.redshift_diagnostic(bias="alpha",
                                        label="Calibrated",
                                        ls="none",
                                        fmt=["^", ">"],
                                        nbins=3,
                                        apply_calibration=True,
                                        colour="purple",
                                        weights=wt2,
                                        bins=zbins,
                                        tophat=tophat)
                plt.ylabel(r"PSF Leakage $\alpha$")
                plt.legend(loc="upper left")
                plt.savefig(
                    "%s/release/%s/alpha-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
                    % (config["output_dir"], sub_dir, split_method, smoothing,
                       sbins, rbins, int(tophat)))
                plt.close()

        # Finally redo the fits with the full catalogue
        gc.collect()
        nbc = cal.nbc()
        wt1, wt2 = nbc.get_split_data(hoopoe,
                                      weights=weights,
                                      method=split_method)
        nbc = calibrate_in_bins(nbc,
                                halfcats=False,
                                split_method=None,
                                scheme=scheme,
                                config=config,
                                sbins=sbins,
                                rbins=rbins,
                                smoothing=smoothing,
                                names=names)

        import pdb
        pdb.set_trace()

        plt.close()
        if "m" in names:
            nbc.redshift_diagnostic(bias="m",
                                    label="Uncalibrated",
                                    ls="none",
                                    nbins=3,
                                    fmt=["o", "D"],
                                    colour="steelblue",
                                    weights=weights,
                                    split_half=0,
                                    bins=zbins,
                                    tophat=tophat,
                                    separate_components=False)
            bias = nbc.redshift_diagnostic(bias="m",
                                           label="Calibrated",
                                           ls="none",
                                           nbins=3,
                                           fmt=["^", ">"],
                                           apply_calibration=True,
                                           colour="purple",
                                           weights=weights,
                                           split_half=0,
                                           bins=zbins,
                                           tophat=tophat,
                                           separate_components=False)
            plt.ylabel("Multiplicative Bias $m$")
            plt.legend(loc="center right")
            plt.savefig(
                "%s/release/%s/m-bias-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
                % (config["output_dir"], sub_dir, smoothing, sbins, rbins,
                   int(tophat)))
            plt.close()

        if "a" in names:
            nbc.redshift_diagnostic(bias="alpha",
                                    label="Uncalibrated",
                                    ls="none",
                                    nbins=3,
                                    fmt=["o", "D"],
                                    colour="steelblue",
                                    weights=weights,
                                    split_half=0,
                                    bins=zbins,
                                    tophat=tophat)
            nbc.redshift_diagnostic(bias="alpha",
                                    label="Calibrated",
                                    ls="none",
                                    fmt=["^", ">"],
                                    nbins=3,
                                    apply_calibration=True,
                                    colour="purple",
                                    weights=weights,
                                    split_half=0,
                                    bins=zbins,
                                    tophat=tophat)
            plt.ylabel(r"PSF Leakage $\alpha$")
            plt.legend(loc="upper left")
            plt.savefig(
                "%s/release/%s/alpha-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d-tomographic_calibration.png"
                % (config["output_dir"], sub_dir, smoothing, sbins, rbins,
                   int(tophat)))
            plt.close()
Example #4
0

def calculate(y1v2, hoopoe, split_method="none", weights=None, outputs=[] method="grid", smoothing=3, config=None, names=["m", "a"], sbins=16, rbins=16):

	if not os.path.exists(config["output_dir"]):
		print "Warning - output plots directory does not exist"

	# First make some diagnostic distributions of the observable parameters relevant for the NBC
	if ("histograms" in outputs):
		pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output"]["dir"], weights=weights)
		pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs"%config["output"]["dir"], weights=weights)

	if ("alpha" in outputs):
		# Global fit of alpha to compare with previous runs
		plt.close()
		b = bias=di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22" ], xlim=(-0.03,0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True)
		plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
		plt.title("Y1 v2 Sim, PSF v02")

		os.system("mkdir -p %s/release/alpha"%config["output_dir"])
		plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png"%config["output_dir"])
		plt.close()

	# split the data into two subsets
	if ("snr" in outputs) or ("half_tables" in outputs): 
		nbc = cal.nbc()
		nbc_disc = cal.nbc()
		nbc_bulge = cal.nbc()
		wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method)

		nbc_disc.load_from_cat(nbc, name="all")
def diagnostics(y1v2, hoopoe, histograms=True, alpha=True, table=True, vssnr=True, vsredshift=True, rbf=True):

	if not os.path.exists(config["output_dir"]):
		print "Warning - output plots directory does not exist"

	# First make some diagnostic distributions of the observable parameters relevant for the NBC
	if histograms:
		pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output_dir"])
		pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, data2=y1v2.res, outdir="%s/release/outputs"%config["output_dir"])

	if alpha:
		# Global fit of alpha to compare with previous runs
		plt.close()
		b = di.get_alpha(hoopoe.res, hoopoe.res, nbins=20, xlim=(-0.015,0.025), binning="equal_number", use_weights=False, visual=True)
		plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
		plt.title("Y1 v2 Sim, PSF v02")

		os.system("mkdir -p %s/release/alpha"%config["output_dir"])
		plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit.png"%config["output_dir"])
		plt.close()


	#Exit here if no calibration diagnostics are needed
	if (not table) and (not vssnr) and (not vsredshift):
		return 0

	# Now compute the calibration using a random half of the simulation
	nbc = cal.nbc()
	nbc_disc = cal.nbc()
	nbc_bulge = cal.nbc()
	nbc.get_split_data(hoopoe)

	nbc_disc.load_from_cat(nbc, name="all")
	nbc_bulge.load_from_cat(nbc, name="all")
	

	#### Process disc then bulge runs

	# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
	if table:
		nbc_disc.compute(split_half=1, fit="disc", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
		nbc_bulge.compute(split_half=1, fit="bulge", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])

		nbc_disc.compute(split_half=0, fit="disc", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
		nbc_bulge.compute(split_half=0, fit="bulge", rbins=10, sbins=10, rlim=(0.9,4.5), slim=(10,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])

	# Now plot out the points and the resulting smooth fit
	if vssnr:
		if rbf:
			nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
			nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
		else:
			nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
			nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"])
			nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])
			nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"])

		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="m", do_half=1, output="%s/m-vs-snr-disc-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="a", do_half=1, output="%s/alpha-vs-snr-disc-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
		plt.close()
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="m", do_half=2, output="%s/m-vs-snr-disc-v1-2.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc.fits"%config["output_dir"], bias_name="a", do_half=2, output="%s/alpha-vs-snr-disc-v1-2.png"%(config["output_dir"]+"release"+"/rbf"*rbf), use_rbf=rbf)
		plt.close()
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="m", do_half=1, output="%s/m-vs-snr-bulge-v1-2.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="a", do_half=1, output="%s/alpha-vs-snr-bulge-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
		plt.close()
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="m", do_half=2, output="%s/m-vs-snr-bulge-v1-1.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge.fits"%config["output_dir"], bias_name="a", do_half=2, output="%s/alpha-vs-snr-bulge-v1-2.png"%(config["output_dir"]+"/release"+"/rbf"*rbf), use_rbf=rbf)
		plt.close()

	if rbf:
		nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
		nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])
	else:
		nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
		nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-disc.fits"%config["output_dir"])
		nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])
		nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-halfcat-bulge.fits"%config["output_dir"])

	# Apply to the other half
	nbc_disc.apply(split_half=2, use_rbf=rbf)
	nbc_bulge.apply(split_half=2, use_rbf=rbf)

	nbc.get_combined_calibration(nbc_disc,nbc_bulge, split_half=2)

	# Finally save some diagnostic plots in tomographic bins
	if vsredshift: 
		nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o","D"], colour="steelblue")
		nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple")
		plt.ylabel("Multiplicative Bias $m$")
		plt.legend(loc="lower left")
		plt.savefig("%s/release/%s/m-vs-redshift-diagnostic-v1.png"%(config["output_dir"],rbf*"/rbf"))
		plt.close()

		nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=3, fmt=["o","D"], colour="steelblue", error_type="std")
		nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=3, apply_calibration=True, colour="purple", error_type="std")
		plt.ylabel(r"PSF Leakage $\alpha$")
		plt.legend(loc="upper left")
		plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1.png"%(config["output_dir"],rbf*"/rbf"))
		plt.close()
Example #6
0
def diagnostics(y1v2, hoopoe, histograms=True, split_method="random", weights=None, alpha=True, table=True, half_tables=True, vssnr=True, vsredshift=True, rbf=True, simple_grid=False, smoothing=3, config=None, names=["m", "a"], sbins=16, rbins=11):

	if rbf:
		sub_dir="rbf"
	elif simple_grid:
		sub_dir="grid"
	else:
		sub_dir="polynomial"

	if not os.path.exists(config["output_dir"]):
		print "Warning - output plots directory does not exist"

	# First make some diagnostic distributions of the observable parameters relevant for the NBC
	if histograms:
		pl.histograms_vs_input(["e", "size", "flux"], hoopoe.truth, data2=y1v2.res, outdir="%s/release/inputs"%config["output_dir"], weights=weights)
		pl.histograms(["snr", "e", "size", "rgpp", "flux", "psfe", "psf_size"], hoopoe.res, kl=True, data2=y1v2.res, outdir="%s/release/outputs"%config["output_dir"], weights=weights)

	if alpha:
		# Global fit of alpha to compare with previous runs
		plt.close()
		b = bias=di.get_alpha(hoopoe.res, hoopoe.res, nbins=15, names=["alpha11", "alpha22" ], xlim=(-0.03,0.02), xdata_name="mean_hsm_psf_e%d_sky", weights=weights, visual=True)
		plt.subplots_adjust(wspace=0, hspace=0, top=0.95, bottom=0.1)
		plt.title("Y1 v2 Sim, PSF v02")

		os.system("mkdir -p %s/release/alpha"%config["output_dir"])
		plt.savefig("%s/release/alpha/alphaplot-e-vs-epsf-linfit0.png"%config["output_dir"])
		plt.close()


	#Exit here if no calibration diagnostics are needed
	if (not table) and (not vssnr) and (not vsredshift):
		return 0

	# Now compute the calibration using a random half of the simulation
	if vssnr or half_tables: 
		nbc = cal.nbc()
		nbc_disc = cal.nbc()
		nbc_bulge = cal.nbc()
		wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights, method=split_method)

		nbc_disc.load_from_cat(nbc, name="all")
		nbc_bulge.load_from_cat(nbc, name="all")

	#### Process disc then bulge runs

	# Fit the binned galaxies. Save one set of calibration data for disc objects, one for bulges
	if table:
		if ("match_gridpoints" in config.keys()):
			redges_bulge, sedges_bulge = bin_edges_from_table(config["match_gridpoints"], type="bulge")
			redges_disc, sedges_disc = bin_edges_from_table(config["match_gridpoints"], type="disc")

			edges_disc = (redges_disc, sedges_disc)
			edges_bulge = (redges_bulge, sedges_bulge)

		else:
			edges_bulge = "equal_number"
			edges_disc = "equal_number"

		if half_tables:
			nbc_disc.compute(split_half=1, fit="disc", weights=wt1, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_disc, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method, sbins,rbins))
			nbc_bulge.compute(split_half=1, fit="bulge", weights=wt1, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_bulge, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method, sbins,rbins))

		if config["full_catalogue"]:
			nbc_disc.compute(split_half=0, fit="disc", weights=weights, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_disc, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], sbins, rbins))
			nbc_bulge.compute(split_half=0, fit="bulge", weights=weights, reweight_per_bin=config["reweight_perbin"], resample_per_bin=config["resample_perbin"], refdata=y1v2, binning=edges_bulge, rbins=rbins, sbins=sbins, rlim=(1.13,3.0), slim=(12,200), table_name="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], sbins,rbins))
		


	# Now plot out the points and the resulting smoothing fit
	if vssnr:
		if rbf:
			nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
			nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
		else:
			nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
			nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
			nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
			nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))

		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-disc-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-disc-v1-1.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-disc-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_disc.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-disc-v1-2.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=1, output="%s/m-vs-snr-bulge-v1-2-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=1, output="%s/alpha-vs-snr-bulge-v1-1.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="m", do_half=2, output="%s/m-vs-snr-bulge-v1-1-s%2.2f-sbins%d-rbins%d.png"%(config["output_dir"]+"/release/"+ sub_dir, smoothing, sbins, rbins), use_rbf=rbf)
		nbc_bulge.bias_fit_vs_pts(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), bias_name="a", do_half=2, output="%s/alpha-vs-snr-bulge-v1-2.png"%(config["output_dir"]+"/release/"+ sub_dir), use_rbf=rbf)
		plt.close()

	if half_tables and rbf:
		nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins), smoothing=smoothing)
		nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins), smoothing=smoothing)
	elif half_tables and simple_grid:
		nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))[1].read()
		nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))[1].read()
	elif half_tables and (not rbf) and (not simple_grid):
		nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))
		nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-disc-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))
		nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))
		nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-%s-halfcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"], split_method,sbins,rbins))

	# Apply to the other half
	if rbf:
		scheme="rbf"
	elif simple_grid:
		scheme="grid"
	else:
		scheme="polynomial"

	if half_tables:
		nbc_disc.apply(split_half=2, scheme=scheme, names=names)
		nbc_bulge.apply(split_half=2, scheme=scheme, names=names)

		nbc.combine_bd(nbc_disc,nbc_bulge, split_half=2, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )

	# Finally save some diagnostic plots in tomographic bins

	if vsredshift: 
		zbins=[ 0.2, 0.43, 0.63, 0.9, 1.3]
		tophat = config["tophat_binning"]

		import pdb ; pdb.set_trace()

		if half_tables:
			plt.close()
			if "m" in names:
				bias0=nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
				bias=nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=4, fmt=["^",">"], apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat, separate_components=False)
				plt.ylabel("Multiplicative Bias $m$")
				plt.legend(loc="center right")
				plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
				plt.close()
			if "a" in names:
				nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=wt2, bins=zbins, tophat=tophat)
				nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=3, apply_calibration=True, colour="purple", weights=wt2, bins=zbins, tophat=tophat)
				plt.ylabel(r"PSF Leakage $\alpha$")
				plt.legend(loc="upper left")
				plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-%s-halfcat-s%2.3f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"], sub_dir, split_method, smoothing, sbins,rbins, int(tophat)))
				plt.close()

		# Finally redo the fits with the full catalogue
		nbc = cal.nbc()
		nbc_disc = cal.nbc()
		nbc_bulge = cal.nbc()
		wt1, wt2 = nbc.get_split_data(hoopoe, weights=weights)

		nbc_disc.load_from_cat(nbc, name="all")
		nbc_bulge.load_from_cat(nbc, name="all")


		if rbf:
			nbc_disc.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
			nbc_bulge.fit_rbf(table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins), smoothing=smoothing)
		elif simple_grid:
			nbc_disc.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))[1].read()
			nbc_bulge.bias_grid = fi.FITS("%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))[1].read()
		else:
			nbc_disc.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
			nbc_disc.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-disc-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
			nbc_bulge.fit("m", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))
			nbc_bulge.fit("a", table="%s/nbc_data/bias_table_hoopoe-v1-fullcat-bulge-%dsbins-%drbins.fits"%(config["output_dir"],sbins,rbins))


		nbc_disc.apply(split_half=0, scheme=scheme, names=names)
		nbc_bulge.apply(split_half=0, scheme=scheme, names=names)
		nbc.combine_bd(nbc_disc,nbc_bulge, split_half=0, names=["m"]*("m" in names) + ["c1", "c2"]*("a" in names) )


		plt.close()
		if "m" in names:
			bias0 = nbc.redshift_diagnostic(bias="m", label="Uncalibrated", ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
			bias = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
			plt.ylabel("Multiplicative Bias $m$")
			plt.legend(loc="center right")
			plt.savefig("%s/release/%s/m-bias-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
			plt.close()

			#Save the calibration data so we can reporoduce the paper plots quickly without rerunning all of the above
			out = fi.FITS("hoopoe-v2-nbc-%s.fits"%scheme, "rw")
			dat = np.empty(nbc.res.size, dtype=[("coadd_objects_id", int), ("m", float), ("c1", float), ("c2", float) ])
			for name in ["coadd_objects_id", "m", "c1", "c2"]: dat[name]=nbc.res[name]
			out.write(dat)
			out[-1].write_key("EXTNAME", "nbc_col")
			out[-1].write_key("METHOD", scheme)
			out.close()


			np.savetxt("m-vs-z-dvec-fullcat-uncalibrated-tophatbins.txt", np.array(bias0).T, header="z m1 em1 m2 em2 m em")
			np.savetxt("shear_pipeline/plot_dump/datavecs/m-vs-z-dvec-fullcat-%s-nbc-tophatbins.txt"%scheme, np.array(bias).T, header="z m1 em1 m2 em2 m em")

			rewt=di.get_weights_to_match(y1v2.res["mean_rgpp_rp"], hoopoe.res["mean_rgpp_rp"],nbins=25)

			bmask = (hoopoe.res["is_bulge"]==1)
			mask = (hoopoe.res["bulge_flux"]<10) & (hoopoe.res["disc_flux"]<10)

			rewtb=di.get_weights_to_match(y1v2.res["bulge_flux"][y1v2.res["bulge_flux"]!=0], hoopoe.res["bulge_flux"][bmask & mask],nbins=25, xlim=(0.0,10.))
			rewtd=di.get_weights_to_match(y1v2.res["disc_flux"][y1v2.res["bulge_flux"]==0], hoopoe.res["disc_flux"][np.invert(bmask) & mask],nbins=25, xlim=(0.0,10.))
			wts_flux = np.zeros(hoopoe.res.size)
			wts_flux[bmask & mask] = rewtb
			wts_flux[np.invert(bmask) & mask] = rewtd
			wts_flux = wts_flux[mask]


			bias_size_wtd = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple", weights=rewt, split_half=0, bins=zbins, tophat=tophat, separate_components=False)
			bias_flux_wtd = nbc.redshift_diagnostic(bias="m", label="Calibrated", ls="none", nbins=3, fmt=["^",">"], apply_calibration=True, colour="purple", weights=wts_flux, split_half=0, bins=zbins, tophat=tophat, separate_components=False)

			out=np.vstack((bias_size_wtd[0], bias_size_wtd[-2], bias_size_wtd[-1], bias[-2], bias[-1]))
			np.savetxt("m-vs-z_sizewtd-%s.txt"%scheme, out.T)

			out=np.vstack((bias_flux_wtd[0], bias_flux_wtd[-2], bias_flux_wtd[-1], bias[-2], bias[-1]))
			np.savetxt("m-vs-z_fluxwtd-%s.txt"%scheme, out.T)








		if "a" in names:
			nbc.redshift_diagnostic(bias="alpha", label="Uncalibrated",ls="none", nbins=3, fmt=["o","D"], colour="steelblue", weights=weights, split_half=0, bins=zbins, tophat=tophat)
			nbc.redshift_diagnostic(bias="alpha", label="Calibrated", ls="none",fmt=["^",">"], nbins=3, apply_calibration=True, colour="purple", weights=weights, split_half=0, bins=zbins, tophat=tophat)
			plt.ylabel(r"PSF Leakage $\alpha$")
			plt.legend(loc="upper left")
			plt.savefig("%s/release/%s/alpha-vs-redshift-diagnostic-v1-fullcat-s%2.2f-sbins%d-rbins%d-tophat%d.png"%(config["output_dir"],sub_dir, smoothing, sbins, rbins, int(tophat)))
			plt.close()