def main(args):
im3shape_columns = ["e1", "e2", "mean_psf_e1_sky", "mean_psf_e2_sky", "mean_psf_fwhm", "snr", "mean_rgpp_rp", "radius", "coadd_objects_id", "mean_flux", "n_exposure", "stamp_size", "info_flag", "is_bulge", "tilename"]
truth_columns = ['DES_id', 'cosmos_ident', 'cosmos_photoz', 'sextractor_pixel_offset', 'true_g1', 'true_g2', 'intrinsic_e1', 'intrinsic_e2', 'ra', 'dec', 'hlr', 'mag', 'flux']
# Load the y1 data
if args.calculate or args.catalogue:
y1v2 = s.shapecat(res=config["i3s_dir"])
y1v2.load(truth=False, prune=True, cols=[im3shape_columns,truth_columns])
y1v2.res=y1v2.res[y1v2.res["info_flag"]==0]
# And the simulation results
if args.calculate:
if ".fits" not in config["hoopoe_dir"]:
hoopoe = s.shapecat(res="%s/bord-fits/main"%config["hoopoe_dir"] ,truth="%s/truth"%config["hoopoe_dir"])
hoopoe.load(truth=True, cols=[im3shape_columns,truth_columns])
else:
hoopoe = s.shapecat(res=config["hoopoe_dir"], truth=config["hoopoe_dir"] )
hoopoe.res = fi.FITS(config["hoopoe_dir"])["i3s"].read()
hoopoe.truth = fi.FITS(config["hoopoe_dir"])["truth"].read()
#hoopoe.res=fi.FITS("/share/des/disc7/samuroff/des/hoopoe-y1a1-v02-bord.fits")["i3s"].read()
#hoopoe.truth=fi.FITS("/share/des/disc7/samuroff/des/hoopoe-y1a1-v02-bord.fits")["truth"].read()
sel = np.isfinite(hoopoe.res["mean_psf_e1_sky"]) & np.isfinite(hoopoe.res["mean_psf_e2_sky"])
hoopoe.res = hoopoe.res[sel]
hoopoe.truth = hoopoe.truth[sel]
diagnostics(y1v2, hoopoe, vssnr=config["output"]["snr"], vsredshift=config["output"]["redshift"], table=config["output"]["tables"], alpha=config["output"]["alpha"], histograms=config["output"]["histograms"], rbf=True)
diagnostics(y1v2, hoopoe, histograms=False, alpha=False, table=False, rbf=False)
if args.catalogue:
calibrate(y1v2)
def main(args): load_sim, load_data = choose_inputs(args) outputs = choose_outputs(config) hoopoe, weights, y1v2 = setup(load_sim, load_data, config) if args.calculate: rbins= config["calibration"]["rbins"] sbins= config["calibration"]["sbins"] print "Using %d SNR bins , %d size bins"%(sbins,rbins) calculate(y1v2, hoopoe, split_method=config["calibration"]["split"], weights=weights, outputs=outputs, method=config["calibration"]["method"], config=config, sbins=sbins, rbins=rbins) if args.catalogue: rbins= config["calibration"]["rbins"] sbins= config["calibration"]["sbins"] print "Using %d SNR bins , %d size bins"%(sbins,rbins) calibrate(y1v2, config=config, sbins=sbins, rbins=rbins) if args.weights: hoopoe = s.shapecat() hoopoe.res = None get_weights(y1v2, hoopoe, config=config)
def run(f, args):
new_meds = f
#"%s/%s"%(args.output,os.path.basename(f))
truth_file = os.path.basename(f).replace("-meds-", "-truth-")
cat = s.shapecat(res=None, truth="%s/%s" % (args.truth, truth_file))
if args.mode == "model":
load_truth = True
cat.load(res=False, truth=True)
meds = s.meds_wrapper(new_meds, update=False)
if (args.mode == "model"):
meds.remove_model_bias(cat,
silent=True,
outdir=args.output,
noise=False,
neighbours=False)
elif (args.mode == "neighbour"):
meds.remove_neighbours(silent=False, outdir=args.output, noise=True)
elif (args.mode == "noise"):
meds.remove_noise(silent=True, outdir=args.output)
elif (args.mode == "neighbour_noise"):
meds.remove_neighbours(silent=False, outdir=args.output, noise=False)
def load(filename, header):
cat = fi.FITS(filename)
if header != "":
suffix = "_" + header
else:
suffix = ""
hoopoe = s.shapecat()
hoopoe.res = cat["i3s%s" % suffix].read()
hoopoe.truth = cat["truth%s" % suffix].read()
ngal = hoopoe.res.size
print "Found %3.4fM galaxies." % (ngal / 1e6)
return ngal, hoopoe
def main(args, nthread=1, rank=0):
load_sim, load_data = choose_inputs(args)
outputs = choose_outputs(config)
if args.mpi:
task_allocation = allocate_tasks(nthread, outputs)
else:
task_allocation = {0:["bulge_table","disc_table","bulge_htable","disc_htable"]}
hoopoe, weights, y1v2 = setup(load_sim, load_data, config)
if config["blinding"]["unblind"]:
if hoopoe is not None:
hoopoe.unblind(location=config["blinding"]["location"])
if y1v2 is not None:
y1v2.unblind(sim=False, location=config["blinding"]["location"])
else:
print "Data is blinded."
if args.calculate:
rbins= config["calibration"]["rbins"]
sbins= config["calibration"]["sbins"]
print "Using %d SNR bins , %d size bins"%(sbins,rbins)
calculate(y1v2, hoopoe,
split_method=config["calibration"]["split"],
weights=weights,
outputs=outputs,
method=config["calibration"]["method"],
config=config,
sbins=sbins,
rbins=rbins,
nthread=nthread,
rank=rank,
task_allocation=task_allocation)
if args.catalogue:
rbins= config["calibration"]["rbins"]
sbins= config["calibration"]["sbins"]
print "Using %d SNR bins , %d size bins"%(sbins,rbins)
calibrate(y1v2, config=config, sbins=sbins, rbins=rbins, nthread=nthread, rank=rank)
if args.weights:
hoopoe = s.shapecat()
hoopoe.res = None
get_weights(y1v2, hoopoe, config=config)
def main():
global args
parser = argparse.ArgumentParser(add_help=True)
parser.add_argument('-v', '--verbosity', type=int, action='store', default=2, choices=(0, 1, 2, 3), help='integer verbosity level: min=0, max=3 [default=2]')
parser.add_argument('-o', '--output', type=str, default="resimulated", help='Directory to write to.')
parser.add_argument('-i', '--input', type=str, default=".", help='Directory to source MEDS files')
parser.add_argument('-t', '--truth', type=str, default="truth", help='Directory to source truth tables')
parser.add_argument('-m', '--mode', type=str, default="model", help='type of bias to remove.')
parser.add_argument('-f', '--field', type=str, default="*", help='Simulation run to resimulate.')
parser.add_argument('--mpi', action='store_true', help='Split the tiles by MPI rank' )
args = parser.parse_args()
if args.mpi:
print "Setting up MPI."
import mpi4py.MPI
rank = mpi4py.MPI.COMM_WORLD.Get_rank()
size = mpi4py.MPI.COMM_WORLD.Get_size()
else:
print "Not using MPI (set the --mpi flag if you do want to parallelise the calculation)"
rank = 0
size = 1
os.system("mkdir -p %s"%args.output)
print "Will remove %s bias"%args.mode
print "Resimulated tiles will be written to %s"%args.output
filelist = glob.glob("%s/DES*%s*fits*.fz"%(args.input,args.field))
print "found %d tiles"%len(filelist)
for i, f in enumerate(filelist):
if os.path.exists("%s/%s"%(args.output,os.path.basename(f))):
print "file exists."
continue
if i%size!=rank:
continue
print i, f
new_meds = f
#"%s/%s"%(args.output,os.path.basename(f))
truth_file = os.path.basename(f).replace("-meds-","-truth-")
cat = s.shapecat(res=None, truth="%s/%s"%(args.truth, truth_file))
if args.mode=="model":
load_truth = True
cat.load(res=False, truth=True)
meds = s.meds_wrapper(new_meds, update=False)
if (args.mode=="model"):
try:
meds.remove_model_bias(cat, silent=True, outdir=args.output, noise=False, neighbours=False)
except:
continue
elif (args.mode=="neighbour"):
meds.remove_neighbours(silent=False, outdir=args.output, noise=True)
elif (args.mode=="noise"):
meds.remove_noise(silent=True, outdir=args.output)
elif (args.mode=="neighbour_noise"):
meds.remove_neighbours(silent=False, outdir=args.output, noise=False)
def setup(load_sim, load_data, config, verbose=True):
im3shape_columns = ["e1", "e2", "mean_hsm_psf_e1_sky", "mean_hsm_psf_e2_sky", "mean_hsm_psf_sigma", "mean_psf_fwhm","snr", "mean_rgpp_rp","mean_mask_fraction","radius", "coadd_objects_id", "is_bulge", "bulge_flux", "disc_flux", "mean_flux", "info_flag", "mag_auto_r", "tilename", "ra", "dec", "stamp_size", "n_exposure", "likelihood", "chi2_pixel"]
truth_columns = ['DES_id', 'cosmos_ident', 'cosmos_photoz', 'sextractor_pixel_offset', 'true_g1', 'true_g2', 'intrinsic_e1', 'intrinsic_e2', 'ra', 'dec', 'hlr', 'mag', 'flux']
y1v2 = None
hoopoe = None
weights = None
# Load the y1 data
if load_data:
if verbose:
print "Loading data %s"%config["input"]["i3s"]
y1v2 = s.shapecat(res=config["input"]["i3s"])
y1v2.load(truth=False, prune=True, cols=[im3shape_columns,truth_columns])
y1v2.res = y1v2.res[y1v2.res["info_flag"]==0] # This should always be true, but just in case...
sel = ((y1v2.res["snr"] > 12) & (y1v2.res["snr"] < 200) & (y1v2.res["mean_rgpp_rp"] > 1.13) & (y1v2.res["mean_rgpp_rp"] < 3.0))
y1v2.res=y1v2.res[sel]
if config["selection"]["cut_glowing_edges"]:
print "Removing glowing edges."
glowing_edges_ids = fi.FITS("/share/des/disc7/samuroff/des/y1a1-im3shape-bad-edge-ids-sorted.fits")[-1].read()
glowing_edges_mask = np.invert(np.in1d(y1v2.res["coadd_objects_id"], glowing_edges_ids["ids"]))
n0 = y1v2.res["coadd_objects_id"].size
y1v2.res = y1v2.res[glowing_edges_mask]
else:
if verbose:
print "Not loading data (either it's been loaded already or it's not needed)"
# And the simulation results
if load_sim:
if verbose:
print "Loading simulation %s"%config["input"]["hoopoe"]
hoopoe = s.shapecat(res=config["input"]["hoopoe"] ,truth=config["input"]["hoopoe"])
hoopoe.res = fi.FITS(hoopoe.res_path)["i3s"].read()
hoopoe.truth = fi.FITS(hoopoe.truth_path)["truth"].read()
hoopoe.res, hoopoe.truth = di.match_results(hoopoe.res, hoopoe.truth, name1="DES_id")
sel = np.isfinite(hoopoe.res["mean_hsm_psf_e1_sky"]) & np.isfinite(hoopoe.res["mean_hsm_psf_e2_sky"])
hoopoe.truth = hoopoe.truth[sel]
hoopoe.res = hoopoe.res[sel]
if (config["selection"]["mask"].lower()!="none"):
apply_selection = True
selection = fi.FITS(config["selection"]["mask"])["sel"].read().astype(bool)
weights = fi.FITS(config["selection"]["mask"])["wts"].read()
sel = ((hoopoe.res["snr"] > 12) & (hoopoe.res["snr"] < 200) & (hoopoe.res["mean_rgpp_rp"] > 1.13) & (hoopoe.res["mean_rgpp_rp"] < 3.0))
selection = selection & sel
if verbose:
print "Applying additional cuts and weights from %s"%config["selection"]["mask"]
hoopoe.res = hoopoe.res[selection]
hoopoe.truth = hoopoe.truth[selection]
weights = weights[selection]
if (not config["selection"]["reweight"]):
if verbose:
print "Ignoring weights."
weights = np.ones(hoopoe.res["coadd_objects_id"].size)
if not (config["calibration"]["ztype"]=="tophat"):
if verbose:
print "Using DES redshift bins"
exclude = (hoopoe.res["des_bin"]!=0 )
hoopoe.truth = hoopoe.truth[exclude]
weights = weights[exclude]
hoopoe.res = hoopoe.res[exclude]
else:
if verbose:
print "Using tophat redshift bins"
if (config["selection"]["resample"]):
print "Will apply resampling to match data"
edat = np.sqrt(y1v2.res["e1"]**2+y1v2.res["e2"]**2)
eh = np.sqrt(hoopoe.res["e1"]**2+hoopoe.res["e2"]**2)
subsample = di.get_selection_to_match(edat,eh,nbins=35)
hoopoe.res = hoopoe.res[subsample]
hoopoe.truth = hoopoe.truth[subsample]
weights = weights[subsample]
if (config["selection"]["weights_file"].lower()!="none"):
import astropy.table as tb
wts = fi.FITS(config["selection"]["weights_file"])["i3s_weights_col"].read()
col_to_replace = "mean_unmasked_flux_frac"
print "Renaming column '%s' as 'weight'"%col_to_replace
tab = tb.Table(hoopoe.res)
tab.rename_column(col_to_replace,"weight")
tab["weight"] = wts["weight"]
hoopoe.res = np.array(tab)
if config["selection"]["cut_glowing_edges"]:
print "Removing glowing edges."
glowing_edges_ids = fi.FITS("/share/des/disc7/samuroff/des/y1a1-im3shape-bad-edge-ids-sorted.fits")[-1].read()
glowing_edges_mask = np.invert(np.in1d(hoopoe.res["coadd_objects_id"], glowing_edges_ids["ids"]))
n0 = hoopoe.res["coadd_objects_id"].size
hoopoe.truth = hoopoe.truth[glowing_edges_mask]
hoopoe.res = hoopoe.res[glowing_edges_mask]
weights = weights[glowing_edges_mask]
print "cut removes %2.2fM/%2.2fM galaxies"%((n0-hoopoe.res["coadd_objects_id"].size)/1e6,n0/1e6)
print "Final selection : %d galaxies"%hoopoe.res["coadd_objects_id"].size
print "Final selection : %d unique COSMOS IDs"%np.unique(hoopoe.truth["cosmos_ident"]).size
else:
if verbose:
print "Not loading simulation."
return hoopoe, weights, y1v2
def setup(calculate, catalogue, config, y1v2=None):
im3shape_columns = [
"e1", "e2", "mean_hsm_psf_e1_sky", "mean_hsm_psf_e2_sky",
"mean_hsm_psf_sigma", "snr", "mean_rgpp_rp", "mean_mask_fraction",
"radius", "coadd_objects_id", "is_bulge", "bulge_flux", "disc_flux",
"info_flag", "mag_auto_r"
]
truth_columns = [
'DES_id', 'cosmos_ident', 'cosmos_photoz', 'sextractor_pixel_offset',
'true_g1', 'true_g2', 'intrinsic_e1', 'intrinsic_e2', 'ra', 'dec',
'hlr', 'mag', 'flux'
]
# Load the y1 data
if (y1v2 is None) and ((calculate and config["output"]["histograms"])
or catalogue or config["resample"]
or config["resample_perbin"]
or config["reweight_perbin"]):
y1v2 = s.shapecat(res=config["i3s_dir"])
y1v2.load(truth=False,
prune=True,
cols=[im3shape_columns, truth_columns])
#y1v2.res=y1v2.res[y1v2.res["info_flag"]==0]
sel = ((y1v2.res["snr"] > 12) & (y1v2.res["snr"] < 200) &
(y1v2.res["mean_rgpp_rp"] > 1.13) &
(y1v2.res["mean_rgpp_rp"] < 3.0))
y1v2.res = y1v2.res[sel]
else:
print "Not loading data (either it's been loaded already or it's not needed)"
# And the simulation results
if calculate:
hoopoe = s.shapecat(res=config["hoopoe_dir"],
truth=config["hoopoe_dir"])
hoopoe.res = fi.FITS(hoopoe.res_path)["i3s"].read()
hoopoe.truth = fi.FITS(hoopoe.truth_path)["truth"].read()
sel = np.isfinite(hoopoe.res["mean_hsm_psf_e1_sky"]) & np.isfinite(
hoopoe.res["mean_hsm_psf_e2_sky"])
hoopoe.truth = hoopoe.truth[sel]
hoopoe.res = hoopoe.res[sel]
apply_selection = False
if ("apply_selection" in config.keys()):
if config["apply_selection"]:
apply_selection = True
selection = fi.FITS(
config["selection"])["sel"].read().astype(bool)
weights = fi.FITS(config["selection"])["wts"].read()
print "Applying additional cuts and weights from %s" % config[
"selection"]
hoopoe.res = hoopoe.res[selection]
hoopoe.truth = hoopoe.truth[selection]
weights = weights[selection]
if (not apply_selection) or (not config["reweight"]):
weights = np.ones(hoopoe.res["coadd_objects_id"].size)
if not config["tophat_binning"]:
print "Using DES redshift bins"
#bin_allocation = fi.FITS("/share/des/disc6/samuroff/y1/hoopoe/hoopoe-v2-zbin_allocation.fits")[1].read()
#bin_num, hoopoe.res = di.match_results(bin_allocation, hoopoe.res)
#hoopoe.res = arr.add_col(hoopoe.res,"des_bin", bin_num["bin"])
exclude = (hoopoe.res["des_bin"] != 0)
hoopoe.truth = hoopoe.truth[exclude]
weights = weights[exclude]
hoopoe.res = hoopoe.res[exclude]
else:
print "Using tophat redshift bins"
if (config["resample"]):
print "Will apply resampling to match data"
edat = np.sqrt(y1v2.res["e1"]**2 + y1v2.res["e2"]**2)
eh = np.sqrt(hoopoe.res["e1"]**2 + hoopoe.res["e2"]**2)
subsample = di.get_selection_to_match(edat, eh, nbins=35)
hoopoe.res = hoopoe.res[subsample]
hoopoe.truth = hoopoe.truth[subsample]
weights = weights[subsample]
print "Final selection : %d galaxies" % hoopoe.res[
"coadd_objects_id"].size
print "Final selection : %d unique COSMOS IDs" % np.unique(
hoopoe.truth["cosmos_ident"]).size
return hoopoe, weights, y1v2
def setup(load_sim, load_data, config, verbose=True): im3shape_columns = ["e1", "e2", "mean_hsm_psf_e1_sky", "mean_hsm_psf_e2_sky", "mean_hsm_psf_sigma", "snr", "mean_rgpp_rp","mean_mask_fraction", "radius", "coadd_objects_id", "is_bulge", "bulge_flux", "disc_flux", "info_flag", "mag_auto_r"] truth_columns = ['DES_id', 'cosmos_ident', 'cosmos_photoz', 'sextractor_pixel_offset', 'true_g1', 'true_g2', 'intrinsic_e1', 'intrinsic_e2', 'ra', 'dec', 'hlr', 'mag', 'flux'] # Load the y1 data if load_data: if verbose: print "Loading data %s"%config["inputs"]["i3s"] y1v2 = s.shapecat(res=config["inputs"]["i3s"]) y1v2.load(truth=False, prune=True, cols=[im3shape_columns,truth_columns]) y1v2.res = y1v2.res[y1v2.res["info_flag"]==0] # This should always be true, but just in case... sel = ((y1v2.res["snr"] > 12) & (y1v2.res["snr"] < 200) & (y1v2.res["mean_rgpp_rp"] > 1.13) & (y1v2.res["mean_rgpp_rp"] < 3.0)) y1v2.res=y1v2.res[sel] else: if verbose: print "Not loading data (either it's been loaded already or it's not needed)" # And the simulation results if load_sim: if verbose: print "Loading simulation %s"%config["inputs"]["hoopoe"] hoopoe = s.shapecat(res=config["inputs"]["hoopoe"] ,truth=config["inputs"]["hoopoe"]) hoopoe.res = fi.FITS(hoopoe.res_path)["i3s"].read() hoopoe.truth = fi.FITS(hoopoe.truth_path)["truth"].read() sel = np.isfinite(hoopoe.res["mean_hsm_psf_e1_sky"]) & np.isfinite(hoopoe.res["mean_hsm_psf_e2_sky"]) hoopoe.truth = hoopoe.truth[sel] hoopoe.res = hoopoe.res[sel] if (config["selection"]["mask"].lower()!="none"): apply_selection = True selection = fi.FITS(config["selection"]["mask"])["sel"].read().astype(bool) weights = fi.FITS(config["selection"]["mask"])["wts"].read() if verbose: print "Applying additional cuts and weights from %s"%config["selection"]["mask"] hoopoe.res = hoopoe.res[selection] hoopoe.truth = hoopoe.truth[selection] weights = weights[selection] if (not config["selection"]["reweight"]): if verbose: print "Ignoring weights." weights = np.ones(hoopoe.res["coadd_objects_id"].size) if not config["calibration"]["zbins"]: if verbose: print "Using DES redshift bins" exclude = (hoopoe.res["des_bin"]!=0 ) hoopoe.truth = hoopoe.truth[exclude] weights = weights[exclude] hoopoe.res = hoopoe.res[exclude] else: if verbose: print "Using tophat redshift bins" if (config["selection"]["weights"].lower()!="none"): if verbose: print "Using im3shape weights from %s"%config["selection"]["weights"].lower() im3shape_weights = fi.FITS(config["selection"]["weights"])[-1].read() hoopoe.res = arr.add_col(hoopoe.res, "weight", im3shape_weights) if (config["selection"]["resample"]): print "Will apply resampling to match data" edat = np.sqrt(y1v2.res["e1"]**2+y1v2.res["e2"]**2) eh = np.sqrt(hoopoe.res["e1"]**2+hoopoe.res["e2"]**2) subsample = di.get_selection_to_match(edat,eh,nbins=35) hoopoe.res = hoopoe.res[subsample] hoopoe.truth = hoopoe.truth[subsample] weights = weights[subsample] print "Final selection : %d galaxies"%hoopoe.res["coadd_objects_id"].size print "Final selection : %d unique COSMOS IDs"%np.unique(hoopoe.truth["cosmos_ident"]).size else: if verbose: print "Not loading simulation." return hoopoe, weights, y1v2
import tools.shapes as s
import glob, argparse, os
parser = argparse.ArgumentParser(add_help=True)
parser.add_argument('--truth',
type=str,
default=".",
help="Directory in which to look for the truth tables")
parser.add_argument('--output',
type=str,
default="./hoopoe_y1a1_ra_dec_flags_mag_v2.hdf5",
help="Name to save the new object flags under")
args = parser.parse_args()
sim = s.shapecat(truth=args.truth)
truth_columns_needed = [
"DES_id", "mag", "flags", "cosmos_photoz", "ra", "dec", "star_flag"
]
sim.load(res=False,
truth=True,
cols=[None, truth_columns_needed],
apply_infocuts=False)
gold_cat_columns = np.dtype([('FLAGS_G', '<i4'), ('FLAGS_R', '<i4'),
('FLAGS_I', '<i4'), ('FLAGS_Z', '<i4'),
('FLAGS_Y', '<i4'), ('RA', '<f8'), ('DEC', '<f8'),
('MAG_AUTO_G', '<f4'), ('MAG_AUTO_R', '<f4'),
('MAG_AUTO_I', '<f4'), ('MAG_AUTO_Z', '<f4'),
('MAG_AUTO_Y', '<f4'), ('DESDM_ZP', '<f8'),
('MODEST', '<f8'), ('HPIX', '<f8'),
def main(args):
im3shape_columns = ["e1", "e2", "mean_hsm_psf_e1_sky", "mean_hsm_psf_e2_sky", "mean_hsm_psf_sigma", "snr", "mean_rgpp_rp", "radius", "coadd_objects_id", "mean_flux", "n_exposure", "stamp_size", "is_bulge", "tilename", "info_flag", "disc_flux", "bulge_flux"]
truth_columns = ['DES_id', 'cosmos_ident', 'cosmos_photoz', 'sextractor_pixel_offset', 'true_g1', 'true_g2', 'intrinsic_e1', 'intrinsic_e2', 'ra', 'dec', 'hlr', 'mag', 'flux']
# Load the y1 data
#if (args.calculate and config["output"]["histograms"]) or args.catalogue or args.weights or config["resample"] or config["resample_perbin"] or config["reweight_perbin"]:
if True:
y1v2 = s.shapecat(res=config["i3s_dir"])
y1v2.load(truth=False, prune=True, cols=[im3shape_columns,truth_columns])
y1v2.res=y1v2.res[y1v2.res["info_flag"]==0]
sel=((y1v2.res["snr"] > 12) & (y1v2.res["snr"] < 200) & (y1v2.res["mean_rgpp_rp"] > 1.13) & (y1v2.res["mean_rgpp_rp"] < 3.0))
y1v2.res=y1v2.res[sel]
else:
y1v2 = None
# And the simulation results
if args.calculate:
hoopoe = s.shapecat(res=config["hoopoe_dir"] ,truth=config["hoopoe_dir"])
hoopoe.res=fi.FITS(hoopoe.res_path)["i3s"].read()
hoopoe.truth=fi.FITS(hoopoe.truth_path)["truth"].read()
sel = np.isfinite(hoopoe.res["mean_hsm_psf_e1_sky"]) & np.isfinite(hoopoe.res["mean_hsm_psf_e2_sky"])
hoopoe.res = hoopoe.res[sel]
hoopoe.truth = hoopoe.truth[sel]
apply_selection = False
if ("apply_selection" in config.keys()):
if config["apply_selection"]:
apply_selection = True
selection = fi.FITS(config["selection"])["sel"].read().astype(bool)
weights = fi.FITS(config["selection"])["wts"].read()
print "Applying additional cuts and weights from %s"%config["selection"]
hoopoe.res = hoopoe.res[selection]
hoopoe.truth = hoopoe.truth[selection]
weights = weights[selection]
if (not apply_selection) or (not config["reweight"]):
weights = np.ones(hoopoe.res["coadd_objects_id"].size)
if not config["tophat_binning"]:
print "Using DES redshift bins"
#bin_allocation = fi.FITS("/share/des/disc6/samuroff/y1/hoopoe/hoopoe-v2-zbin_allocation.fits")[1].read()
#bin_num, hoopoe.res = di.match_results(bin_allocation, hoopoe.res)
#hoopoe.res = arr.add_col(hoopoe.res,"des_bin", bin_num["bin"])
exclude = (hoopoe.res["des_bin"]!=0 )
hoopoe.truth = hoopoe.truth[exclude]
weights = weights[exclude]
hoopoe.res = hoopoe.res[exclude]
if (config["resample"]):
print "Will apply resampling to match data"
edat = np.sqrt(y1v2.res["e1"]**2+y1v2.res["e2"]**2)
eh = np.sqrt(hoopoe.res["e1"]**2+hoopoe.res["e2"]**2)
subsample = di.get_selection_to_match(edat,eh,nbins=35)
hoopoe.res = hoopoe.res[subsample]
hoopoe.truth = hoopoe.truth[subsample]
weights = weights[subsample]
else:
print "Using tophat redshift bins"
if (config["selection"]["weights_file"] is not None):
import astropy.table as tb
wts = fi.FITS(config["selection"]["weights_file"])[-1].read()
col_to_replace = "mean_unmasked_flux_frac"
print "Renaming column '%s' as 'weight'"%col_to_replace
tab = tb.Table(hoopoe.res)
tab.rename_column(col_to_replace,"weight")
tab["weight"] = wts["weight"]
hoopoe.res = np.array(tab)
print "Final selection : %d galaxies"%hoopoe.res["coadd_objects_id"].size
print "Final selection : %d unique COSMOS IDs"%np.unique(hoopoe.truth["cosmos_ident"]).size
if args.calculate:
rbins= config["rbins"]
sbins= config["sbins"]
print "Using %d SNR bins , %d size bins"%(sbins,rbins)
if config["random_halves"]:
diagnostics(y1v2, hoopoe, split_method="random", weights=weights, histograms=False, alpha=False, table=config["output"]["tables"], half_tables=True, vsredshift=config["output"]["redshift"], rbf=False, simple_grid=True, config=config, sbins=sbins, rbins=rbins)
diagnostics(y1v2, hoopoe, split_method="random", weights=weights, histograms=False, alpha=False, table=False, half_tables=config["output"]["tables"], vsredshift=config["output"]["redshift"], rbf=True, config=config, sbins=sbins, rbins=rbins)
diagnostics(y1v2, hoopoe, split_method="random", weights=weights, histograms=False, alpha=False, table=False, half_tables=False, vsredshift=config["output"]["redshift"], rbf=False, simple_grid=False, config=config, sbins=sbins, rbins=rbins)
diagnostics(y1v2, hoopoe, weights=weights, histograms=config["output"]["histograms"], alpha=False, table=config["output"]["tables"], vsredshift=True, rbf=False, simple_grid=True, config=config, sbins=sbins, rbins=rbins, half_tables=config["random_halves"])
diagnostics(y1v2, hoopoe, weights=weights, vssnr=config["output"]["snr"], vsredshift=config["output"]["redshift"], table=config["output"]["tables"], alpha=config["output"]["alpha"], histograms=False, rbf=True, config=config, sbins=sbins, rbins=rbins, half_tables=config["random_halves"])
diagnostics(y1v2, hoopoe, weights=weights, histograms=False, alpha=False, table=False, vsredshift=True, rbf=False, simple_grid=False, config=config, sbins=sbins, rbins=rbins, half_tables=config["random_halves"])
if config["cosmos_halves"]:
diagnostics(y1v2, hoopoe, split_method="cosmos", weights=weights, histograms=False, alpha=False, table=False, half_tables=config["output"]["tables"], vsredshift=config["output"]["redshift"], rbf=True, config=config, sbins=sbins, rbins=rbins)
diagnostics(y1v2, hoopoe, split_method="cosmos", weights=weights, histograms=False, alpha=False, table=False, half_tables=False, vsredshift=config["output"]["redshift"], rbf=False, simple_grid=False, config=config, sbins=sbins, rbins=rbins)
diagnostics(y1v2, hoopoe, split_method="cosmos", weights=weights, histograms=False, alpha=False, table=False, half_tables=False, vsredshift=config["output"]["redshift"], rbf=False, simple_grid=True, config=config, sbins=sbins, rbins=rbins)
if args.catalogue:
rbins= config["rbins"]
sbins= config["sbins"]
print "Using %d SNR bins , %d size bins"%(sbins,rbins)
calibrate(y1v2, config=config, sbins=sbins, rbins=rbins)
if args.weights:
hoopoe = s.shapecat()
hoopoe.res = None
get_weights(y1v2, hoopoe, config=config)
import numpy as np
from astropy.table import Table, Column
import tools.shapes as s
import glob, argparse, os
parser = argparse.ArgumentParser(add_help=True)
parser.add_argument('--truth', type=str, default=".", help="Directory in which to look for the truth tables")
parser.add_argument('--output', type=str, default="./hoopoe_y1a1_ra_dec_flags_mag_v2.hdf5", help="Name to save the new object flags under")
args = parser.parse_args()
sim = s.shapecat(truth=args.truth)
truth_columns_needed = ["DES_id", "mag", "flags", "cosmos_photoz", "ra", "dec", "star_flag"]
sim.load(res=False, truth=True, cols=[None, truth_columns_needed], apply_infocuts=False)
gold_cat_columns = np.dtype([('FLAGS_G', '<i4'), ('FLAGS_R', '<i4'), ('FLAGS_I', '<i4'), ('FLAGS_Z', '<i4'), ('FLAGS_Y', '<i4'), ('RA', '<f8'), ('DEC', '<f8'), ('MAG_AUTO_G', '<f4'), ('MAG_AUTO_R', '<f4'), ('MAG_AUTO_I', '<f4'), ('MAG_AUTO_Z', '<f4'), ('MAG_AUTO_Y', '<f4'), ('DESDM_ZP', '<f8'), ('MODEST', '<f8'), ('HPIX', '<f8'), ('COADD_OBJECTS_ID', '<i8')])
mock_cat = np.zeros(sim.truth.size, dtype=gold_cat_columns)
mock_cat["COADD_OBJECTS_ID"] = sim.truth["DES_id"]
mock_cat["MAG_AUTO_R"] = sim.truth["mag"]
mock_cat["FLAGS_R"] = sim.truth["flags"]
mock_cat["DESDM_ZP"] = sim.truth["cosmos_photoz"]
mock_cat["RA"] = sim.truth["ra"]
mock_cat["DEC"] = sim.truth["dec"]
mock_cat["MODEST"] = sim.truth["star_flag"]+1
mock_cat = Table(mock_cat)
print "Writing mock gold catalogue flags table to %s"%args.output
try:
print "Setting up MPI"
import mpi4py.MPI
rank = mpi4py.MPI.COMM_WORLD.Get_rank()
size = mpi4py.MPI.COMM_WORLD.Get_size()
except:
rank=0
size=1
analysis = toy_model.toy_model2(komplexitet=args.complexity, random_seed=rank*size)
analysis.setup()
data = s.shapecat()
data.res = fi.FITS("/home/samuroff/16tiles_results_nf.fits")[1].read()
data.truth = fi.FITS("/home/samuroff/16tiles_results_nf.fits")[2].read()
data.truth_path="/share/des/disc3/samuroff/y1/sims/v2.2/y1a1_16tiles/truth"
ncat = s.shapecat()
ncat.res = fi.FITS("/home/samuroff/16tiles_neighbours_nf.fits")[1].read()
ncat.truth = fi.FITS("/home/samuroff/16tiles_neighbours_nf.fits")[2].read()
fcat = data.match_to_faint()
# If there is a subdetection object closer than the nearest detectable object
# we'll treat that as the nearest neighbour
Rn = (data.truth["ra"]-ncat.truth["ra"])*(data.truth["ra"]-ncat.truth["ra"]) + (data.truth["dec"]-ncat.truth["dec"])*(data.truth["dec"]-ncat.truth["dec"])
Rn=np.sqrt(Rn)*60*60/0.27
Rf = (data.truth["ra"]-fcat.truth["ra"])*(data.truth["ra"]-fcat.truth["ra"]) + (data.truth["dec"]-fcat.truth["dec"])*(data.truth["dec"]-fcat.truth["dec"])
Rf=np.sqrt(Rf)*60*60/0.27
print "Will calculate m samples"
print "-------------------------------------------"
try:
print "Setting up MPI"
import mpi4py.MPI
rank = mpi4py.MPI.COMM_WORLD.Get_rank()
size = mpi4py.MPI.COMM_WORLD.Get_size()
except:
rank = 0
size = 1
config = yaml.load(open("/home/samuroff/calibration_config_no_box_cut.yaml"))
hoopoe = s.shapecat(res=config["hoopoe_dir"], truth=config["hoopoe_dir"])
hoopoe.res = fi.FITS(config["hoopoe_dir"])["i3s"][:args.nrealisations * 4]
hoopoe.truth = fi.FITS(config["hoopoe_dir"])["truth"][:args.nrealisations * 4]
sel = ((hoopoe.res["snr"] > 12) & (hoopoe.res["snr"] < 200) &
(hoopoe.res["mean_rgpp_rp"] > 1.13) &
(hoopoe.res["mean_rgpp_rp"] < 3.0))
hoopoe.res = hoopoe.res[sel]
hoopoe.truth = hoopoe.truth[sel]
ncat = fi.FITS(
"/home/samuroff/neighbours.fits")[1].read()[:args.nrealisations * 4]
f = np.sqrt(-2 * np.log(0.5)) # HLR - sigma conversion
fac = 2.35 # sigma - FWHM conversion
#ets=np.sqrt( (hoopoe.truth["intrinsic_e1"]+hoopoe.truth["true_g1"])**2 + (hoopoe.truth["intrinsic_e2"]+hoopoe.truth["true_g2"])**2 )
