def make_gold_flags(config):
"""
code to make gold flags for DES WL analysis
People to bug if things are weird
=================================
Matthew Becker
Eli Rykoff
Erin Sheldon
Input Data
==========
We use the following sources of data.
1) SVA1 gold v1.0.4 with the additional star masking,
bad regions mask, crazy colors, and large g- vs i-band
offsets. See this page
https://cdcvs.fnal.gov/redmine/projects/des-sci-verification/wiki/SVA1_Gold_Catalog_v10#Gold-104
2) ngmix11 - used to cut low surface brightness junk,
additonal stars and to set the ngmix flags of course
See these pages
https://cdcvs.fnal.gov/redmine/projects/deswlwg/wiki/Ngmix011
https://cdcvs.fnal.gov/redmine/projects/des-sci-verification/wiki/A_Modest_Proposal_for_Preliminary_StarGalaxy_Separation
Flag File Description
======================
The flag file consists of three columns which are
sva1_gold_flags
---------------
These are flags related to SVA1 image processing and galaxy selection. We
have attempted to preserve as much information as possible. The field is
created by concatenating the SExtractor flags (i-band), SVA1 gold flags,
and additional galaxy selection criteria from ngmix11
bit 2^bit name notes
----------------------------------------------------------------------
0 1 SX_KINDOF_BLEND bit 0 from the i-band sexctractor flags
1 2 SX_DEF_BLEND bit 1 from the i-band sexctractor flags
2 4 MODEST_STAR SVA1 Gold MODEST_CLASS == 2
3 8 MODEST_JUNK SVA1 Gold MODEST_CLASS == 0
4 16 CRAZY_COLORS SVA1 Gold crazy mag auto colors
5 32 BADMASK04 in region with lots of JUNK_DPOS objects (removes ~4% of area)
6 64 NEAR_2MASS near a 2MASS star
7 128 JUNK_DPOS large offset in g and i band windowed positions
8 256 NOT_IN_NGMIX011 object did not get measured by ngmix011
9 512 NGMIX011_STAR object is a star according to ngmix011 modest classifier
10 1024 LOW_SB_NGMIX011 object has low surface brightness in ngmix011
11 2048 BAD_MEAS_NGMIX011 object does not satisfy good measurement cuts in ngmix
Some cuts from ngmix011 are applied to the full gold sample. These are detailed below
and are in flag bits 8 through 12.
We use a modest-style star galaxy separation such that objects
with
exp_t + exp_t_err > 0.02
are galaxies. We also cut objects with
exp_mag_i + 3.5*log10(exp_flux_i/exp_t) < 28.0
where
exp_flux_i = 10.0^((exp_mag_i-30.0)/(-2.5))
Note that for the ngmix cuts we require that for ngmix
flags == 0
exp_flags == 0
0.4 < exp_arate < 0.6
so that object has a good measurement at all.
sva1_spte_flags
---------------
We flag all galaxies with ra,dec not in
50 < ra < 100
dec < -35.0
as outside of the SPTE region.
sva1_gold_mag_flags
-------------------
We flag all objects that do not have griz mags.
bit 2^bit name notes
----------------------------------------------------------------------
0 1 MISSING_GRIZ_MAGS object does not have all griz mags
"""
# first read in the data
gold = fitsio.read(config['gold'],lower=True,columns=['coadd_objects_id','modest_class','flags_i','ra','dec'])
auto = fitsio.read(config['auto'],lower=True)
badflag = fitsio.read(config['badflag'],lower=True)
ng = fitsio.read(config['ngmix'])
# make output data
dflag = np.zeros(len(gold),dtype=[('coadd_objects_id','i8'),('ra','f8'),('dec','f8'),
('sva1_gold_flags','i8'),('sva1_spte_flags','i8'),
('sva1_gold_mag_flags','i8'),
('mag_auto_g','f8'),('mag_auto_r','f8'),
('mag_auto_i','f8'),('mag_auto_z','f8')])
for tag in ['coadd_objects_id','ra','dec']:
dflag[tag] = gold[tag]
for tag in ['mag_auto_g','mag_auto_r','mag_auto_i','mag_auto_z']:
dflag[tag] = auto[tag]
# set sva1_gold_flags
tag = 'sva1_gold_flags'
dflag[tag] = gold['flags_i']
q, = np.where(gold['modest_class'] == 2)
if len(q) > 0:
dflag[tag][q] |= 4
q, = np.where(gold['modest_class'] == 0)
if len(q) > 0:
dflag[tag][q] |= 8
# crazy colors
# (g-r) < -1, (g-r) > 4, (i-z) < -1, (i-z) > 4
q, = np.where((auto['mag_auto_g'] - auto['mag_auto_r'] < -1.0) |
(auto['mag_auto_g'] - auto['mag_auto_r'] > 4.0) |
(auto['mag_auto_i'] - auto['mag_auto_z'] < -1.0) |
(auto['mag_auto_i'] - auto['mag_auto_z'] > 4.0))
if len(q) > 0:
dflag[tag][q] |= 16
# badflags masking
q, = np.where((badflag['badflag'] & 2) != 0)
if len(q) > 0:
dflag[tag][q] |= 32
q, = np.where((badflag['badflag'] & 4) != 0)
if len(q) > 0:
dflag[tag][q] |= 64
q, = np.where((badflag['badflag'] & 8) != 0)
if len(q) > 0:
dflag[tag][q] |= 128
del badflag
# ngmix cuts
# first have to match
gids = gold['coadd_objects_id']
ngids = ng['coadd_objects_id']
ginds,nginds = numpy_util.match(gids,ngids)
# flag stuff not in ngmix011
sginds = set(ginds)
totginds = set(range(len(gold)))
diffginds = totginds - sginds
q = np.fromiter(diffginds,dtype=int,count=len(diffginds))
if len(q) > 0:
dflag[tag][q] |= 256
# bad measurements
q, = np.where(~((ng['flags'][nginds] == 0) &
(ng['exp_flags'][nginds] == 0) &
(ng['exp_arate'][nginds] > 0.4) &
(ng['exp_arate'][nginds] < 0.6)))
if len(q) > 0:
dflag[tag][ginds[q]] |= 2048
# low SB
sb = ng['exp_mag_i'] + 3.5*np.log10(np.abs(ng['exp_flux_i']/ng['exp_t']))
q, = np.where(~((ng['flags'][nginds] == 0) &
(ng['exp_flags'][nginds] == 0) &
(ng['exp_arate'][nginds] > 0.4) &
(ng['exp_arate'][nginds] < 0.6) &
(sb[nginds] > 28.0)))
if len(q) > 0:
dflag[tag][ginds[q]] |= 1024
# ngmix star
nsig = 1.0
sgc = ng['exp_t'] + nsig*ng['exp_t_err']
q, = np.where(~((ng['flags'][nginds] == 0) &
(ng['exp_flags'][nginds] == 0) &
(ng['exp_arate'][nginds] > 0.4) &
(ng['exp_arate'][nginds] < 0.6) &
(sgc[nginds] > 0.02)))
if len(q) > 0:
dflag[tag][ginds[q]] |= 512
# non-99 mag_auto in all bands
tag = 'sva1_gold_mag_flags'
q, = np.where(~((np.abs(auto['mag_auto_g']) < 99.0) &
(np.abs(auto['mag_auto_r']) < 99.0) &
(np.abs(auto['mag_auto_i']) < 99.0) &
(np.abs(auto['mag_auto_z']) < 99.0)))
if len(q) > 0:
dflag[tag][q] |= 1
del auto
#set sva1_spte_flags
q, = np.where(~((gold['ra'] > 50.0) & (gold['ra'] < 100.0) & (gold['dec'] < -35.0)))
if len(q) > 0:
dflag['sva1_spte_flags'][q] != 1
#write to disk
fitsio.write(config['gold_flags'],dflag,clobber=True)
def main(config):
###############################
# info file
# do the flags
flags = fitsio.read(config['gold_flags'])
imflags = fitsio.read(config['im3shape_flags'])
ngflags = fitsio.read(config['ngmix_flags'])
if False:
q, = np.where((flags['sva1_gold_flags'] <= 3)
& (flags['sva1_spte_flags'] == 0)
& (flags['sva1_gold_mag_flags'] == 0)
& ((imflags['im3shape_flags'] == 0)
| (ngflags['ngmix_flags'] == 0)))
else:
q, = np.where((flags['sva1_gold_flags'] <= 3)
& (flags['sva1_spte_flags'] == 0)
& (flags['sva1_gold_mag_flags'] == 0))
flags = flags[q]
imflags = imflags[q]
ngflags = ngflags[q]
# print number of sources
for code, cflags, tag in zip(['ngmix', 'im3shape'], [ngflags, imflags],
['ngmix_flags', 'im3shape_flags']):
q, = np.where((flags['sva1_gold_flags'] == 0) & (cflags[tag] == 0))
print "%s:" % code, len(q)
# read in the photo-z bins - match
dtot = []
dz = []
if 'photoz_binning_im3shape' in config:
dim = np.load(config['photoz_binning_im3shape']).astype(int)
for i in xrange(len(dim)):
dtot.append(tuple(dim[i]))
dim = np.load(config['photoz_binning_im3shape']).astype(float)
for i in xrange(len(dim)):
dz.append(tuple(dim[i])[-1])
del dim
if 'photoz_binning_ngmix' in config:
dng = np.load(config['photoz_binning_ngmix']).astype(int)
for i in xrange(len(dng)):
dtot.append(tuple(dng[i]))
dng = np.load(config['photoz_binning_ngmix']).astype(float)
for i in xrange(len(dng)):
dz.append(tuple(dng[i])[-1])
del dng
if 'photoz_binning_extra' in config:
de = np.load(config['photoz_binning_extra']).astype(int)
for i in xrange(len(de)):
dtot.append(tuple(de[i]))
de = np.load(config['photoz_binning_extra']).astype(float)
for i in xrange(len(de)):
dz.append(tuple(de[i])[-1])
del de
if len(dtot) > 0:
dtot = np.array(dtot,
dtype=[('coadd_objects_id', 'i8'),
('photoz_bin', 'i4'), ('mean_photoz', 'f8')])
dz = np.array(dz, dtype='f8')
dtot['mean_photoz'][:] = dz[:]
u, indices = np.unique(dtot['coadd_objects_id'], return_index=True)
dtot = dtot[indices]
del u, indices
# make sure all in file
fids = set(flags['coadd_objects_id'])
pzids = set(dtot['coadd_objects_id'])
if not fids <= pzids:
print "not all gals got photoz binning indexes!"
if True:
fids = flags['coadd_objects_id']
pzids = dtot['coadd_objects_id']
finds, pzinds = numpy_util.match(fids, pzids)
finds = set(finds)
tinds = set(np.arange(len(flags)))
minds = tinds - finds
minds = np.array(list(minds))
#print set(flags['coadd_objects_id'][minds])
print '# of missing gals is', len(minds)
#assert False
fids = flags['coadd_objects_id']
pzids = dtot['coadd_objects_id']
finds, pzinds = numpy_util.match(fids, pzids)
# make final catalog
ftags = [
'mag_auto_g', 'mag_auto_r', 'mag_auto_i', 'mag_auto_z', 'ra', 'dec',
'mean_photoz'
]
itags = [
'coadd_objects_id', 'sva1_flags', 'im3shape_flags', 'ngmix_flags',
'sva1_gold_flags', 'sva1_spte_flags', 'sva1_gold_mag_flags',
'photoz_bin'
]
dlist = []
for ftag in ftags:
dlist.append((ftag, 'f8'))
for itag in itags:
dlist.append((itag, 'i8'))
d = np.zeros(len(flags), dtype=dlist)
d['photoz_bin'][:] = -99
d['mean_photoz'][:] = -99.0
for tag in flags.dtype.names:
d[tag] = flags[tag]
for tag in imflags.dtype.names:
d[tag] = imflags[tag]
for tag in ngflags.dtype.names:
d[tag] = ngflags[tag]
if len(dtot) > 0:
d['photoz_bin'][finds] = dtot['photoz_bin'][pzinds]
d['mean_photoz'][finds] = dtot['mean_photoz'][pzinds]
q, = np.where(flags['sva1_gold_flags'] & 1)
d['sva1_flags'][q] |= 1
q, = np.where(flags['sva1_gold_flags'] & 2)
d['sva1_flags'][q] |= 2
q, = np.where((flags['sva1_gold_flags'] >> 2) != 0)
d['sva1_flags'][q] |= 4
q, = np.where(flags['sva1_spte_flags'] != 0)
d['sva1_flags'][q] |= 8
q, = np.where(flags['sva1_gold_mag_flags'] != 0)
d['sva1_flags'][q] |= 16
fitsio.write(config['flatcats_info'], d, clobber=True)
del flags
del imflags
del ngflags
del dtot
###############################
# ngmix file
# first have to match to psf and s2n
ng = fitsio.read(config['ngmix'])
psf = fitsio.read(config['ngmix_psfs'])
pids = psf['id']
ngids = ng['coadd_objects_id']
pinds, nginds = numpy_util.match(pids, ngids)
assert np.all(psf['flags'][pinds] == 0)
ng['psfrec_t'][nginds] = psf['psfrec_T'][pinds]
ng['psfrec_e_1'][nginds] = psf['psfrec_e'][pinds, 0]
ng['psfrec_e_2'][nginds] = psf['psfrec_e'][pinds, 1]
ng = ng[nginds]
del psf, pids, ngids, pinds, nginds
# now match to new s2n measures
ngs2n = fitsio.read(config['ngmix_s2n'])
s2nids = ngs2n['id']
ngids = ng['coadd_objects_id']
s2ninds, nginds = numpy_util.match(s2nids, ngids)
ng = ng[nginds]
ngs2n = ngs2n[s2ninds]
del s2ninds, nginds, s2nids, ngids
# make sure everything is in the ngmix file
q, = np.where(d['ngmix_flags'] == 0)
flagged_ids = set(d['coadd_objects_id'][q])
ngids = set(ng['coadd_objects_id'])
assert flagged_ids <= ngids
# match to total file
fids = d['coadd_objects_id']
ngids = ng['coadd_objects_id']
finds, nginds = numpy_util.match(fids, ngids)
assert len(finds) == len(d)
# extra tags
dlist = ng.dtype.descr
dlist.append(('exp_log10sb_i', 'f8'))
dlist.append(('exp_w', 'f8'))
dlist.append(('exp_e_sens_avg', 'f8'))
dlist.append(('exp_T_r', 'f8'))
dlist.append(('exp_s2n_r', 'f8'))
dlist.append(('exp_T_s2n_r', 'f8'))
dnew = []
for dt in dlist:
if '_t' in dt[0]:
name = dt[0]
name = name.replace('_t', '_T')
dnew.append((name, dt[1]))
elif dt[0] not in [
'exp_e_1', 'exp_e_2', 'exp_e_cov_1_1', 'exp_e_cov_1_2',
'exp_e_cov_2_2'
]:
dnew.append(dt)
dlist = dnew
for tag in [
'exp_e_1', 'exp_e_2', 'exp_e_cov_1_1', 'exp_e_cov_1_2',
'exp_e_cov_2_1', 'exp_e_cov_2_2'
]:
dlist.append((tag, 'f8'))
ng_final = np.zeros(len(d), dtype=dlist)
ng_final['coadd_objects_id'] = d['coadd_objects_id']
for tag in ng.dtype.names:
if tag != 'coadd_objects_id':
if '_t' in tag:
final_tag = tag.replace('_t', '_T')
else:
final_tag = tag
ng_final[final_tag][finds] = ng[tag][nginds]
else:
assert np.array_equal(ng_final[tag][finds], ng[tag][nginds])
del ng
ng_final['exp_log10sb_i'][finds] = np.log10(
np.abs(ng_final['exp_flux_i'][finds] / ng_final['exp_T'][finds]))
ng_final['exp_e_sens_avg'] = (ng_final['exp_e_sens_1'] +
ng_final['exp_e_sens_2']) / 2.0
# s2n measures
for tag in ['exp_T_r', 'exp_s2n_r', 'exp_T_s2n_r']:
ng_final[tag][finds] = ngs2n[tag][nginds]
del ngs2n
# swap e1 signs
for tag in ['exp_e_1', 'exp_e_cov_1_2', 'psfrec_e_1']:
ng_final[tag] *= -1.0
# fianl tag comp that depends on psf sign
ng_final['exp_e_cov_2_1'] = ng_final['exp_e_cov_1_2']
ng_final['exp_w'] = 1.0 / (2.0 * 0.22 * 0.22 + ng_final['exp_e_cov_1_1'] +
ng_final['exp_e_cov_2_2'])
# unblind
for tag in ['exp_e_1', 'exp_e_2']:
ng_final[tag] = ng_final[tag] / unblind.get_factor()
fitsio.write(config['flatcats_ngmix'], ng_final, clobber=True)
del ng_final
#####################
# im3shape file
# find stuff in catalog
im = fitsio.read(config['im3shape'])
# make sure everything is in the im3shape file
q, = np.where(d['im3shape_flags'] == 0)
flagged_ids = set(d['coadd_objects_id'][q])
imids = set(im['coadd_objects_id'])
assert flagged_ids <= imids
fids = d['coadd_objects_id']
imids = im['coadd_objects_id']
finds, iminds = numpy_util.match(fids, imids)
# fill data
dlist = im.dtype.descr
im_final = np.zeros(len(d), dtype=dlist)
im_final['coadd_objects_id'] = d['coadd_objects_id']
for tag in im.dtype.names:
if tag != 'coadd_objects_id':
im_final[tag][finds] = im[tag][iminds]
else:
assert np.array_equal(im_final[tag][finds], im[tag][iminds])
# clip the weights
np.clip(im_final['w'], 0.0, 0.24**(-2.0), im_final['w'])
# unblind
for tag in ['e1', 'e2']:
im_final[tag] = im_final[tag] / unblind.get_factor()
fitsio.write(config['flatcats_im3shape'], im_final, clobber=True)
def main(config):
###############################
# info file
# do the flags
flags = fitsio.read(config['gold_flags'])
imflags = fitsio.read(config['im3shape_flags'])
ngflags = fitsio.read(config['ngmix_flags'])
if False:
q, = np.where((flags['sva1_gold_flags'] <= 3) &
(flags['sva1_spte_flags'] == 0) &
(flags['sva1_gold_mag_flags'] == 0) &
((imflags['im3shape_flags'] == 0) |
(ngflags['ngmix_flags'] == 0)))
else:
q, = np.where((flags['sva1_gold_flags'] <= 3) &
(flags['sva1_spte_flags'] == 0) &
(flags['sva1_gold_mag_flags'] == 0))
flags = flags[q]
imflags = imflags[q]
ngflags = ngflags[q]
# print number of sources
for code,cflags,tag in zip(['ngmix','im3shape'],[ngflags,imflags],['ngmix_flags','im3shape_flags']):
q, = np.where((flags['sva1_gold_flags'] == 0) & (cflags[tag] == 0))
print "%s:" % code,len(q)
# read in the photo-z bins - match
dtot = []
dz = []
if 'photoz_binning_im3shape' in config:
dim = np.load(config['photoz_binning_im3shape']).astype(int)
for i in xrange(len(dim)):
dtot.append(tuple(dim[i]))
dim = np.load(config['photoz_binning_im3shape']).astype(float)
for i in xrange(len(dim)):
dz.append(tuple(dim[i])[-1])
del dim
if 'photoz_binning_ngmix' in config:
dng = np.load(config['photoz_binning_ngmix']).astype(int)
for i in xrange(len(dng)):
dtot.append(tuple(dng[i]))
dng = np.load(config['photoz_binning_ngmix']).astype(float)
for i in xrange(len(dng)):
dz.append(tuple(dng[i])[-1])
del dng
if 'photoz_binning_extra' in config:
de = np.load(config['photoz_binning_extra']).astype(int)
for i in xrange(len(de)):
dtot.append(tuple(de[i]))
de = np.load(config['photoz_binning_extra']).astype(float)
for i in xrange(len(de)):
dz.append(tuple(de[i])[-1])
del de
if len(dtot) > 0:
dtot = np.array(dtot,dtype=[('coadd_objects_id','i8'),('photoz_bin','i4'),('mean_photoz','f8')])
dz = np.array(dz,dtype='f8')
dtot['mean_photoz'][:] = dz[:]
u, indices = np.unique(dtot['coadd_objects_id'], return_index=True)
dtot = dtot[indices]
del u,indices
# make sure all in file
fids = set(flags['coadd_objects_id'])
pzids = set(dtot['coadd_objects_id'])
if not fids <= pzids:
print "not all gals got photoz binning indexes!"
if True:
fids = flags['coadd_objects_id']
pzids = dtot['coadd_objects_id']
finds,pzinds = numpy_util.match(fids,pzids)
finds = set(finds)
tinds = set(np.arange(len(flags)))
minds = tinds - finds
minds = np.array(list(minds))
#print set(flags['coadd_objects_id'][minds])
print '# of missing gals is',len(minds)
#assert False
fids = flags['coadd_objects_id']
pzids = dtot['coadd_objects_id']
finds,pzinds = numpy_util.match(fids,pzids)
# make final catalog
ftags = ['mag_auto_g','mag_auto_r','mag_auto_i','mag_auto_z',
'ra','dec','mean_photoz']
itags = ['coadd_objects_id','sva1_flags','im3shape_flags','ngmix_flags',
'sva1_gold_flags','sva1_spte_flags','sva1_gold_mag_flags',
'photoz_bin']
dlist = []
for ftag in ftags:
dlist.append((ftag,'f8'))
for itag in itags:
dlist.append((itag,'i8'))
d = np.zeros(len(flags),dtype=dlist)
d['photoz_bin'][:] = -99
d['mean_photoz'][:] = -99.0
for tag in flags.dtype.names:
d[tag] = flags[tag]
for tag in imflags.dtype.names:
d[tag] = imflags[tag]
for tag in ngflags.dtype.names:
d[tag] = ngflags[tag]
if len(dtot) > 0:
d['photoz_bin'][finds] = dtot['photoz_bin'][pzinds]
d['mean_photoz'][finds] = dtot['mean_photoz'][pzinds]
q, = np.where(flags['sva1_gold_flags'] & 1)
d['sva1_flags'][q] |= 1
q, = np.where(flags['sva1_gold_flags'] & 2)
d['sva1_flags'][q] |= 2
q, = np.where((flags['sva1_gold_flags'] >> 2) != 0)
d['sva1_flags'][q] |= 4
q, = np.where(flags['sva1_spte_flags'] != 0)
d['sva1_flags'][q] |= 8
q, = np.where(flags['sva1_gold_mag_flags'] != 0)
d['sva1_flags'][q] |= 16
fitsio.write(config['flatcats_info'],d,clobber=True)
del flags
del imflags
del ngflags
del dtot
###############################
# ngmix file
# first have to match to psf and s2n
ng = fitsio.read(config['ngmix'])
psf = fitsio.read(config['ngmix_psfs'])
pids = psf['id']
ngids = ng['coadd_objects_id']
pinds,nginds = numpy_util.match(pids,ngids)
assert np.all(psf['flags'][pinds] == 0)
ng['psfrec_t'][nginds] = psf['psfrec_T'][pinds]
ng['psfrec_e_1'][nginds] = psf['psfrec_e'][pinds,0]
ng['psfrec_e_2'][nginds] = psf['psfrec_e'][pinds,1]
ng = ng[nginds]
del psf,pids,ngids,pinds,nginds
# now match to new s2n measures
ngs2n = fitsio.read(config['ngmix_s2n'])
s2nids = ngs2n['id']
ngids = ng['coadd_objects_id']
s2ninds,nginds = numpy_util.match(s2nids,ngids)
ng = ng[nginds]
ngs2n = ngs2n[s2ninds]
del s2ninds,nginds,s2nids,ngids
# make sure everything is in the ngmix file
q, = np.where(d['ngmix_flags'] == 0)
flagged_ids = set(d['coadd_objects_id'][q])
ngids = set(ng['coadd_objects_id'])
assert flagged_ids <= ngids
# match to total file
fids = d['coadd_objects_id']
ngids = ng['coadd_objects_id']
finds,nginds = numpy_util.match(fids,ngids)
assert len(finds) == len(d)
# extra tags
dlist = ng.dtype.descr
dlist.append(('exp_log10sb_i','f8'))
dlist.append(('exp_w','f8'))
dlist.append(('exp_e_sens_avg','f8'))
dlist.append(('exp_T_r','f8'))
dlist.append(('exp_s2n_r','f8'))
dlist.append(('exp_T_s2n_r','f8'))
dnew = []
for dt in dlist:
if '_t' in dt[0]:
name = dt[0]
name = name.replace('_t','_T')
dnew.append((name,dt[1]))
elif dt[0] not in ['exp_e_1','exp_e_2','exp_e_cov_1_1','exp_e_cov_1_2','exp_e_cov_2_2']:
dnew.append(dt)
dlist = dnew
for tag in ['exp_e_1','exp_e_2','exp_e_cov_1_1','exp_e_cov_1_2','exp_e_cov_2_1','exp_e_cov_2_2']:
dlist.append((tag,'f8'))
ng_final = np.zeros(len(d),dtype=dlist)
ng_final['coadd_objects_id'] = d['coadd_objects_id']
for tag in ng.dtype.names:
if tag != 'coadd_objects_id':
if '_t' in tag:
final_tag = tag.replace('_t','_T')
else:
final_tag = tag
ng_final[final_tag][finds] = ng[tag][nginds]
else:
assert np.array_equal(ng_final[tag][finds],ng[tag][nginds])
del ng
ng_final['exp_log10sb_i'][finds] = np.log10(np.abs(ng_final['exp_flux_i'][finds]/ng_final['exp_T'][finds]))
ng_final['exp_e_sens_avg'] = (ng_final['exp_e_sens_1'] + ng_final['exp_e_sens_2'])/2.0
# s2n measures
for tag in ['exp_T_r','exp_s2n_r','exp_T_s2n_r']:
ng_final[tag][finds] = ngs2n[tag][nginds]
del ngs2n
# swap e1 signs
for tag in ['exp_e_1','exp_e_cov_1_2','psfrec_e_1']:
ng_final[tag] *= -1.0
# fianl tag comp that depends on psf sign
ng_final['exp_e_cov_2_1'] = ng_final['exp_e_cov_1_2']
ng_final['exp_w'] = 1.0/(2.0*0.22*0.22 + ng_final['exp_e_cov_1_1'] + ng_final['exp_e_cov_2_2'])
# unblind
for tag in ['exp_e_1','exp_e_2']:
ng_final[tag] = ng_final[tag]/unblind.get_factor()
fitsio.write(config['flatcats_ngmix'],ng_final,clobber=True)
del ng_final
#####################
# im3shape file
# find stuff in catalog
im = fitsio.read(config['im3shape'])
# make sure everything is in the im3shape file
q, = np.where(d['im3shape_flags'] == 0)
flagged_ids = set(d['coadd_objects_id'][q])
imids = set(im['coadd_objects_id'])
assert flagged_ids <= imids
fids = d['coadd_objects_id']
imids = im['coadd_objects_id']
finds,iminds = numpy_util.match(fids,imids)
# fill data
dlist = im.dtype.descr
im_final = np.zeros(len(d),dtype=dlist)
im_final['coadd_objects_id'] = d['coadd_objects_id']
for tag in im.dtype.names:
if tag != 'coadd_objects_id':
im_final[tag][finds] = im[tag][iminds]
else:
assert np.array_equal(im_final[tag][finds],im[tag][iminds])
# clip the weights
np.clip(im_final['w'], 0.0, 0.24**(-2.0), im_final['w'])
# unblind
for tag in ['e1','e2']:
im_final[tag] = im_final[tag]/unblind.get_factor()
fitsio.write(config['flatcats_im3shape'],im_final,clobber=True)
def make_im3shape_flags(config):
"""
code to make im3shape flags
People to bug if things are weird
=================================
Matthew Becker
im3shape team
Input Data
==========
im3shape v97 - use to set the flags
im3shape_flags
==============
bit 2^bit name notes
----------------------------------------------------------------------
2 4 NOTORBAD_MEAS object is not in gold or does not meet required flags below
Required Cuts
--------------
Any source which does not meet these cuts is not well measured in im3shape
error_flag == 0
info_flag == 0
snr > 15
mean_rgpp_rp > 1.2
finite(psf_e1,psf_e1,nbc_m,nbc_c1,nbc_c2)
"""
# first read in the data
gold = fitsio.read(config['gold'],lower=True,columns=['coadd_objects_id'])
im = fitsio.read(config['im3shape'])
dflag = np.zeros(len(gold),dtype=[('im3shape_flags','i8')])
#####################
# do im3shape cuts
tag = 'im3shape_flags'
# find stuff in catalog
gids = gold['coadd_objects_id']
imids = im['coadd_objects_id']
ginds,iminds = numpy_util.match(gids,imids)
# flag stuff not in cat
sginds = set(ginds)
totginds = set(range(len(gold)))
diffginds = totginds - sginds
q = np.fromiter(diffginds,dtype=int,count=len(diffginds))
if len(q) > 0:
dflag[tag][q] |= 4
# required cuts
q, = np.where(~((im['error_flag'][iminds] == 0) &
(im['info_flag'][iminds] == 0) &
(im['snr'][iminds] > 15.0) &
(im['mean_rgpp_rp'][iminds] > 1.2) &
(np.isfinite(im['mean_psf_e1_sky'][iminds])) &
(np.isfinite(im['mean_psf_e2_sky'][iminds])) &
(np.isfinite(im['nbc_m'][iminds])) &
(np.isfinite(im['nbc_c1'][iminds])) &
(np.isfinite(im['nbc_c2'][iminds]))
))
if len(q) > 0:
dflag[tag][ginds[q]] |= 4
#write to disk
fitsio.write(config['im3shape_flags'],dflag,clobber=True)
def make_im3shape_flags(config):
"""
code to make im3shape flags
People to bug if things are weird
=================================
Matthew Becker
im3shape team
Input Data
==========
im3shape v97 - use to set the flags
im3shape_flags
==============
bit 2^bit name notes
----------------------------------------------------------------------
2 4 NOTORBAD_MEAS object is not in gold or does not meet required flags below
Required Cuts
--------------
Any source which does not meet these cuts is not well measured in im3shape
error_flag == 0
info_flag == 0
snr > 15
mean_rgpp_rp > 1.2
finite(psf_e1,psf_e1,nbc_m,nbc_c1,nbc_c2)
"""
# first read in the data
gold = fitsio.read(config['gold'],
lower=True,
columns=['coadd_objects_id'])
im = fitsio.read(config['im3shape'])
dflag = np.zeros(len(gold), dtype=[('im3shape_flags', 'i8')])
#####################
# do im3shape cuts
tag = 'im3shape_flags'
# find stuff in catalog
gids = gold['coadd_objects_id']
imids = im['coadd_objects_id']
ginds, iminds = numpy_util.match(gids, imids)
# flag stuff not in cat
sginds = set(ginds)
totginds = set(range(len(gold)))
diffginds = totginds - sginds
q = np.fromiter(diffginds, dtype=int, count=len(diffginds))
if len(q) > 0:
dflag[tag][q] |= 4
# required cuts
q, = np.where(~(
(im['error_flag'][iminds] == 0) & (im['info_flag'][iminds] == 0)
& (im['snr'][iminds] > 15.0) & (im['mean_rgpp_rp'][iminds] > 1.2)
& (np.isfinite(im['mean_psf_e1_sky'][iminds]))
& (np.isfinite(im['mean_psf_e2_sky'][iminds]))
& (np.isfinite(im['nbc_m'][iminds]))
& (np.isfinite(im['nbc_c1'][iminds]))
& (np.isfinite(im['nbc_c2'][iminds]))))
if len(q) > 0:
dflag[tag][ginds[q]] |= 4
#write to disk
fitsio.write(config['im3shape_flags'], dflag, clobber=True)
def make_ngmix_flags(config):
"""
code to make ngmix flags
People to bug if things are weird
=================================
Matthew Becker
Erin Sheldon
Mike Jarvis
Input Data
==========
ngmix11 - use to set the flags
See these pages
https://cdcvs.fnal.gov/redmine/projects/deswlwg/wiki/Ngmix011
https://cdcvs.fnal.gov/redmine/projects/des-sci-verification/wiki/A_Modest_Proposal_for_Preliminary_StarGalaxy_Separation
ngmix_flags
===========
We use three levels of flagging, selecting sets of ngmix
sources which have various potential levels of additive and
multiplicative biases.
bit 2^bit name notes
----------------------------------------------------------------------
2 4 NOTORBAD_MEAS object is not in ngmix or does not meet required flags belo
Required Cuts
--------------
Any source which does not meet these cuts is not well measured in ngmix
flags == 0
exp_flags == 0
0.4 < exp_arate < 0.6
exp_e_sens_1 > 0.0
exp_e_sens_2 > 0.0
round_flags == 0
exp_round_flags == 0
exp_s2n_r > 15
exp_T_r/psfrec_T > 0.15
"""
# first read in the data
gold = fitsio.read(config['gold'],lower=True,columns=['coadd_objects_id'])
ng = fitsio.read(config['ngmix'])
psf = fitsio.read(config['ngmix_psfs'])
ngs2n = fitsio.read(config['ngmix_s2n'])
# match to psf
pids = psf['id']
ngids = ng['coadd_objects_id']
pinds,nginds = numpy_util.match(pids,ngids)
assert np.all(psf['flags'][pinds] == 0)
ng = ng[nginds]
psf = psf[pinds]
del pids,pinds
# match to s/n
s2nids = ngs2n['id']
ngids = ng['coadd_objects_id']
s2ninds,nginds = numpy_util.match(s2nids,ngids)
ng = ng[nginds]
ngs2n = ngs2n[s2ninds]
del s2ninds,s2nids
# now do output, matching to gold
dflag = np.zeros(len(gold),dtype=[('ngmix_flags','i8')])
tag = 'ngmix_flags'
gids = gold['coadd_objects_id']
ngids = ng['coadd_objects_id']
ginds,nginds = numpy_util.match(gids,ngids)
# flag stuff not in ngmix011
sginds = set(ginds)
totginds = set(range(len(gold)))
diffginds = totginds - sginds
q = np.fromiter(diffginds,dtype=int,count=len(diffginds))
if len(q) > 0:
dflag[tag][q] |= 4
# bad measurements
q, = np.where(~((ng['flags'][nginds] == 0) &
(ng['exp_flags'][nginds] == 0) &
(ng['exp_arate'][nginds] > 0.4) &
(ng['exp_arate'][nginds] < 0.6) &
(ng['exp_e_sens_2'][nginds] > 0.0) &
(ng['exp_e_sens_1'][nginds] > 0.0) &
(ngs2n['round_flags'][nginds] == 0) &
(ngs2n['exp_round_flags'][nginds] == 0) &
(ngs2n['exp_s2n_r'][nginds] > 15.0) &
(ngs2n['exp_T_r'][nginds]/psf['psfrec_T'][nginds] > 0.15)
))
if len(q) > 0:
dflag[tag][ginds[q]] |= 4
#write to disk
fitsio.write(config['ngmix_flags'],dflag,clobber=True)
def make_gold_flags(config):
"""
code to make gold flags for DES WL analysis
People to bug if things are weird
=================================
Matthew Becker
Eli Rykoff
Erin Sheldon
Input Data
==========
We use the following sources of data.
1) SVA1 gold v1.0.4 with the additional star masking,
bad regions mask, crazy colors, and large g- vs i-band
offsets. See this page
https://cdcvs.fnal.gov/redmine/projects/des-sci-verification/wiki/SVA1_Gold_Catalog_v10#Gold-104
2) ngmix11 - used to cut low surface brightness junk,
additonal stars and to set the ngmix flags of course
See these pages
https://cdcvs.fnal.gov/redmine/projects/deswlwg/wiki/Ngmix011
https://cdcvs.fnal.gov/redmine/projects/des-sci-verification/wiki/A_Modest_Proposal_for_Preliminary_StarGalaxy_Separation
Flag File Description
======================
The flag file consists of three columns which are
sva1_gold_flags
---------------
These are flags related to SVA1 image processing and galaxy selection. We
have attempted to preserve as much information as possible. The field is
created by concatenating the SExtractor flags (i-band), SVA1 gold flags,
and additional galaxy selection criteria from ngmix11
bit 2^bit name notes
----------------------------------------------------------------------
0 1 SX_KINDOF_BLEND bit 0 from the i-band sexctractor flags
1 2 SX_DEF_BLEND bit 1 from the i-band sexctractor flags
2 4 MODEST_STAR SVA1 Gold MODEST_CLASS == 2
3 8 MODEST_JUNK SVA1 Gold MODEST_CLASS == 0
4 16 CRAZY_COLORS SVA1 Gold crazy mag auto colors
5 32 BADMASK04 in region with lots of JUNK_DPOS objects (removes ~4% of area)
6 64 NEAR_2MASS near a 2MASS star
7 128 JUNK_DPOS large offset in g and i band windowed positions
8 256 NOT_IN_NGMIX011 object did not get measured by ngmix011
9 512 NGMIX011_STAR object is a star according to ngmix011 modest classifier
10 1024 LOW_SB_NGMIX011 object has low surface brightness in ngmix011
11 2048 BAD_MEAS_NGMIX011 object does not satisfy good measurement cuts in ngmix
Some cuts from ngmix011 are applied to the full gold sample. These are detailed below
and are in flag bits 8 through 12.
We use a modest-style star galaxy separation such that objects
with
exp_t + exp_t_err > 0.02
are galaxies. We also cut objects with
exp_mag_i + 3.5*log10(exp_flux_i/exp_t) < 28.0
where
exp_flux_i = 10.0^((exp_mag_i-30.0)/(-2.5))
Note that for the ngmix cuts we require that for ngmix
flags == 0
exp_flags == 0
0.4 < exp_arate < 0.6
so that object has a good measurement at all.
sva1_spte_flags
---------------
We flag all galaxies with ra,dec not in
50 < ra < 100
dec < -35.0
as outside of the SPTE region.
sva1_gold_mag_flags
-------------------
We flag all objects that do not have griz mags.
bit 2^bit name notes
----------------------------------------------------------------------
0 1 MISSING_GRIZ_MAGS object does not have all griz mags
"""
# first read in the data
gold = fitsio.read(
config['gold'],
lower=True,
columns=['coadd_objects_id', 'modest_class', 'flags_i', 'ra', 'dec'])
auto = fitsio.read(config['auto'], lower=True)
badflag = fitsio.read(config['badflag'], lower=True)
ng = fitsio.read(config['ngmix'])
# make output data
dflag = np.zeros(len(gold),
dtype=[('coadd_objects_id', 'i8'), ('ra', 'f8'),
('dec', 'f8'), ('sva1_gold_flags', 'i8'),
('sva1_spte_flags', 'i8'),
('sva1_gold_mag_flags', 'i8'),
('mag_auto_g', 'f8'), ('mag_auto_r', 'f8'),
('mag_auto_i', 'f8'), ('mag_auto_z', 'f8')])
for tag in ['coadd_objects_id', 'ra', 'dec']:
dflag[tag] = gold[tag]
for tag in ['mag_auto_g', 'mag_auto_r', 'mag_auto_i', 'mag_auto_z']:
dflag[tag] = auto[tag]
# set sva1_gold_flags
tag = 'sva1_gold_flags'
dflag[tag] = gold['flags_i']
q, = np.where(gold['modest_class'] == 2)
if len(q) > 0:
dflag[tag][q] |= 4
q, = np.where(gold['modest_class'] == 0)
if len(q) > 0:
dflag[tag][q] |= 8
# crazy colors
# (g-r) < -1, (g-r) > 4, (i-z) < -1, (i-z) > 4
q, = np.where((auto['mag_auto_g'] - auto['mag_auto_r'] < -1.0)
| (auto['mag_auto_g'] - auto['mag_auto_r'] > 4.0)
| (auto['mag_auto_i'] - auto['mag_auto_z'] < -1.0)
| (auto['mag_auto_i'] - auto['mag_auto_z'] > 4.0))
if len(q) > 0:
dflag[tag][q] |= 16
# badflags masking
q, = np.where((badflag['badflag'] & 2) != 0)
if len(q) > 0:
dflag[tag][q] |= 32
q, = np.where((badflag['badflag'] & 4) != 0)
if len(q) > 0:
dflag[tag][q] |= 64
q, = np.where((badflag['badflag'] & 8) != 0)
if len(q) > 0:
dflag[tag][q] |= 128
del badflag
# ngmix cuts
# first have to match
gids = gold['coadd_objects_id']
ngids = ng['coadd_objects_id']
ginds, nginds = numpy_util.match(gids, ngids)
# flag stuff not in ngmix011
sginds = set(ginds)
totginds = set(range(len(gold)))
diffginds = totginds - sginds
q = np.fromiter(diffginds, dtype=int, count=len(diffginds))
if len(q) > 0:
dflag[tag][q] |= 256
# bad measurements
q, = np.where(~(
(ng['flags'][nginds] == 0) & (ng['exp_flags'][nginds] == 0)
& (ng['exp_arate'][nginds] > 0.4) & (ng['exp_arate'][nginds] < 0.6)))
if len(q) > 0:
dflag[tag][ginds[q]] |= 2048
# low SB
sb = ng['exp_mag_i'] + 3.5 * np.log10(
np.abs(ng['exp_flux_i'] / ng['exp_t']))
q, = np.where(~((ng['flags'][nginds] == 0) & (ng['exp_flags'][nginds] == 0)
& (ng['exp_arate'][nginds] > 0.4)
& (ng['exp_arate'][nginds] < 0.6) & (sb[nginds] > 28.0)))
if len(q) > 0:
dflag[tag][ginds[q]] |= 1024
# ngmix star
nsig = 1.0
sgc = ng['exp_t'] + nsig * ng['exp_t_err']
q, = np.where(~((ng['flags'][nginds] == 0) & (ng['exp_flags'][nginds] == 0)
& (ng['exp_arate'][nginds] > 0.4)
& (ng['exp_arate'][nginds] < 0.6) & (sgc[nginds] > 0.02)))
if len(q) > 0:
dflag[tag][ginds[q]] |= 512
# non-99 mag_auto in all bands
tag = 'sva1_gold_mag_flags'
q, = np.where(~((np.abs(auto['mag_auto_g']) < 99.0)
& (np.abs(auto['mag_auto_r']) < 99.0)
& (np.abs(auto['mag_auto_i']) < 99.0)
& (np.abs(auto['mag_auto_z']) < 99.0)))
if len(q) > 0:
dflag[tag][q] |= 1
del auto
#set sva1_spte_flags
q, = np.where(~(
(gold['ra'] > 50.0) & (gold['ra'] < 100.0) & (gold['dec'] < -35.0)))
if len(q) > 0:
dflag['sva1_spte_flags'][q] != 1
#write to disk
fitsio.write(config['gold_flags'], dflag, clobber=True)