def main(config):
###############################
# info file
# do the flags
flags = fitsio.read(config['gold_flags'])
imflags = fitsio.read(config['im3shape_flags'])
ngflags = fitsio.read(config['ngmix_flags'])
if False:
q, = np.where((flags['sva1_gold_flags'] <= 3)
& (flags['sva1_spte_flags'] == 0)
& (flags['sva1_gold_mag_flags'] == 0)
& ((imflags['im3shape_flags'] == 0)
| (ngflags['ngmix_flags'] == 0)))
else:
q, = np.where((flags['sva1_gold_flags'] <= 3)
& (flags['sva1_spte_flags'] == 0)
& (flags['sva1_gold_mag_flags'] == 0))
flags = flags[q]
imflags = imflags[q]
ngflags = ngflags[q]
# print number of sources
for code, cflags, tag in zip(['ngmix', 'im3shape'], [ngflags, imflags],
['ngmix_flags', 'im3shape_flags']):
q, = np.where((flags['sva1_gold_flags'] == 0) & (cflags[tag] == 0))
print "%s:" % code, len(q)
# read in the photo-z bins - match
dtot = []
dz = []
if 'photoz_binning_im3shape' in config:
dim = np.load(config['photoz_binning_im3shape']).astype(int)
for i in xrange(len(dim)):
dtot.append(tuple(dim[i]))
dim = np.load(config['photoz_binning_im3shape']).astype(float)
for i in xrange(len(dim)):
dz.append(tuple(dim[i])[-1])
del dim
if 'photoz_binning_ngmix' in config:
dng = np.load(config['photoz_binning_ngmix']).astype(int)
for i in xrange(len(dng)):
dtot.append(tuple(dng[i]))
dng = np.load(config['photoz_binning_ngmix']).astype(float)
for i in xrange(len(dng)):
dz.append(tuple(dng[i])[-1])
del dng
if 'photoz_binning_extra' in config:
de = np.load(config['photoz_binning_extra']).astype(int)
for i in xrange(len(de)):
dtot.append(tuple(de[i]))
de = np.load(config['photoz_binning_extra']).astype(float)
for i in xrange(len(de)):
dz.append(tuple(de[i])[-1])
del de
if len(dtot) > 0:
dtot = np.array(dtot,
dtype=[('coadd_objects_id', 'i8'),
('photoz_bin', 'i4'), ('mean_photoz', 'f8')])
dz = np.array(dz, dtype='f8')
dtot['mean_photoz'][:] = dz[:]
u, indices = np.unique(dtot['coadd_objects_id'], return_index=True)
dtot = dtot[indices]
del u, indices
# make sure all in file
fids = set(flags['coadd_objects_id'])
pzids = set(dtot['coadd_objects_id'])
if not fids <= pzids:
print "not all gals got photoz binning indexes!"
if True:
fids = flags['coadd_objects_id']
pzids = dtot['coadd_objects_id']
finds, pzinds = numpy_util.match(fids, pzids)
finds = set(finds)
tinds = set(np.arange(len(flags)))
minds = tinds - finds
minds = np.array(list(minds))
#print set(flags['coadd_objects_id'][minds])
print '# of missing gals is', len(minds)
#assert False
fids = flags['coadd_objects_id']
pzids = dtot['coadd_objects_id']
finds, pzinds = numpy_util.match(fids, pzids)
# make final catalog
ftags = [
'mag_auto_g', 'mag_auto_r', 'mag_auto_i', 'mag_auto_z', 'ra', 'dec',
'mean_photoz'
]
itags = [
'coadd_objects_id', 'sva1_flags', 'im3shape_flags', 'ngmix_flags',
'sva1_gold_flags', 'sva1_spte_flags', 'sva1_gold_mag_flags',
'photoz_bin'
]
dlist = []
for ftag in ftags:
dlist.append((ftag, 'f8'))
for itag in itags:
dlist.append((itag, 'i8'))
d = np.zeros(len(flags), dtype=dlist)
d['photoz_bin'][:] = -99
d['mean_photoz'][:] = -99.0
for tag in flags.dtype.names:
d[tag] = flags[tag]
for tag in imflags.dtype.names:
d[tag] = imflags[tag]
for tag in ngflags.dtype.names:
d[tag] = ngflags[tag]
if len(dtot) > 0:
d['photoz_bin'][finds] = dtot['photoz_bin'][pzinds]
d['mean_photoz'][finds] = dtot['mean_photoz'][pzinds]
q, = np.where(flags['sva1_gold_flags'] & 1)
d['sva1_flags'][q] |= 1
q, = np.where(flags['sva1_gold_flags'] & 2)
d['sva1_flags'][q] |= 2
q, = np.where((flags['sva1_gold_flags'] >> 2) != 0)
d['sva1_flags'][q] |= 4
q, = np.where(flags['sva1_spte_flags'] != 0)
d['sva1_flags'][q] |= 8
q, = np.where(flags['sva1_gold_mag_flags'] != 0)
d['sva1_flags'][q] |= 16
fitsio.write(config['flatcats_info'], d, clobber=True)
del flags
del imflags
del ngflags
del dtot
###############################
# ngmix file
# first have to match to psf and s2n
ng = fitsio.read(config['ngmix'])
psf = fitsio.read(config['ngmix_psfs'])
pids = psf['id']
ngids = ng['coadd_objects_id']
pinds, nginds = numpy_util.match(pids, ngids)
assert np.all(psf['flags'][pinds] == 0)
ng['psfrec_t'][nginds] = psf['psfrec_T'][pinds]
ng['psfrec_e_1'][nginds] = psf['psfrec_e'][pinds, 0]
ng['psfrec_e_2'][nginds] = psf['psfrec_e'][pinds, 1]
ng = ng[nginds]
del psf, pids, ngids, pinds, nginds
# now match to new s2n measures
ngs2n = fitsio.read(config['ngmix_s2n'])
s2nids = ngs2n['id']
ngids = ng['coadd_objects_id']
s2ninds, nginds = numpy_util.match(s2nids, ngids)
ng = ng[nginds]
ngs2n = ngs2n[s2ninds]
del s2ninds, nginds, s2nids, ngids
# make sure everything is in the ngmix file
q, = np.where(d['ngmix_flags'] == 0)
flagged_ids = set(d['coadd_objects_id'][q])
ngids = set(ng['coadd_objects_id'])
assert flagged_ids <= ngids
# match to total file
fids = d['coadd_objects_id']
ngids = ng['coadd_objects_id']
finds, nginds = numpy_util.match(fids, ngids)
assert len(finds) == len(d)
# extra tags
dlist = ng.dtype.descr
dlist.append(('exp_log10sb_i', 'f8'))
dlist.append(('exp_w', 'f8'))
dlist.append(('exp_e_sens_avg', 'f8'))
dlist.append(('exp_T_r', 'f8'))
dlist.append(('exp_s2n_r', 'f8'))
dlist.append(('exp_T_s2n_r', 'f8'))
dnew = []
for dt in dlist:
if '_t' in dt[0]:
name = dt[0]
name = name.replace('_t', '_T')
dnew.append((name, dt[1]))
elif dt[0] not in [
'exp_e_1', 'exp_e_2', 'exp_e_cov_1_1', 'exp_e_cov_1_2',
'exp_e_cov_2_2'
]:
dnew.append(dt)
dlist = dnew
for tag in [
'exp_e_1', 'exp_e_2', 'exp_e_cov_1_1', 'exp_e_cov_1_2',
'exp_e_cov_2_1', 'exp_e_cov_2_2'
]:
dlist.append((tag, 'f8'))
ng_final = np.zeros(len(d), dtype=dlist)
ng_final['coadd_objects_id'] = d['coadd_objects_id']
for tag in ng.dtype.names:
if tag != 'coadd_objects_id':
if '_t' in tag:
final_tag = tag.replace('_t', '_T')
else:
final_tag = tag
ng_final[final_tag][finds] = ng[tag][nginds]
else:
assert np.array_equal(ng_final[tag][finds], ng[tag][nginds])
del ng
ng_final['exp_log10sb_i'][finds] = np.log10(
np.abs(ng_final['exp_flux_i'][finds] / ng_final['exp_T'][finds]))
ng_final['exp_e_sens_avg'] = (ng_final['exp_e_sens_1'] +
ng_final['exp_e_sens_2']) / 2.0
# s2n measures
for tag in ['exp_T_r', 'exp_s2n_r', 'exp_T_s2n_r']:
ng_final[tag][finds] = ngs2n[tag][nginds]
del ngs2n
# swap e1 signs
for tag in ['exp_e_1', 'exp_e_cov_1_2', 'psfrec_e_1']:
ng_final[tag] *= -1.0
# fianl tag comp that depends on psf sign
ng_final['exp_e_cov_2_1'] = ng_final['exp_e_cov_1_2']
ng_final['exp_w'] = 1.0 / (2.0 * 0.22 * 0.22 + ng_final['exp_e_cov_1_1'] +
ng_final['exp_e_cov_2_2'])
# unblind
for tag in ['exp_e_1', 'exp_e_2']:
ng_final[tag] = ng_final[tag] / unblind.get_factor()
fitsio.write(config['flatcats_ngmix'], ng_final, clobber=True)
del ng_final
#####################
# im3shape file
# find stuff in catalog
im = fitsio.read(config['im3shape'])
# make sure everything is in the im3shape file
q, = np.where(d['im3shape_flags'] == 0)
flagged_ids = set(d['coadd_objects_id'][q])
imids = set(im['coadd_objects_id'])
assert flagged_ids <= imids
fids = d['coadd_objects_id']
imids = im['coadd_objects_id']
finds, iminds = numpy_util.match(fids, imids)
# fill data
dlist = im.dtype.descr
im_final = np.zeros(len(d), dtype=dlist)
im_final['coadd_objects_id'] = d['coadd_objects_id']
for tag in im.dtype.names:
if tag != 'coadd_objects_id':
im_final[tag][finds] = im[tag][iminds]
else:
assert np.array_equal(im_final[tag][finds], im[tag][iminds])
# clip the weights
np.clip(im_final['w'], 0.0, 0.24**(-2.0), im_final['w'])
# unblind
for tag in ['e1', 'e2']:
im_final[tag] = im_final[tag] / unblind.get_factor()
fitsio.write(config['flatcats_im3shape'], im_final, clobber=True)
def make_im3shape(config, info):
"""Make the im3shape catalog file
"""
# Setup the dtype:
dt = np.dtype([
('coadd_objects_id', 'i8'),
('e_1', 'f8'),
('e_2', 'f8'),
('nbc_m', 'f8'),
('nbc_c1', 'f8'),
('nbc_c2', 'f8'),
('w', 'f8'),
('error_flag', 'i4'),
('info_flag', 'i4'),
('snr_w', 'f4'),
('snr_r', 'f4'),
('flux_r', 'f4'),
('radius', 'f4'),
('is_bulge', 'i4'),
('mean_rgpp_rp', 'f4'),
('mean_psf_e1', 'f8'),
('mean_psf_e2', 'f8'),
('mean_psf_fwhm', 'f4'),
('ra_shift', 'f4'),
('dec_shift', 'f4'),
('chi2', 'f4'),
('likelihood', 'f4'),
('stamp_size', 'i4'),
('n_exposure', 'i4') ])
# Read the source catalog
imcat = fitsio.read(config['im3shape'])
# im3shape objects are sorted, but has some that are not in info catalog.
im, dim = numpy_util.match(imcat['coadd_objects_id'], info['COADD_OBJECTS_ID'])
#assert np.all(imcat['coadd_objects_id'] == sorted(imcat['coadd_objects_id']))
# Set reasonable defaults for float fields:
data = np.zeros(len(info), dtype=dt)
for col, t in dt.descr:
if 'f' in t:
data[col] = -9999
# The coadd_objects_id column can match for all rows
data['coadd_objects_id'] = info['COADD_OBJECTS_ID']
# Check that the overlap with imcat is correct
assert np.all(data['coadd_objects_id'][dim] == imcat['coadd_objects_id'][im])
# Default error_flag is NO_ATTEMPT
data['error_flag'] = 2**30
data['info_flag'] = 2**25
print 'unique error_flags in original catalog = ',np.unique(imcat['error_flag'])
print 'unique error_flags that match info = ',np.unique(imcat['error_flag'][im])
# Copy in the columns from the source catalog that keep the same name:
for col in ['nbc_m', 'nbc_c1', 'nbc_c2', 'w', 'error_flag', 'info_flag',
'radius', 'mean_rgpp_rp', 'mean_psf_fwhm', 'likelihood',
'stamp_size', 'n_exposure']:
data[col][dim] = imcat[col][im]
print 'unique error_flags in final = ',np.unique(data['error_flag'][dim])
# Some get a new name:
data['e_1'][dim] = imcat['e1'][im]
data['e_2'][dim] = imcat['e2'][im]
data['snr_w'][dim] = imcat['snr'][im]
data['snr_r'][dim] = imcat['round_snr'][im]
data['mean_psf_e1'][dim] = imcat['mean_psf_e1_sky'][im]
data['mean_psf_e2'][dim] = imcat['mean_psf_e2_sky'][im]
data['ra_shift'][dim] = imcat['ra_as'][im]
data['dec_shift'][dim] = imcat['dec_as'][im]
data['chi2'][dim] = imcat['chi2_pixel'][im]
# Do a calculation to get the flux from separate bulge/disc fluxes.
data['is_bulge'][dim] = imcat['bulge_flux'][im] != 0.
# Only one is non-zero:
bulge = imcat['bulge_flux'][im] != 0.
disc = imcat['disc_flux'][im] != 0.
print 'sum isbulge = ',np.sum(data['is_bulge'][dim])
print 'sum bulge_flux > 0. = ',np.sum(imcat['bulge_flux'][im] != 0.)
print 'sum disc_flux > 0. = ',np.sum(imcat['disc_flux'][im] != 0.)
assert np.all((imcat['bulge_flux'][im] != 0) != (imcat['disc_flux'][im] != 0))
data['flux_r'][dim] = imcat['mean_flux'][im] * (imcat['bulge_flux'][im]+imcat['disc_flux'][im])
# clip the weights
data['w'][dim] = np.clip(data['w'][dim], 0.0, 0.24**(-2.0))
# unblind
for tag in ['e_1','e_2']:
data[tag][dim] /= unblind.get_factor()
# Before writing out, make all the names upper case
data.dtype.names = tuple([name.upper() for name in data.dtype.names])
fitsio.write(config['release_im3shape'],data,clobber=True)
def make_ngmix(config, info):
"""Make the ngmix catalog file
"""
# Setup the dtype:
dt = np.dtype([
('coadd_objects_id', 'i8'),
('e_1', 'f8'),
('e_2', 'f8'),
('sens_avg', 'f8'),
('w', 'f8'),
('e_cov_1_1', 'f8'),
('e_cov_1_2', 'f8'),
('e_cov_2_1', 'f8'),
('e_cov_2_2', 'f8'),
('error_flag', 'i4'),
('snr_w', 'f4'),
('snr_r', 'f4'),
('flux_i', 'f4'),
('mag_i', 'f4'),
('t', 'f4'),
('t_err', 'f4'),
('t_r', 'f4'),
('snr_t', 'f4'),
('snr_t_r', 'f4'),
('log10_sb_i', 'f4'),
('mean_psf_e1', 'f8'),
('mean_psf_e2', 'f8'),
('mean_psf_t', 'f4'),
('sens_1', 'f4'),
('sens_2', 'f4'),
('arate', 'f4'),
('stamp_size', 'i4'),
('mask_frac', 'f4') ])
# Read the source catalogs
ngcat = fitsio.read(config['ngmix'])
# ngmix catalogs are not sorted, so need to use match function
# Also, ngmix has some object that are not in info catalog.
ng, dng = numpy_util.match(ngcat['coadd_objects_id'], info['COADD_OBJECTS_ID'])
# Set reasonable defaults for float fields:
data = np.zeros(len(info), dtype=dt)
for col, t in dt.descr:
if 'f' in t:
data[col] = -9999
# The coadd_objects_id column can match for all rows
data['coadd_objects_id'] = info['COADD_OBJECTS_ID']
# Check that the overlap with ngcat is correct
assert np.all(data['coadd_objects_id'][dng] == ngcat['coadd_objects_id'][ng])
# Copy in the columns from the source catalog that keep the same name:
for col in ['mask_frac']:
data[col][dng] = ngcat[col][ng]
# Some columns just need to drop the 'exp_' prefix:
for col in ['e_1', 'e_2', 'flux_i', 'mag_i', 'arate', 't', 't_err',
'e_cov_1_1', 'e_cov_1_2', 'e_cov_2_2']:
data[col][dng] = ngcat['exp_' + col][ng]
# Some need to be renamed:
data['snr_w'][dng] = ngcat['exp_s2n_w'][ng]
data['snr_t'][dng] = ngcat['exp_t_s2n'][ng]
data['sens_1'][dng] = ngcat['exp_e_sens_1'][ng]
data['sens_2'][dng] = ngcat['exp_e_sens_2'][ng]
data['stamp_size'][dng] = ngcat['box_size'][ng]
# Combine the flags we have so far:
print 'flags range from %d to %d'%(
np.min(ngcat['flags'][ngcat['flags']>0]),
np.max(ngcat['flags']))
print 'exp_flags range from %d to %d'%(
np.min(ngcat['exp_flags'][ngcat['exp_flags']>0]),
np.max(ngcat['exp_flags']))
data['error_flag'] = 2**30
data['error_flag'][dng] = ngcat['flags'][ng]
data['error_flag'][dng] |= ngcat['exp_flags'][ng]
# Calculate mean sensitivity
data['sens_avg'][dng] = (ngcat['exp_e_sens_1'][ng] + ngcat['exp_e_sens_2'][ng]) / 2.
# Calculate the recommended weight.
data['w'][dng] = 1.0/(2.0*0.22*0.22 + ngcat['exp_e_cov_1_1'][ng] + ngcat['exp_e_cov_2_2'][ng])
# Calculate log10(sb)
data['log10_sb_i'][dng] = np.log10(np.abs(ngcat['exp_flux_i'][ng]/ngcat['exp_t'][ng]))
# swap e1 signs
for tag in ['e_1', 'e_cov_1_2', 'e_cov_2_1']:
data[tag][dng] *= -1.0
data['e_cov_2_1'][dng] = data['e_cov_1_2'][dng]
# unblind
for tag in ['e_1','e_2']:
data[tag][dng] /= unblind.get_factor()
# Bring in the round columns from the s2n catalog
scat = fitsio.read(config['ngmix_s2n'])
sc, dsc = numpy_util.match(scat['id'], info['COADD_OBJECTS_ID'])
data['snr_r'][dsc] = scat['exp_s2n_r'][sc]
data['t_r'][dsc] = scat['exp_T_r'][sc]
data['snr_t_r'][dsc] = scat['exp_T_s2n_r'][sc]
# Check that round_flags are consisten with current error_flag
print 'round_flags range from %d to %d'%(
np.min(scat['round_flags'][scat['round_flags']>0]),
np.max(scat['round_flags']))
print 'exp_round_flags range from %d to %d'%(
np.min(scat['exp_round_flags'][scat['exp_round_flags']>0]),
np.max(scat['exp_round_flags'][scat['exp_round_flags']<2**30]))
assert np.all(data['error_flag'][dsc][ scat['round_flags'][sc] > 0 ] > 0)
assert np.all(data['error_flag'][dsc][ scat['exp_round_flags'][sc] == 2**30 ] > 0)
# Combine the round flags into the error_flag column
data['error_flag'][dsc] |= scat['round_flags'][sc]
data['error_flag'][dsc] |= scat['exp_round_flags'][sc]
# Bring in the mean psf information from the psf catalog
pcat = fitsio.read(config['ngmix_psfs'])
pc, dpc = numpy_util.match(pcat['id'], info['COADD_OBJECTS_ID'])
data['mean_psf_e1'][dpc] = pcat['psfrec_e'][pc,0]
data['mean_psf_e2'][dpc] = pcat['psfrec_e'][pc,1]
data['mean_psf_t'][dpc] = pcat['psfrec_T'][pc]
# swap e1 signs
for tag in ['mean_psf_e1']:
data[tag][dng] *= -1.0
# Before writing out, make all the names upper case
data.dtype.names = tuple([name.upper() for name in data.dtype.names])
fitsio.write(config['release_ngmix'],data,clobber=True)
def make_im3shape(config, info):
"""Make the im3shape catalog file
"""
# Setup the dtype:
dt = np.dtype([('coadd_objects_id', 'i8'), ('e_1', 'f8'), ('e_2', 'f8'),
('nbc_m', 'f8'), ('nbc_c1', 'f8'), ('nbc_c2', 'f8'),
('w', 'f8'), ('error_flag', 'i4'), ('info_flag', 'i4'),
('snr_w', 'f4'), ('snr_r', 'f4'), ('flux_r', 'f4'),
('radius', 'f4'), ('is_bulge', 'i4'),
('mean_rgpp_rp', 'f4'), ('mean_psf_e1', 'f8'),
('mean_psf_e2', 'f8'), ('mean_psf_fwhm', 'f4'),
('ra_shift', 'f4'), ('dec_shift', 'f4'), ('chi2', 'f4'),
('likelihood', 'f4'), ('stamp_size', 'i4'),
('n_exposure', 'i4')])
# Read the source catalog
imcat = fitsio.read(config['im3shape'])
# im3shape objects are sorted, but has some that are not in info catalog.
im, dim = numpy_util.match(imcat['coadd_objects_id'],
info['COADD_OBJECTS_ID'])
#assert np.all(imcat['coadd_objects_id'] == sorted(imcat['coadd_objects_id']))
# Set reasonable defaults for float fields:
data = np.zeros(len(info), dtype=dt)
for col, t in dt.descr:
if 'f' in t:
data[col] = -9999
# The coadd_objects_id column can match for all rows
data['coadd_objects_id'] = info['COADD_OBJECTS_ID']
# Check that the overlap with imcat is correct
assert np.all(
data['coadd_objects_id'][dim] == imcat['coadd_objects_id'][im])
# Default error_flag is NO_ATTEMPT
data['error_flag'] = 2**30
data['info_flag'] = 2**25
print 'unique error_flags in original catalog = ', np.unique(
imcat['error_flag'])
print 'unique error_flags that match info = ', np.unique(
imcat['error_flag'][im])
# Copy in the columns from the source catalog that keep the same name:
for col in [
'nbc_m', 'nbc_c1', 'nbc_c2', 'w', 'error_flag', 'info_flag',
'radius', 'mean_rgpp_rp', 'mean_psf_fwhm', 'likelihood',
'stamp_size', 'n_exposure'
]:
data[col][dim] = imcat[col][im]
print 'unique error_flags in final = ', np.unique(data['error_flag'][dim])
# Some get a new name:
data['e_1'][dim] = imcat['e1'][im]
data['e_2'][dim] = imcat['e2'][im]
data['snr_w'][dim] = imcat['snr'][im]
data['snr_r'][dim] = imcat['round_snr'][im]
data['mean_psf_e1'][dim] = imcat['mean_psf_e1_sky'][im]
data['mean_psf_e2'][dim] = imcat['mean_psf_e2_sky'][im]
data['ra_shift'][dim] = imcat['ra_as'][im]
data['dec_shift'][dim] = imcat['dec_as'][im]
data['chi2'][dim] = imcat['chi2_pixel'][im]
# Do a calculation to get the flux from separate bulge/disc fluxes.
data['is_bulge'][dim] = imcat['bulge_flux'][im] != 0.
# Only one is non-zero:
bulge = imcat['bulge_flux'][im] != 0.
disc = imcat['disc_flux'][im] != 0.
print 'sum isbulge = ', np.sum(data['is_bulge'][dim])
print 'sum bulge_flux > 0. = ', np.sum(imcat['bulge_flux'][im] != 0.)
print 'sum disc_flux > 0. = ', np.sum(imcat['disc_flux'][im] != 0.)
assert np.all(
(imcat['bulge_flux'][im] != 0) != (imcat['disc_flux'][im] != 0))
data['flux_r'][dim] = imcat['mean_flux'][im] * (imcat['bulge_flux'][im] +
imcat['disc_flux'][im])
# clip the weights
data['w'][dim] = np.clip(data['w'][dim], 0.0, 0.24**(-2.0))
# unblind
for tag in ['e_1', 'e_2']:
data[tag][dim] /= unblind.get_factor()
# Before writing out, make all the names upper case
data.dtype.names = tuple([name.upper() for name in data.dtype.names])
fitsio.write(config['release_im3shape'], data, clobber=True)
def make_ngmix(config, info):
"""Make the ngmix catalog file
"""
# Setup the dtype:
dt = np.dtype([('coadd_objects_id', 'i8'), ('e_1', 'f8'), ('e_2', 'f8'),
('sens_avg', 'f8'), ('w', 'f8'), ('e_cov_1_1', 'f8'),
('e_cov_1_2', 'f8'),
('e_cov_2_1', 'f8'), ('e_cov_2_2', 'f8'),
('error_flag', 'i4'), ('snr_w', 'f4'), ('snr_r', 'f4'),
('flux_i', 'f4'), ('mag_i', 'f4'), ('t', 'f4'),
('t_err', 'f4'), ('t_r', 'f4'), ('snr_t', 'f4'),
('snr_t_r', 'f4'), ('log10_sb_i', 'f4'),
('mean_psf_e1', 'f8'), ('mean_psf_e2', 'f8'),
('mean_psf_t', 'f4'), ('sens_1', 'f4'), ('sens_2', 'f4'),
('arate', 'f4'), ('stamp_size', 'i4'), ('mask_frac', 'f4')])
# Read the source catalogs
ngcat = fitsio.read(config['ngmix'])
# ngmix catalogs are not sorted, so need to use match function
# Also, ngmix has some object that are not in info catalog.
ng, dng = numpy_util.match(ngcat['coadd_objects_id'],
info['COADD_OBJECTS_ID'])
# Set reasonable defaults for float fields:
data = np.zeros(len(info), dtype=dt)
for col, t in dt.descr:
if 'f' in t:
data[col] = -9999
# The coadd_objects_id column can match for all rows
data['coadd_objects_id'] = info['COADD_OBJECTS_ID']
# Check that the overlap with ngcat is correct
assert np.all(
data['coadd_objects_id'][dng] == ngcat['coadd_objects_id'][ng])
# Copy in the columns from the source catalog that keep the same name:
for col in ['mask_frac']:
data[col][dng] = ngcat[col][ng]
# Some columns just need to drop the 'exp_' prefix:
for col in [
'e_1', 'e_2', 'flux_i', 'mag_i', 'arate', 't', 't_err',
'e_cov_1_1', 'e_cov_1_2', 'e_cov_2_2'
]:
data[col][dng] = ngcat['exp_' + col][ng]
# Some need to be renamed:
data['snr_w'][dng] = ngcat['exp_s2n_w'][ng]
data['snr_t'][dng] = ngcat['exp_t_s2n'][ng]
data['sens_1'][dng] = ngcat['exp_e_sens_1'][ng]
data['sens_2'][dng] = ngcat['exp_e_sens_2'][ng]
data['stamp_size'][dng] = ngcat['box_size'][ng]
# Combine the flags we have so far:
print 'flags range from %d to %d' % (np.min(
ngcat['flags'][ngcat['flags'] > 0]), np.max(ngcat['flags']))
print 'exp_flags range from %d to %d' % (np.min(
ngcat['exp_flags'][ngcat['exp_flags'] > 0]), np.max(
ngcat['exp_flags']))
data['error_flag'] = 2**30
data['error_flag'][dng] = ngcat['flags'][ng]
data['error_flag'][dng] |= ngcat['exp_flags'][ng]
# Calculate mean sensitivity
data['sens_avg'][dng] = (ngcat['exp_e_sens_1'][ng] +
ngcat['exp_e_sens_2'][ng]) / 2.
# Calculate the recommended weight.
data['w'][dng] = 1.0 / (2.0 * 0.22 * 0.22 + ngcat['exp_e_cov_1_1'][ng] +
ngcat['exp_e_cov_2_2'][ng])
# Calculate log10(sb)
data['log10_sb_i'][dng] = np.log10(
np.abs(ngcat['exp_flux_i'][ng] / ngcat['exp_t'][ng]))
# swap e1 signs
for tag in ['e_1', 'e_cov_1_2', 'e_cov_2_1']:
data[tag][dng] *= -1.0
data['e_cov_2_1'][dng] = data['e_cov_1_2'][dng]
# unblind
for tag in ['e_1', 'e_2']:
data[tag][dng] /= unblind.get_factor()
# Bring in the round columns from the s2n catalog
scat = fitsio.read(config['ngmix_s2n'])
sc, dsc = numpy_util.match(scat['id'], info['COADD_OBJECTS_ID'])
data['snr_r'][dsc] = scat['exp_s2n_r'][sc]
data['t_r'][dsc] = scat['exp_T_r'][sc]
data['snr_t_r'][dsc] = scat['exp_T_s2n_r'][sc]
# Check that round_flags are consisten with current error_flag
print 'round_flags range from %d to %d' % (np.min(
scat['round_flags'][scat['round_flags'] > 0]),
np.max(scat['round_flags']))
print 'exp_round_flags range from %d to %d' % (
np.min(scat['exp_round_flags'][scat['exp_round_flags'] > 0]),
np.max(scat['exp_round_flags'][scat['exp_round_flags'] < 2**30]))
assert np.all(data['error_flag'][dsc][scat['round_flags'][sc] > 0] > 0)
assert np.all(
data['error_flag'][dsc][scat['exp_round_flags'][sc] == 2**30] > 0)
# Combine the round flags into the error_flag column
data['error_flag'][dsc] |= scat['round_flags'][sc]
data['error_flag'][dsc] |= scat['exp_round_flags'][sc]
# Bring in the mean psf information from the psf catalog
pcat = fitsio.read(config['ngmix_psfs'])
pc, dpc = numpy_util.match(pcat['id'], info['COADD_OBJECTS_ID'])
data['mean_psf_e1'][dpc] = pcat['psfrec_e'][pc, 0]
data['mean_psf_e2'][dpc] = pcat['psfrec_e'][pc, 1]
data['mean_psf_t'][dpc] = pcat['psfrec_T'][pc]
# swap e1 signs
for tag in ['mean_psf_e1']:
data[tag][dng] *= -1.0
# Before writing out, make all the names upper case
data.dtype.names = tuple([name.upper() for name in data.dtype.names])
fitsio.write(config['release_ngmix'], data, clobber=True)
def main(config):
###############################
# info file
# do the flags
flags = fitsio.read(config['gold_flags'])
imflags = fitsio.read(config['im3shape_flags'])
ngflags = fitsio.read(config['ngmix_flags'])
if False:
q, = np.where((flags['sva1_gold_flags'] <= 3) &
(flags['sva1_spte_flags'] == 0) &
(flags['sva1_gold_mag_flags'] == 0) &
((imflags['im3shape_flags'] == 0) |
(ngflags['ngmix_flags'] == 0)))
else:
q, = np.where((flags['sva1_gold_flags'] <= 3) &
(flags['sva1_spte_flags'] == 0) &
(flags['sva1_gold_mag_flags'] == 0))
flags = flags[q]
imflags = imflags[q]
ngflags = ngflags[q]
# print number of sources
for code,cflags,tag in zip(['ngmix','im3shape'],[ngflags,imflags],['ngmix_flags','im3shape_flags']):
q, = np.where((flags['sva1_gold_flags'] == 0) & (cflags[tag] == 0))
print "%s:" % code,len(q)
# read in the photo-z bins - match
dtot = []
dz = []
if 'photoz_binning_im3shape' in config:
dim = np.load(config['photoz_binning_im3shape']).astype(int)
for i in xrange(len(dim)):
dtot.append(tuple(dim[i]))
dim = np.load(config['photoz_binning_im3shape']).astype(float)
for i in xrange(len(dim)):
dz.append(tuple(dim[i])[-1])
del dim
if 'photoz_binning_ngmix' in config:
dng = np.load(config['photoz_binning_ngmix']).astype(int)
for i in xrange(len(dng)):
dtot.append(tuple(dng[i]))
dng = np.load(config['photoz_binning_ngmix']).astype(float)
for i in xrange(len(dng)):
dz.append(tuple(dng[i])[-1])
del dng
if 'photoz_binning_extra' in config:
de = np.load(config['photoz_binning_extra']).astype(int)
for i in xrange(len(de)):
dtot.append(tuple(de[i]))
de = np.load(config['photoz_binning_extra']).astype(float)
for i in xrange(len(de)):
dz.append(tuple(de[i])[-1])
del de
if len(dtot) > 0:
dtot = np.array(dtot,dtype=[('coadd_objects_id','i8'),('photoz_bin','i4'),('mean_photoz','f8')])
dz = np.array(dz,dtype='f8')
dtot['mean_photoz'][:] = dz[:]
u, indices = np.unique(dtot['coadd_objects_id'], return_index=True)
dtot = dtot[indices]
del u,indices
# make sure all in file
fids = set(flags['coadd_objects_id'])
pzids = set(dtot['coadd_objects_id'])
if not fids <= pzids:
print "not all gals got photoz binning indexes!"
if True:
fids = flags['coadd_objects_id']
pzids = dtot['coadd_objects_id']
finds,pzinds = numpy_util.match(fids,pzids)
finds = set(finds)
tinds = set(np.arange(len(flags)))
minds = tinds - finds
minds = np.array(list(minds))
#print set(flags['coadd_objects_id'][minds])
print '# of missing gals is',len(minds)
#assert False
fids = flags['coadd_objects_id']
pzids = dtot['coadd_objects_id']
finds,pzinds = numpy_util.match(fids,pzids)
# make final catalog
ftags = ['mag_auto_g','mag_auto_r','mag_auto_i','mag_auto_z',
'ra','dec','mean_photoz']
itags = ['coadd_objects_id','sva1_flags','im3shape_flags','ngmix_flags',
'sva1_gold_flags','sva1_spte_flags','sva1_gold_mag_flags',
'photoz_bin']
dlist = []
for ftag in ftags:
dlist.append((ftag,'f8'))
for itag in itags:
dlist.append((itag,'i8'))
d = np.zeros(len(flags),dtype=dlist)
d['photoz_bin'][:] = -99
d['mean_photoz'][:] = -99.0
for tag in flags.dtype.names:
d[tag] = flags[tag]
for tag in imflags.dtype.names:
d[tag] = imflags[tag]
for tag in ngflags.dtype.names:
d[tag] = ngflags[tag]
if len(dtot) > 0:
d['photoz_bin'][finds] = dtot['photoz_bin'][pzinds]
d['mean_photoz'][finds] = dtot['mean_photoz'][pzinds]
q, = np.where(flags['sva1_gold_flags'] & 1)
d['sva1_flags'][q] |= 1
q, = np.where(flags['sva1_gold_flags'] & 2)
d['sva1_flags'][q] |= 2
q, = np.where((flags['sva1_gold_flags'] >> 2) != 0)
d['sva1_flags'][q] |= 4
q, = np.where(flags['sva1_spte_flags'] != 0)
d['sva1_flags'][q] |= 8
q, = np.where(flags['sva1_gold_mag_flags'] != 0)
d['sva1_flags'][q] |= 16
fitsio.write(config['flatcats_info'],d,clobber=True)
del flags
del imflags
del ngflags
del dtot
###############################
# ngmix file
# first have to match to psf and s2n
ng = fitsio.read(config['ngmix'])
psf = fitsio.read(config['ngmix_psfs'])
pids = psf['id']
ngids = ng['coadd_objects_id']
pinds,nginds = numpy_util.match(pids,ngids)
assert np.all(psf['flags'][pinds] == 0)
ng['psfrec_t'][nginds] = psf['psfrec_T'][pinds]
ng['psfrec_e_1'][nginds] = psf['psfrec_e'][pinds,0]
ng['psfrec_e_2'][nginds] = psf['psfrec_e'][pinds,1]
ng = ng[nginds]
del psf,pids,ngids,pinds,nginds
# now match to new s2n measures
ngs2n = fitsio.read(config['ngmix_s2n'])
s2nids = ngs2n['id']
ngids = ng['coadd_objects_id']
s2ninds,nginds = numpy_util.match(s2nids,ngids)
ng = ng[nginds]
ngs2n = ngs2n[s2ninds]
del s2ninds,nginds,s2nids,ngids
# make sure everything is in the ngmix file
q, = np.where(d['ngmix_flags'] == 0)
flagged_ids = set(d['coadd_objects_id'][q])
ngids = set(ng['coadd_objects_id'])
assert flagged_ids <= ngids
# match to total file
fids = d['coadd_objects_id']
ngids = ng['coadd_objects_id']
finds,nginds = numpy_util.match(fids,ngids)
assert len(finds) == len(d)
# extra tags
dlist = ng.dtype.descr
dlist.append(('exp_log10sb_i','f8'))
dlist.append(('exp_w','f8'))
dlist.append(('exp_e_sens_avg','f8'))
dlist.append(('exp_T_r','f8'))
dlist.append(('exp_s2n_r','f8'))
dlist.append(('exp_T_s2n_r','f8'))
dnew = []
for dt in dlist:
if '_t' in dt[0]:
name = dt[0]
name = name.replace('_t','_T')
dnew.append((name,dt[1]))
elif dt[0] not in ['exp_e_1','exp_e_2','exp_e_cov_1_1','exp_e_cov_1_2','exp_e_cov_2_2']:
dnew.append(dt)
dlist = dnew
for tag in ['exp_e_1','exp_e_2','exp_e_cov_1_1','exp_e_cov_1_2','exp_e_cov_2_1','exp_e_cov_2_2']:
dlist.append((tag,'f8'))
ng_final = np.zeros(len(d),dtype=dlist)
ng_final['coadd_objects_id'] = d['coadd_objects_id']
for tag in ng.dtype.names:
if tag != 'coadd_objects_id':
if '_t' in tag:
final_tag = tag.replace('_t','_T')
else:
final_tag = tag
ng_final[final_tag][finds] = ng[tag][nginds]
else:
assert np.array_equal(ng_final[tag][finds],ng[tag][nginds])
del ng
ng_final['exp_log10sb_i'][finds] = np.log10(np.abs(ng_final['exp_flux_i'][finds]/ng_final['exp_T'][finds]))
ng_final['exp_e_sens_avg'] = (ng_final['exp_e_sens_1'] + ng_final['exp_e_sens_2'])/2.0
# s2n measures
for tag in ['exp_T_r','exp_s2n_r','exp_T_s2n_r']:
ng_final[tag][finds] = ngs2n[tag][nginds]
del ngs2n
# swap e1 signs
for tag in ['exp_e_1','exp_e_cov_1_2','psfrec_e_1']:
ng_final[tag] *= -1.0
# fianl tag comp that depends on psf sign
ng_final['exp_e_cov_2_1'] = ng_final['exp_e_cov_1_2']
ng_final['exp_w'] = 1.0/(2.0*0.22*0.22 + ng_final['exp_e_cov_1_1'] + ng_final['exp_e_cov_2_2'])
# unblind
for tag in ['exp_e_1','exp_e_2']:
ng_final[tag] = ng_final[tag]/unblind.get_factor()
fitsio.write(config['flatcats_ngmix'],ng_final,clobber=True)
del ng_final
#####################
# im3shape file
# find stuff in catalog
im = fitsio.read(config['im3shape'])
# make sure everything is in the im3shape file
q, = np.where(d['im3shape_flags'] == 0)
flagged_ids = set(d['coadd_objects_id'][q])
imids = set(im['coadd_objects_id'])
assert flagged_ids <= imids
fids = d['coadd_objects_id']
imids = im['coadd_objects_id']
finds,iminds = numpy_util.match(fids,imids)
# fill data
dlist = im.dtype.descr
im_final = np.zeros(len(d),dtype=dlist)
im_final['coadd_objects_id'] = d['coadd_objects_id']
for tag in im.dtype.names:
if tag != 'coadd_objects_id':
im_final[tag][finds] = im[tag][iminds]
else:
assert np.array_equal(im_final[tag][finds],im[tag][iminds])
# clip the weights
np.clip(im_final['w'], 0.0, 0.24**(-2.0), im_final['w'])
# unblind
for tag in ['e1','e2']:
im_final[tag] = im_final[tag]/unblind.get_factor()
fitsio.write(config['flatcats_im3shape'],im_final,clobber=True)