Example #1
0
def make_gold_flags(config):
    """
    code to make gold flags for DES WL analysis

    People to bug if things are weird
    =================================
    
    Matthew Becker
    Eli Rykoff
    Erin Sheldon
    
    Input Data
    ==========
    We use the following sources of data.
    
    1) SVA1 gold v1.0.4 with the additional star masking, 
    bad regions mask, crazy colors, and large g- vs i-band 
    offsets. See this page
    
    https://cdcvs.fnal.gov/redmine/projects/des-sci-verification/wiki/SVA1_Gold_Catalog_v10#Gold-104
    
    2) ngmix11 - used to cut low surface brightness junk, 
    additonal stars and to set the ngmix flags of course
    
    See these pages
    
    https://cdcvs.fnal.gov/redmine/projects/deswlwg/wiki/Ngmix011
    https://cdcvs.fnal.gov/redmine/projects/des-sci-verification/wiki/A_Modest_Proposal_for_Preliminary_StarGalaxy_Separation
    

    Flag File Description
    ======================
    The flag file consists of three columns which are
    
    sva1_gold_flags
    ---------------
    
    These are flags related to SVA1 image processing and galaxy selection. We 
    have attempted to preserve as much information as possible. The field is 
    created by concatenating the SExtractor flags (i-band), SVA1 gold flags, 
    and additional galaxy selection criteria from ngmix11
    
    bit 2^bit   name              notes
    ----------------------------------------------------------------------
    0   1       SX_KINDOF_BLEND    bit 0 from the i-band sexctractor flags
    1   2       SX_DEF_BLEND       bit 1 from the i-band sexctractor flags
    2   4       MODEST_STAR        SVA1 Gold MODEST_CLASS == 2
    3   8       MODEST_JUNK        SVA1 Gold MODEST_CLASS == 0
    4   16      CRAZY_COLORS       SVA1 Gold crazy mag auto colors
    5   32      BADMASK04          in region with lots of JUNK_DPOS objects (removes ~4% of area)
    6   64      NEAR_2MASS         near a 2MASS star
    7   128     JUNK_DPOS          large offset in g and i band windowed positions
    8   256     NOT_IN_NGMIX011    object did not get measured by ngmix011
    9   512     NGMIX011_STAR      object is a star according to ngmix011 modest classifier 
    10  1024    LOW_SB_NGMIX011    object has low surface brightness in ngmix011
    11  2048    BAD_MEAS_NGMIX011  object does not satisfy good measurement cuts in ngmix
    
    Some cuts from ngmix011 are applied to the full gold sample. These are detailed below 
    and are in flag bits 8 through 12.
    
    We use a modest-style star galaxy separation such that objects 
    with 
    
        exp_t + exp_t_err > 0.02
    
    are galaxies. We also cut objects with 

        exp_mag_i + 3.5*log10(exp_flux_i/exp_t) < 28.0

    where

        exp_flux_i = 10.0^((exp_mag_i-30.0)/(-2.5))

    Note that for the ngmix cuts we require that for ngmix

        flags == 0
        exp_flags == 0
        0.4 < exp_arate < 0.6

    so that object has a good measurement at all. 

    sva1_spte_flags
    ---------------
    
    We flag all galaxies with ra,dec not in 

        50 < ra < 100
        dec < -35.0

    as outside of the SPTE region.

    sva1_gold_mag_flags
    -------------------

    We flag all objects that do not have griz mags.
    
    bit 2^bit   name              notes
    ----------------------------------------------------------------------
    0   1       MISSING_GRIZ_MAGS object does not have all griz mags
    
    """

    # first read in the data
    gold = fitsio.read(config['gold'],lower=True,columns=['coadd_objects_id','modest_class','flags_i','ra','dec'])
    auto = fitsio.read(config['auto'],lower=True)
    badflag = fitsio.read(config['badflag'],lower=True)
    ng = fitsio.read(config['ngmix'])
    
    # make output data
    dflag = np.zeros(len(gold),dtype=[('coadd_objects_id','i8'),('ra','f8'),('dec','f8'),
                                      ('sva1_gold_flags','i8'),('sva1_spte_flags','i8'),
                                      ('sva1_gold_mag_flags','i8'),
                                      ('mag_auto_g','f8'),('mag_auto_r','f8'),
                                      ('mag_auto_i','f8'),('mag_auto_z','f8')])
    for tag in ['coadd_objects_id','ra','dec']:
        dflag[tag] = gold[tag]
    for tag in ['mag_auto_g','mag_auto_r','mag_auto_i','mag_auto_z']:
        dflag[tag] = auto[tag]
    
    # set sva1_gold_flags
    tag = 'sva1_gold_flags'
    dflag[tag] = gold['flags_i']
    
    q, = np.where(gold['modest_class'] == 2)
    if len(q) > 0:
        dflag[tag][q] |= 4
        
    q, = np.where(gold['modest_class'] == 0)
    if len(q) > 0:
        dflag[tag][q] |= 8
    
    # crazy colors
    # (g-r) < -1, (g-r) > 4, (i-z) < -1, (i-z) > 4
    q, = np.where((auto['mag_auto_g'] - auto['mag_auto_r'] < -1.0) |
                  (auto['mag_auto_g'] - auto['mag_auto_r'] > 4.0) |
                  (auto['mag_auto_i'] - auto['mag_auto_z'] < -1.0) |
                  (auto['mag_auto_i'] - auto['mag_auto_z'] > 4.0))
    if len(q) > 0:
        dflag[tag][q] |= 16
         
    # badflags masking
    q, = np.where((badflag['badflag'] & 2) != 0)
    if len(q) > 0:
        dflag[tag][q] |= 32
        
    q, = np.where((badflag['badflag'] & 4) != 0)
    if len(q) > 0:
        dflag[tag][q] |= 64
        
    q, = np.where((badflag['badflag'] & 8) != 0)
    if len(q) > 0:
        dflag[tag][q] |= 128
        
    del badflag
        
    # ngmix cuts
    # first have to match
    gids = gold['coadd_objects_id']
    ngids = ng['coadd_objects_id']
    ginds,nginds = numpy_util.match(gids,ngids)
    
    # flag stuff not in ngmix011
    sginds = set(ginds)
    totginds = set(range(len(gold)))
    diffginds = totginds - sginds
    q = np.fromiter(diffginds,dtype=int,count=len(diffginds))
    if len(q) > 0:
        dflag[tag][q] |= 256
                   
    # bad measurements
    q, = np.where(~((ng['flags'][nginds] == 0) &
                    (ng['exp_flags'][nginds] == 0) &
                    (ng['exp_arate'][nginds] > 0.4) & 
                    (ng['exp_arate'][nginds] < 0.6)))
    if len(q) > 0:
        dflag[tag][ginds[q]] |= 2048

    # low SB
    sb = ng['exp_mag_i'] + 3.5*np.log10(np.abs(ng['exp_flux_i']/ng['exp_t']))
    q, = np.where(~((ng['flags'][nginds] == 0) &
                    (ng['exp_flags'][nginds] == 0) &
                    (ng['exp_arate'][nginds] > 0.4) & 
                    (ng['exp_arate'][nginds] < 0.6) & 
                    (sb[nginds] > 28.0)))
    if len(q) > 0:
        dflag[tag][ginds[q]] |= 1024
        
    # ngmix star
    nsig = 1.0
    sgc = ng['exp_t'] + nsig*ng['exp_t_err'] 
    q, = np.where(~((ng['flags'][nginds] == 0) &
                    (ng['exp_flags'][nginds] == 0) &
                    (ng['exp_arate'][nginds] > 0.4) & 
                    (ng['exp_arate'][nginds] < 0.6) & 
                    (sgc[nginds] > 0.02)))
    if len(q) > 0:
        dflag[tag][ginds[q]] |= 512
        
    # non-99 mag_auto in all bands
    tag = 'sva1_gold_mag_flags'
    q, = np.where(~((np.abs(auto['mag_auto_g']) < 99.0) &
                    (np.abs(auto['mag_auto_r']) < 99.0) &
                    (np.abs(auto['mag_auto_i']) < 99.0) &
                    (np.abs(auto['mag_auto_z']) < 99.0)))
    if len(q) > 0:
        dflag[tag][q] |= 1
    del auto
    
    #set sva1_spte_flags
    q, = np.where(~((gold['ra'] > 50.0) & (gold['ra'] < 100.0) & (gold['dec'] < -35.0)))
    if len(q) > 0:
        dflag['sva1_spte_flags'][q] != 1
        
    #write to disk
    fitsio.write(config['gold_flags'],dflag,clobber=True)
Example #2
0
def main(config):
    ###############################
    # info file

    # do the flags
    flags = fitsio.read(config['gold_flags'])
    imflags = fitsio.read(config['im3shape_flags'])
    ngflags = fitsio.read(config['ngmix_flags'])
    if False:
        q, = np.where((flags['sva1_gold_flags'] <= 3)
                      & (flags['sva1_spte_flags'] == 0)
                      & (flags['sva1_gold_mag_flags'] == 0)
                      & ((imflags['im3shape_flags'] == 0)
                         | (ngflags['ngmix_flags'] == 0)))
    else:
        q, = np.where((flags['sva1_gold_flags'] <= 3)
                      & (flags['sva1_spte_flags'] == 0)
                      & (flags['sva1_gold_mag_flags'] == 0))
    flags = flags[q]
    imflags = imflags[q]
    ngflags = ngflags[q]

    # print number of sources
    for code, cflags, tag in zip(['ngmix', 'im3shape'], [ngflags, imflags],
                                 ['ngmix_flags', 'im3shape_flags']):
        q, = np.where((flags['sva1_gold_flags'] == 0) & (cflags[tag] == 0))
        print "%s:" % code, len(q)

    # read in the photo-z bins - match
    dtot = []
    dz = []
    if 'photoz_binning_im3shape' in config:
        dim = np.load(config['photoz_binning_im3shape']).astype(int)
        for i in xrange(len(dim)):
            dtot.append(tuple(dim[i]))
        dim = np.load(config['photoz_binning_im3shape']).astype(float)
        for i in xrange(len(dim)):
            dz.append(tuple(dim[i])[-1])
        del dim
    if 'photoz_binning_ngmix' in config:
        dng = np.load(config['photoz_binning_ngmix']).astype(int)
        for i in xrange(len(dng)):
            dtot.append(tuple(dng[i]))
        dng = np.load(config['photoz_binning_ngmix']).astype(float)
        for i in xrange(len(dng)):
            dz.append(tuple(dng[i])[-1])
        del dng
    if 'photoz_binning_extra' in config:
        de = np.load(config['photoz_binning_extra']).astype(int)
        for i in xrange(len(de)):
            dtot.append(tuple(de[i]))
        de = np.load(config['photoz_binning_extra']).astype(float)
        for i in xrange(len(de)):
            dz.append(tuple(de[i])[-1])
        del de

    if len(dtot) > 0:
        dtot = np.array(dtot,
                        dtype=[('coadd_objects_id', 'i8'),
                               ('photoz_bin', 'i4'), ('mean_photoz', 'f8')])
        dz = np.array(dz, dtype='f8')
        dtot['mean_photoz'][:] = dz[:]
        u, indices = np.unique(dtot['coadd_objects_id'], return_index=True)
        dtot = dtot[indices]
        del u, indices

        # make sure all in file
        fids = set(flags['coadd_objects_id'])
        pzids = set(dtot['coadd_objects_id'])
        if not fids <= pzids:
            print "not all gals got photoz binning indexes!"
            if True:
                fids = flags['coadd_objects_id']
                pzids = dtot['coadd_objects_id']
                finds, pzinds = numpy_util.match(fids, pzids)
                finds = set(finds)
                tinds = set(np.arange(len(flags)))
                minds = tinds - finds
                minds = np.array(list(minds))
                #print set(flags['coadd_objects_id'][minds])
                print '# of missing gals is', len(minds)
                #assert False

        fids = flags['coadd_objects_id']
        pzids = dtot['coadd_objects_id']
        finds, pzinds = numpy_util.match(fids, pzids)

    # make final catalog
    ftags = [
        'mag_auto_g', 'mag_auto_r', 'mag_auto_i', 'mag_auto_z', 'ra', 'dec',
        'mean_photoz'
    ]
    itags = [
        'coadd_objects_id', 'sva1_flags', 'im3shape_flags', 'ngmix_flags',
        'sva1_gold_flags', 'sva1_spte_flags', 'sva1_gold_mag_flags',
        'photoz_bin'
    ]
    dlist = []
    for ftag in ftags:
        dlist.append((ftag, 'f8'))
    for itag in itags:
        dlist.append((itag, 'i8'))
    d = np.zeros(len(flags), dtype=dlist)
    d['photoz_bin'][:] = -99
    d['mean_photoz'][:] = -99.0
    for tag in flags.dtype.names:
        d[tag] = flags[tag]
    for tag in imflags.dtype.names:
        d[tag] = imflags[tag]
    for tag in ngflags.dtype.names:
        d[tag] = ngflags[tag]
    if len(dtot) > 0:
        d['photoz_bin'][finds] = dtot['photoz_bin'][pzinds]
        d['mean_photoz'][finds] = dtot['mean_photoz'][pzinds]

    q, = np.where(flags['sva1_gold_flags'] & 1)
    d['sva1_flags'][q] |= 1

    q, = np.where(flags['sva1_gold_flags'] & 2)
    d['sva1_flags'][q] |= 2

    q, = np.where((flags['sva1_gold_flags'] >> 2) != 0)
    d['sva1_flags'][q] |= 4

    q, = np.where(flags['sva1_spte_flags'] != 0)
    d['sva1_flags'][q] |= 8

    q, = np.where(flags['sva1_gold_mag_flags'] != 0)
    d['sva1_flags'][q] |= 16

    fitsio.write(config['flatcats_info'], d, clobber=True)
    del flags
    del imflags
    del ngflags
    del dtot

    ###############################
    # ngmix file
    # first have to match to psf and s2n
    ng = fitsio.read(config['ngmix'])
    psf = fitsio.read(config['ngmix_psfs'])
    pids = psf['id']
    ngids = ng['coadd_objects_id']
    pinds, nginds = numpy_util.match(pids, ngids)
    assert np.all(psf['flags'][pinds] == 0)
    ng['psfrec_t'][nginds] = psf['psfrec_T'][pinds]
    ng['psfrec_e_1'][nginds] = psf['psfrec_e'][pinds, 0]
    ng['psfrec_e_2'][nginds] = psf['psfrec_e'][pinds, 1]
    ng = ng[nginds]
    del psf, pids, ngids, pinds, nginds

    # now match to new s2n measures
    ngs2n = fitsio.read(config['ngmix_s2n'])
    s2nids = ngs2n['id']
    ngids = ng['coadd_objects_id']
    s2ninds, nginds = numpy_util.match(s2nids, ngids)
    ng = ng[nginds]
    ngs2n = ngs2n[s2ninds]
    del s2ninds, nginds, s2nids, ngids

    # make sure everything is in the ngmix file
    q, = np.where(d['ngmix_flags'] == 0)
    flagged_ids = set(d['coadd_objects_id'][q])
    ngids = set(ng['coadd_objects_id'])
    assert flagged_ids <= ngids

    # match to total file
    fids = d['coadd_objects_id']
    ngids = ng['coadd_objects_id']
    finds, nginds = numpy_util.match(fids, ngids)
    assert len(finds) == len(d)

    # extra tags
    dlist = ng.dtype.descr
    dlist.append(('exp_log10sb_i', 'f8'))
    dlist.append(('exp_w', 'f8'))
    dlist.append(('exp_e_sens_avg', 'f8'))
    dlist.append(('exp_T_r', 'f8'))
    dlist.append(('exp_s2n_r', 'f8'))
    dlist.append(('exp_T_s2n_r', 'f8'))
    dnew = []
    for dt in dlist:
        if '_t' in dt[0]:
            name = dt[0]
            name = name.replace('_t', '_T')
            dnew.append((name, dt[1]))
        elif dt[0] not in [
                'exp_e_1', 'exp_e_2', 'exp_e_cov_1_1', 'exp_e_cov_1_2',
                'exp_e_cov_2_2'
        ]:
            dnew.append(dt)
    dlist = dnew
    for tag in [
            'exp_e_1', 'exp_e_2', 'exp_e_cov_1_1', 'exp_e_cov_1_2',
            'exp_e_cov_2_1', 'exp_e_cov_2_2'
    ]:
        dlist.append((tag, 'f8'))

    ng_final = np.zeros(len(d), dtype=dlist)
    ng_final['coadd_objects_id'] = d['coadd_objects_id']
    for tag in ng.dtype.names:
        if tag != 'coadd_objects_id':
            if '_t' in tag:
                final_tag = tag.replace('_t', '_T')
            else:
                final_tag = tag
            ng_final[final_tag][finds] = ng[tag][nginds]
        else:
            assert np.array_equal(ng_final[tag][finds], ng[tag][nginds])
    del ng

    ng_final['exp_log10sb_i'][finds] = np.log10(
        np.abs(ng_final['exp_flux_i'][finds] / ng_final['exp_T'][finds]))
    ng_final['exp_e_sens_avg'] = (ng_final['exp_e_sens_1'] +
                                  ng_final['exp_e_sens_2']) / 2.0

    # s2n measures
    for tag in ['exp_T_r', 'exp_s2n_r', 'exp_T_s2n_r']:
        ng_final[tag][finds] = ngs2n[tag][nginds]
    del ngs2n

    # swap e1 signs
    for tag in ['exp_e_1', 'exp_e_cov_1_2', 'psfrec_e_1']:
        ng_final[tag] *= -1.0

    # fianl tag comp that depends on psf sign
    ng_final['exp_e_cov_2_1'] = ng_final['exp_e_cov_1_2']
    ng_final['exp_w'] = 1.0 / (2.0 * 0.22 * 0.22 + ng_final['exp_e_cov_1_1'] +
                               ng_final['exp_e_cov_2_2'])

    # unblind
    for tag in ['exp_e_1', 'exp_e_2']:
        ng_final[tag] = ng_final[tag] / unblind.get_factor()

    fitsio.write(config['flatcats_ngmix'], ng_final, clobber=True)
    del ng_final

    #####################
    # im3shape file
    # find stuff in catalog
    im = fitsio.read(config['im3shape'])

    # make sure everything is in the im3shape file
    q, = np.where(d['im3shape_flags'] == 0)
    flagged_ids = set(d['coadd_objects_id'][q])
    imids = set(im['coadd_objects_id'])
    assert flagged_ids <= imids

    fids = d['coadd_objects_id']
    imids = im['coadd_objects_id']
    finds, iminds = numpy_util.match(fids, imids)

    # fill data
    dlist = im.dtype.descr
    im_final = np.zeros(len(d), dtype=dlist)
    im_final['coadd_objects_id'] = d['coadd_objects_id']
    for tag in im.dtype.names:
        if tag != 'coadd_objects_id':
            im_final[tag][finds] = im[tag][iminds]
        else:
            assert np.array_equal(im_final[tag][finds], im[tag][iminds])

    # clip the weights
    np.clip(im_final['w'], 0.0, 0.24**(-2.0), im_final['w'])

    # unblind
    for tag in ['e1', 'e2']:
        im_final[tag] = im_final[tag] / unblind.get_factor()

    fitsio.write(config['flatcats_im3shape'], im_final, clobber=True)
Example #3
0
def main(config):
    ###############################
    # info file

    # do the flags
    flags = fitsio.read(config['gold_flags'])
    imflags = fitsio.read(config['im3shape_flags'])
    ngflags = fitsio.read(config['ngmix_flags'])    
    if False:
        q, = np.where((flags['sva1_gold_flags'] <= 3) &
                      (flags['sva1_spte_flags'] == 0) &
                      (flags['sva1_gold_mag_flags'] == 0) &
                      ((imflags['im3shape_flags'] == 0) | 
                       (ngflags['ngmix_flags'] == 0)))
    else:
        q, = np.where((flags['sva1_gold_flags'] <= 3) &
                      (flags['sva1_spte_flags'] == 0) &
                      (flags['sva1_gold_mag_flags'] == 0))
    flags = flags[q]
    imflags = imflags[q]
    ngflags = ngflags[q]
    
    # print number of sources
    for code,cflags,tag in zip(['ngmix','im3shape'],[ngflags,imflags],['ngmix_flags','im3shape_flags']):
        q, = np.where((flags['sva1_gold_flags'] == 0) & (cflags[tag] == 0))
        print "%s:" % code,len(q)
    
    # read in the photo-z bins - match
    dtot = []
    dz = []
    if 'photoz_binning_im3shape' in config:
        dim = np.load(config['photoz_binning_im3shape']).astype(int)        
        for i in xrange(len(dim)):
            dtot.append(tuple(dim[i]))
        dim = np.load(config['photoz_binning_im3shape']).astype(float)
        for i in xrange(len(dim)):
            dz.append(tuple(dim[i])[-1])        
        del dim
    if 'photoz_binning_ngmix' in config:
        dng = np.load(config['photoz_binning_ngmix']).astype(int)
        for i in xrange(len(dng)):
            dtot.append(tuple(dng[i]))    
        dng = np.load(config['photoz_binning_ngmix']).astype(float)
        for i in xrange(len(dng)):
            dz.append(tuple(dng[i])[-1])        
        del dng
    if 'photoz_binning_extra' in config:
        de = np.load(config['photoz_binning_extra']).astype(int)
        for i in xrange(len(de)):
            dtot.append(tuple(de[i]))
        de = np.load(config['photoz_binning_extra']).astype(float)
        for i in xrange(len(de)):
            dz.append(tuple(de[i])[-1])        
        del de

    if len(dtot) > 0:
        dtot = np.array(dtot,dtype=[('coadd_objects_id','i8'),('photoz_bin','i4'),('mean_photoz','f8')])    
        dz = np.array(dz,dtype='f8')
        dtot['mean_photoz'][:] = dz[:]
        u, indices = np.unique(dtot['coadd_objects_id'], return_index=True)
        dtot = dtot[indices]
        del u,indices

        # make sure all in file
        fids = set(flags['coadd_objects_id'])
        pzids = set(dtot['coadd_objects_id'])
        if not fids <= pzids:
            print "not all gals got photoz binning indexes!"
            if True:
                fids = flags['coadd_objects_id']
                pzids = dtot['coadd_objects_id']    
                finds,pzinds = numpy_util.match(fids,pzids)
                finds = set(finds)
                tinds = set(np.arange(len(flags)))
                minds = tinds - finds
                minds = np.array(list(minds))
                #print set(flags['coadd_objects_id'][minds])
                print '# of missing gals is',len(minds)
                #assert False

        fids = flags['coadd_objects_id']
        pzids = dtot['coadd_objects_id']    
        finds,pzinds = numpy_util.match(fids,pzids)
    
    # make final catalog
    ftags = ['mag_auto_g','mag_auto_r','mag_auto_i','mag_auto_z',
             'ra','dec','mean_photoz']
    itags = ['coadd_objects_id','sva1_flags','im3shape_flags','ngmix_flags',
             'sva1_gold_flags','sva1_spte_flags','sva1_gold_mag_flags',
             'photoz_bin']
    dlist = []
    for ftag in ftags:
        dlist.append((ftag,'f8'))
    for itag in itags:
        dlist.append((itag,'i8'))
    d = np.zeros(len(flags),dtype=dlist)
    d['photoz_bin'][:] = -99
    d['mean_photoz'][:] = -99.0
    for tag in flags.dtype.names:
        d[tag] = flags[tag]
    for tag in imflags.dtype.names:
        d[tag] = imflags[tag]
    for tag in ngflags.dtype.names:
        d[tag] = ngflags[tag]
    if len(dtot) > 0:
        d['photoz_bin'][finds] = dtot['photoz_bin'][pzinds]    
        d['mean_photoz'][finds] = dtot['mean_photoz'][pzinds]    
            
    q, = np.where(flags['sva1_gold_flags'] & 1)
    d['sva1_flags'][q] |= 1
    
    q, = np.where(flags['sva1_gold_flags'] & 2)
    d['sva1_flags'][q] |= 2

    q, = np.where((flags['sva1_gold_flags'] >> 2) != 0)
    d['sva1_flags'][q] |= 4
    
    q, = np.where(flags['sva1_spte_flags'] != 0)
    d['sva1_flags'][q] |= 8
    
    q, = np.where(flags['sva1_gold_mag_flags'] != 0)
    d['sva1_flags'][q] |= 16
    
    fitsio.write(config['flatcats_info'],d,clobber=True)
    del flags
    del imflags
    del ngflags
    del dtot
    
    ###############################
    # ngmix file
    # first have to match to psf and s2n
    ng = fitsio.read(config['ngmix'])
    psf = fitsio.read(config['ngmix_psfs'])
    pids = psf['id']
    ngids = ng['coadd_objects_id']
    pinds,nginds = numpy_util.match(pids,ngids)
    assert np.all(psf['flags'][pinds] == 0)    
    ng['psfrec_t'][nginds] = psf['psfrec_T'][pinds]
    ng['psfrec_e_1'][nginds] = psf['psfrec_e'][pinds,0]
    ng['psfrec_e_2'][nginds] = psf['psfrec_e'][pinds,1]
    ng = ng[nginds]    
    del psf,pids,ngids,pinds,nginds
    
    # now match to new s2n measures
    ngs2n = fitsio.read(config['ngmix_s2n'])
    s2nids = ngs2n['id']
    ngids = ng['coadd_objects_id']
    s2ninds,nginds = numpy_util.match(s2nids,ngids)
    ng = ng[nginds]
    ngs2n = ngs2n[s2ninds]
    del s2ninds,nginds,s2nids,ngids
    
    # make sure everything is in the ngmix file
    q, = np.where(d['ngmix_flags'] == 0)
    flagged_ids = set(d['coadd_objects_id'][q])
    ngids = set(ng['coadd_objects_id'])
    assert flagged_ids <= ngids    
    
    # match to total file
    fids = d['coadd_objects_id']
    ngids = ng['coadd_objects_id']
    finds,nginds = numpy_util.match(fids,ngids)
    assert len(finds) == len(d)
    
    # extra tags
    dlist = ng.dtype.descr
    dlist.append(('exp_log10sb_i','f8'))
    dlist.append(('exp_w','f8'))
    dlist.append(('exp_e_sens_avg','f8'))
    dlist.append(('exp_T_r','f8'))
    dlist.append(('exp_s2n_r','f8'))
    dlist.append(('exp_T_s2n_r','f8'))
    dnew = []
    for dt in dlist:
        if '_t' in dt[0]:
            name = dt[0]
            name = name.replace('_t','_T')
            dnew.append((name,dt[1]))
        elif dt[0] not in ['exp_e_1','exp_e_2','exp_e_cov_1_1','exp_e_cov_1_2','exp_e_cov_2_2']:
            dnew.append(dt)
    dlist = dnew
    for tag in ['exp_e_1','exp_e_2','exp_e_cov_1_1','exp_e_cov_1_2','exp_e_cov_2_1','exp_e_cov_2_2']:
        dlist.append((tag,'f8'))

    ng_final = np.zeros(len(d),dtype=dlist)
    ng_final['coadd_objects_id'] = d['coadd_objects_id']
    for tag in ng.dtype.names:
        if tag != 'coadd_objects_id':
            if '_t' in tag:
                final_tag = tag.replace('_t','_T')
            else:
                final_tag = tag
            ng_final[final_tag][finds] = ng[tag][nginds]
        else:
            assert np.array_equal(ng_final[tag][finds],ng[tag][nginds])
    del ng
    
    ng_final['exp_log10sb_i'][finds] = np.log10(np.abs(ng_final['exp_flux_i'][finds]/ng_final['exp_T'][finds]))
    ng_final['exp_e_sens_avg'] = (ng_final['exp_e_sens_1'] + ng_final['exp_e_sens_2'])/2.0
    
    # s2n measures
    for tag in ['exp_T_r','exp_s2n_r','exp_T_s2n_r']:
        ng_final[tag][finds] = ngs2n[tag][nginds]
    del ngs2n
    
    # swap e1 signs
    for tag in ['exp_e_1','exp_e_cov_1_2','psfrec_e_1']:
        ng_final[tag] *= -1.0
    
    # fianl tag comp that depends on psf sign
    ng_final['exp_e_cov_2_1'] = ng_final['exp_e_cov_1_2']
    ng_final['exp_w'] = 1.0/(2.0*0.22*0.22 + ng_final['exp_e_cov_1_1'] + ng_final['exp_e_cov_2_2'])
    
    # unblind
    for tag in ['exp_e_1','exp_e_2']:
        ng_final[tag] = ng_final[tag]/unblind.get_factor()
    
    fitsio.write(config['flatcats_ngmix'],ng_final,clobber=True)
    del ng_final
    
    #####################
    # im3shape file
    # find stuff in catalog
    im = fitsio.read(config['im3shape'])
    
    # make sure everything is in the im3shape file
    q, = np.where(d['im3shape_flags'] == 0)
    flagged_ids = set(d['coadd_objects_id'][q])
    imids = set(im['coadd_objects_id'])
    assert flagged_ids <= imids
    
    fids = d['coadd_objects_id']
    imids = im['coadd_objects_id']
    finds,iminds = numpy_util.match(fids,imids)
    
    # fill data
    dlist = im.dtype.descr
    im_final = np.zeros(len(d),dtype=dlist)
    im_final['coadd_objects_id'] = d['coadd_objects_id']
    for tag in im.dtype.names:
        if tag != 'coadd_objects_id':
            im_final[tag][finds] = im[tag][iminds]
        else:
            assert np.array_equal(im_final[tag][finds],im[tag][iminds])
        
    # clip the weights
    np.clip(im_final['w'], 0.0, 0.24**(-2.0), im_final['w'])
            
    # unblind
    for tag in ['e1','e2']:
        im_final[tag] = im_final[tag]/unblind.get_factor()
    
    fitsio.write(config['flatcats_im3shape'],im_final,clobber=True)
Example #4
0
def make_im3shape_flags(config):
    """
    code to make im3shape flags

    People to bug if things are weird
    =================================

    Matthew Becker
    im3shape team
    
    Input Data
    ==========
    
    im3shape v97 - use to set the flags
    
    im3shape_flags
    ==============
    
    bit 2^bit   name              notes
    ----------------------------------------------------------------------
    2   4       NOTORBAD_MEAS     object is not in gold or does not meet required flags below
    
    Required Cuts
    --------------
    Any source which does not meet these cuts is not well measured in im3shape
    
    error_flag == 0
    info_flag == 0
    snr > 15    
    mean_rgpp_rp > 1.2
    finite(psf_e1,psf_e1,nbc_m,nbc_c1,nbc_c2)
    """

    # first read in the data
    gold = fitsio.read(config['gold'],lower=True,columns=['coadd_objects_id'])
    im = fitsio.read(config['im3shape'])
    
    dflag = np.zeros(len(gold),dtype=[('im3shape_flags','i8')])
    
    #####################
    # do im3shape cuts
    tag = 'im3shape_flags'
    
    # find stuff in catalog
    gids = gold['coadd_objects_id']
    imids = im['coadd_objects_id']
    ginds,iminds = numpy_util.match(gids,imids)
    
    # flag stuff not in cat
    sginds = set(ginds)
    totginds = set(range(len(gold)))
    diffginds = totginds - sginds
    q = np.fromiter(diffginds,dtype=int,count=len(diffginds))
    if len(q) > 0:
        dflag[tag][q] |= 4
        
    # required cuts
    q, = np.where(~((im['error_flag'][iminds] == 0) & 
                    (im['info_flag'][iminds] == 0) & 
                    (im['snr'][iminds] > 15.0) &
                    (im['mean_rgpp_rp'][iminds] > 1.2) &
                    (np.isfinite(im['mean_psf_e1_sky'][iminds])) &
                    (np.isfinite(im['mean_psf_e2_sky'][iminds])) & 
                    (np.isfinite(im['nbc_m'][iminds])) &
                    (np.isfinite(im['nbc_c1'][iminds])) &
                    (np.isfinite(im['nbc_c2'][iminds]))
                    ))
    if len(q) > 0:
        dflag[tag][ginds[q]] |= 4
        
    #write to disk
    fitsio.write(config['im3shape_flags'],dflag,clobber=True)
Example #5
0
def make_im3shape_flags(config):
    """
    code to make im3shape flags

    People to bug if things are weird
    =================================

    Matthew Becker
    im3shape team
    
    Input Data
    ==========
    
    im3shape v97 - use to set the flags
    
    im3shape_flags
    ==============
    
    bit 2^bit   name              notes
    ----------------------------------------------------------------------
    2   4       NOTORBAD_MEAS     object is not in gold or does not meet required flags below
    
    Required Cuts
    --------------
    Any source which does not meet these cuts is not well measured in im3shape
    
    error_flag == 0
    info_flag == 0
    snr > 15    
    mean_rgpp_rp > 1.2
    finite(psf_e1,psf_e1,nbc_m,nbc_c1,nbc_c2)
    """

    # first read in the data
    gold = fitsio.read(config['gold'],
                       lower=True,
                       columns=['coadd_objects_id'])
    im = fitsio.read(config['im3shape'])

    dflag = np.zeros(len(gold), dtype=[('im3shape_flags', 'i8')])

    #####################
    # do im3shape cuts
    tag = 'im3shape_flags'

    # find stuff in catalog
    gids = gold['coadd_objects_id']
    imids = im['coadd_objects_id']
    ginds, iminds = numpy_util.match(gids, imids)

    # flag stuff not in cat
    sginds = set(ginds)
    totginds = set(range(len(gold)))
    diffginds = totginds - sginds
    q = np.fromiter(diffginds, dtype=int, count=len(diffginds))
    if len(q) > 0:
        dflag[tag][q] |= 4

    # required cuts
    q, = np.where(~(
        (im['error_flag'][iminds] == 0) & (im['info_flag'][iminds] == 0)
        & (im['snr'][iminds] > 15.0) & (im['mean_rgpp_rp'][iminds] > 1.2)
        & (np.isfinite(im['mean_psf_e1_sky'][iminds]))
        & (np.isfinite(im['mean_psf_e2_sky'][iminds]))
        & (np.isfinite(im['nbc_m'][iminds]))
        & (np.isfinite(im['nbc_c1'][iminds]))
        & (np.isfinite(im['nbc_c2'][iminds]))))
    if len(q) > 0:
        dflag[tag][ginds[q]] |= 4

    #write to disk
    fitsio.write(config['im3shape_flags'], dflag, clobber=True)
Example #6
0
def make_ngmix_flags(config):
    """
    code to make ngmix flags

    People to bug if things are weird
    =================================

    Matthew Becker
    Erin Sheldon    
    Mike Jarvis
    
    Input Data
    ==========
    
    ngmix11 - use to set the flags
    
    See these pages
    
    https://cdcvs.fnal.gov/redmine/projects/deswlwg/wiki/Ngmix011
    https://cdcvs.fnal.gov/redmine/projects/des-sci-verification/wiki/A_Modest_Proposal_for_Preliminary_StarGalaxy_Separation
   
    ngmix_flags
    ===========

    We use three levels of flagging, selecting sets of ngmix 
    sources which have various potential levels of additive and 
    multiplicative biases.
    

    bit 2^bit   name              notes
    ----------------------------------------------------------------------
    2   4       NOTORBAD_MEAS     object is not in ngmix or does not meet required flags belo
    
    Required Cuts
    --------------
    Any source which does not meet these cuts is not well measured in ngmix
    
    flags == 0
    exp_flags == 0
    0.4 < exp_arate < 0.6
    exp_e_sens_1 > 0.0
    exp_e_sens_2 > 0.0
    round_flags == 0
    exp_round_flags == 0
    exp_s2n_r > 15
    exp_T_r/psfrec_T > 0.15 
    
    """        
    
    # first read in the data
    gold = fitsio.read(config['gold'],lower=True,columns=['coadd_objects_id'])
    ng = fitsio.read(config['ngmix'])
    psf = fitsio.read(config['ngmix_psfs'])
    ngs2n = fitsio.read(config['ngmix_s2n'])
    
    # match to psf
    pids = psf['id']
    ngids = ng['coadd_objects_id']
    pinds,nginds = numpy_util.match(pids,ngids)
    assert np.all(psf['flags'][pinds] == 0)    
    ng = ng[nginds]
    psf = psf[pinds]
    del pids,pinds
    
    # match to s/n
    s2nids = ngs2n['id']
    ngids = ng['coadd_objects_id']
    s2ninds,nginds = numpy_util.match(s2nids,ngids)
    ng = ng[nginds]
    ngs2n = ngs2n[s2ninds]
    del s2ninds,s2nids
    
    # now do output, matching to gold
    dflag = np.zeros(len(gold),dtype=[('ngmix_flags','i8')])
    
    tag = 'ngmix_flags'
    gids = gold['coadd_objects_id']
    ngids = ng['coadd_objects_id']
    ginds,nginds = numpy_util.match(gids,ngids)
    
    # flag stuff not in ngmix011
    sginds = set(ginds)
    totginds = set(range(len(gold)))
    diffginds = totginds - sginds
    q = np.fromiter(diffginds,dtype=int,count=len(diffginds))
    if len(q) > 0:
        dflag[tag][q] |= 4
        
    # bad measurements
    q, = np.where(~((ng['flags'][nginds] == 0) &
                    (ng['exp_flags'][nginds] == 0) &
                    (ng['exp_arate'][nginds] > 0.4) &
                    (ng['exp_arate'][nginds] < 0.6) &
                    (ng['exp_e_sens_2'][nginds] > 0.0) &
                    (ng['exp_e_sens_1'][nginds] > 0.0) &
                    (ngs2n['round_flags'][nginds] == 0) & 
                    (ngs2n['exp_round_flags'][nginds] == 0) &
                    (ngs2n['exp_s2n_r'][nginds] > 15.0) & 
                    (ngs2n['exp_T_r'][nginds]/psf['psfrec_T'][nginds] > 0.15)                    
                    ))
    if len(q) > 0:
        dflag[tag][ginds[q]] |= 4
    
    #write to disk
    fitsio.write(config['ngmix_flags'],dflag,clobber=True)
Example #7
0
def make_gold_flags(config):
    """
    code to make gold flags for DES WL analysis

    People to bug if things are weird
    =================================
    
    Matthew Becker
    Eli Rykoff
    Erin Sheldon
    
    Input Data
    ==========
    We use the following sources of data.
    
    1) SVA1 gold v1.0.4 with the additional star masking, 
    bad regions mask, crazy colors, and large g- vs i-band 
    offsets. See this page
    
    https://cdcvs.fnal.gov/redmine/projects/des-sci-verification/wiki/SVA1_Gold_Catalog_v10#Gold-104
    
    2) ngmix11 - used to cut low surface brightness junk, 
    additonal stars and to set the ngmix flags of course
    
    See these pages
    
    https://cdcvs.fnal.gov/redmine/projects/deswlwg/wiki/Ngmix011
    https://cdcvs.fnal.gov/redmine/projects/des-sci-verification/wiki/A_Modest_Proposal_for_Preliminary_StarGalaxy_Separation
    

    Flag File Description
    ======================
    The flag file consists of three columns which are
    
    sva1_gold_flags
    ---------------
    
    These are flags related to SVA1 image processing and galaxy selection. We 
    have attempted to preserve as much information as possible. The field is 
    created by concatenating the SExtractor flags (i-band), SVA1 gold flags, 
    and additional galaxy selection criteria from ngmix11
    
    bit 2^bit   name              notes
    ----------------------------------------------------------------------
    0   1       SX_KINDOF_BLEND    bit 0 from the i-band sexctractor flags
    1   2       SX_DEF_BLEND       bit 1 from the i-band sexctractor flags
    2   4       MODEST_STAR        SVA1 Gold MODEST_CLASS == 2
    3   8       MODEST_JUNK        SVA1 Gold MODEST_CLASS == 0
    4   16      CRAZY_COLORS       SVA1 Gold crazy mag auto colors
    5   32      BADMASK04          in region with lots of JUNK_DPOS objects (removes ~4% of area)
    6   64      NEAR_2MASS         near a 2MASS star
    7   128     JUNK_DPOS          large offset in g and i band windowed positions
    8   256     NOT_IN_NGMIX011    object did not get measured by ngmix011
    9   512     NGMIX011_STAR      object is a star according to ngmix011 modest classifier 
    10  1024    LOW_SB_NGMIX011    object has low surface brightness in ngmix011
    11  2048    BAD_MEAS_NGMIX011  object does not satisfy good measurement cuts in ngmix
    
    Some cuts from ngmix011 are applied to the full gold sample. These are detailed below 
    and are in flag bits 8 through 12.
    
    We use a modest-style star galaxy separation such that objects 
    with 
    
        exp_t + exp_t_err > 0.02
    
    are galaxies. We also cut objects with 

        exp_mag_i + 3.5*log10(exp_flux_i/exp_t) < 28.0

    where

        exp_flux_i = 10.0^((exp_mag_i-30.0)/(-2.5))

    Note that for the ngmix cuts we require that for ngmix

        flags == 0
        exp_flags == 0
        0.4 < exp_arate < 0.6

    so that object has a good measurement at all. 

    sva1_spte_flags
    ---------------
    
    We flag all galaxies with ra,dec not in 

        50 < ra < 100
        dec < -35.0

    as outside of the SPTE region.

    sva1_gold_mag_flags
    -------------------

    We flag all objects that do not have griz mags.
    
    bit 2^bit   name              notes
    ----------------------------------------------------------------------
    0   1       MISSING_GRIZ_MAGS object does not have all griz mags
    
    """

    # first read in the data
    gold = fitsio.read(
        config['gold'],
        lower=True,
        columns=['coadd_objects_id', 'modest_class', 'flags_i', 'ra', 'dec'])
    auto = fitsio.read(config['auto'], lower=True)
    badflag = fitsio.read(config['badflag'], lower=True)
    ng = fitsio.read(config['ngmix'])

    # make output data
    dflag = np.zeros(len(gold),
                     dtype=[('coadd_objects_id', 'i8'), ('ra', 'f8'),
                            ('dec', 'f8'), ('sva1_gold_flags', 'i8'),
                            ('sva1_spte_flags', 'i8'),
                            ('sva1_gold_mag_flags', 'i8'),
                            ('mag_auto_g', 'f8'), ('mag_auto_r', 'f8'),
                            ('mag_auto_i', 'f8'), ('mag_auto_z', 'f8')])
    for tag in ['coadd_objects_id', 'ra', 'dec']:
        dflag[tag] = gold[tag]
    for tag in ['mag_auto_g', 'mag_auto_r', 'mag_auto_i', 'mag_auto_z']:
        dflag[tag] = auto[tag]

    # set sva1_gold_flags
    tag = 'sva1_gold_flags'
    dflag[tag] = gold['flags_i']

    q, = np.where(gold['modest_class'] == 2)
    if len(q) > 0:
        dflag[tag][q] |= 4

    q, = np.where(gold['modest_class'] == 0)
    if len(q) > 0:
        dflag[tag][q] |= 8

    # crazy colors
    # (g-r) < -1, (g-r) > 4, (i-z) < -1, (i-z) > 4
    q, = np.where((auto['mag_auto_g'] - auto['mag_auto_r'] < -1.0)
                  | (auto['mag_auto_g'] - auto['mag_auto_r'] > 4.0)
                  | (auto['mag_auto_i'] - auto['mag_auto_z'] < -1.0)
                  | (auto['mag_auto_i'] - auto['mag_auto_z'] > 4.0))
    if len(q) > 0:
        dflag[tag][q] |= 16

    # badflags masking
    q, = np.where((badflag['badflag'] & 2) != 0)
    if len(q) > 0:
        dflag[tag][q] |= 32

    q, = np.where((badflag['badflag'] & 4) != 0)
    if len(q) > 0:
        dflag[tag][q] |= 64

    q, = np.where((badflag['badflag'] & 8) != 0)
    if len(q) > 0:
        dflag[tag][q] |= 128

    del badflag

    # ngmix cuts
    # first have to match
    gids = gold['coadd_objects_id']
    ngids = ng['coadd_objects_id']
    ginds, nginds = numpy_util.match(gids, ngids)

    # flag stuff not in ngmix011
    sginds = set(ginds)
    totginds = set(range(len(gold)))
    diffginds = totginds - sginds
    q = np.fromiter(diffginds, dtype=int, count=len(diffginds))
    if len(q) > 0:
        dflag[tag][q] |= 256

    # bad measurements
    q, = np.where(~(
        (ng['flags'][nginds] == 0) & (ng['exp_flags'][nginds] == 0)
        & (ng['exp_arate'][nginds] > 0.4) & (ng['exp_arate'][nginds] < 0.6)))
    if len(q) > 0:
        dflag[tag][ginds[q]] |= 2048

    # low SB
    sb = ng['exp_mag_i'] + 3.5 * np.log10(
        np.abs(ng['exp_flux_i'] / ng['exp_t']))
    q, = np.where(~((ng['flags'][nginds] == 0) & (ng['exp_flags'][nginds] == 0)
                    & (ng['exp_arate'][nginds] > 0.4)
                    & (ng['exp_arate'][nginds] < 0.6) & (sb[nginds] > 28.0)))
    if len(q) > 0:
        dflag[tag][ginds[q]] |= 1024

    # ngmix star
    nsig = 1.0
    sgc = ng['exp_t'] + nsig * ng['exp_t_err']
    q, = np.where(~((ng['flags'][nginds] == 0) & (ng['exp_flags'][nginds] == 0)
                    & (ng['exp_arate'][nginds] > 0.4)
                    & (ng['exp_arate'][nginds] < 0.6) & (sgc[nginds] > 0.02)))
    if len(q) > 0:
        dflag[tag][ginds[q]] |= 512

    # non-99 mag_auto in all bands
    tag = 'sva1_gold_mag_flags'
    q, = np.where(~((np.abs(auto['mag_auto_g']) < 99.0)
                    & (np.abs(auto['mag_auto_r']) < 99.0)
                    & (np.abs(auto['mag_auto_i']) < 99.0)
                    & (np.abs(auto['mag_auto_z']) < 99.0)))
    if len(q) > 0:
        dflag[tag][q] |= 1
    del auto

    #set sva1_spte_flags
    q, = np.where(~(
        (gold['ra'] > 50.0) & (gold['ra'] < 100.0) & (gold['dec'] < -35.0)))
    if len(q) > 0:
        dflag['sva1_spte_flags'][q] != 1

    #write to disk
    fitsio.write(config['gold_flags'], dflag, clobber=True)