Exemplo n.º 1
0
def main(config):
    ###############################
    # info file

    # do the flags
    flags = fitsio.read(config['gold_flags'])
    imflags = fitsio.read(config['im3shape_flags'])
    ngflags = fitsio.read(config['ngmix_flags'])
    if False:
        q, = np.where((flags['sva1_gold_flags'] <= 3)
                      & (flags['sva1_spte_flags'] == 0)
                      & (flags['sva1_gold_mag_flags'] == 0)
                      & ((imflags['im3shape_flags'] == 0)
                         | (ngflags['ngmix_flags'] == 0)))
    else:
        q, = np.where((flags['sva1_gold_flags'] <= 3)
                      & (flags['sva1_spte_flags'] == 0)
                      & (flags['sva1_gold_mag_flags'] == 0))
    flags = flags[q]
    imflags = imflags[q]
    ngflags = ngflags[q]

    # print number of sources
    for code, cflags, tag in zip(['ngmix', 'im3shape'], [ngflags, imflags],
                                 ['ngmix_flags', 'im3shape_flags']):
        q, = np.where((flags['sva1_gold_flags'] == 0) & (cflags[tag] == 0))
        print "%s:" % code, len(q)

    # read in the photo-z bins - match
    dtot = []
    dz = []
    if 'photoz_binning_im3shape' in config:
        dim = np.load(config['photoz_binning_im3shape']).astype(int)
        for i in xrange(len(dim)):
            dtot.append(tuple(dim[i]))
        dim = np.load(config['photoz_binning_im3shape']).astype(float)
        for i in xrange(len(dim)):
            dz.append(tuple(dim[i])[-1])
        del dim
    if 'photoz_binning_ngmix' in config:
        dng = np.load(config['photoz_binning_ngmix']).astype(int)
        for i in xrange(len(dng)):
            dtot.append(tuple(dng[i]))
        dng = np.load(config['photoz_binning_ngmix']).astype(float)
        for i in xrange(len(dng)):
            dz.append(tuple(dng[i])[-1])
        del dng
    if 'photoz_binning_extra' in config:
        de = np.load(config['photoz_binning_extra']).astype(int)
        for i in xrange(len(de)):
            dtot.append(tuple(de[i]))
        de = np.load(config['photoz_binning_extra']).astype(float)
        for i in xrange(len(de)):
            dz.append(tuple(de[i])[-1])
        del de

    if len(dtot) > 0:
        dtot = np.array(dtot,
                        dtype=[('coadd_objects_id', 'i8'),
                               ('photoz_bin', 'i4'), ('mean_photoz', 'f8')])
        dz = np.array(dz, dtype='f8')
        dtot['mean_photoz'][:] = dz[:]
        u, indices = np.unique(dtot['coadd_objects_id'], return_index=True)
        dtot = dtot[indices]
        del u, indices

        # make sure all in file
        fids = set(flags['coadd_objects_id'])
        pzids = set(dtot['coadd_objects_id'])
        if not fids <= pzids:
            print "not all gals got photoz binning indexes!"
            if True:
                fids = flags['coadd_objects_id']
                pzids = dtot['coadd_objects_id']
                finds, pzinds = numpy_util.match(fids, pzids)
                finds = set(finds)
                tinds = set(np.arange(len(flags)))
                minds = tinds - finds
                minds = np.array(list(minds))
                #print set(flags['coadd_objects_id'][minds])
                print '# of missing gals is', len(minds)
                #assert False

        fids = flags['coadd_objects_id']
        pzids = dtot['coadd_objects_id']
        finds, pzinds = numpy_util.match(fids, pzids)

    # make final catalog
    ftags = [
        'mag_auto_g', 'mag_auto_r', 'mag_auto_i', 'mag_auto_z', 'ra', 'dec',
        'mean_photoz'
    ]
    itags = [
        'coadd_objects_id', 'sva1_flags', 'im3shape_flags', 'ngmix_flags',
        'sva1_gold_flags', 'sva1_spte_flags', 'sva1_gold_mag_flags',
        'photoz_bin'
    ]
    dlist = []
    for ftag in ftags:
        dlist.append((ftag, 'f8'))
    for itag in itags:
        dlist.append((itag, 'i8'))
    d = np.zeros(len(flags), dtype=dlist)
    d['photoz_bin'][:] = -99
    d['mean_photoz'][:] = -99.0
    for tag in flags.dtype.names:
        d[tag] = flags[tag]
    for tag in imflags.dtype.names:
        d[tag] = imflags[tag]
    for tag in ngflags.dtype.names:
        d[tag] = ngflags[tag]
    if len(dtot) > 0:
        d['photoz_bin'][finds] = dtot['photoz_bin'][pzinds]
        d['mean_photoz'][finds] = dtot['mean_photoz'][pzinds]

    q, = np.where(flags['sva1_gold_flags'] & 1)
    d['sva1_flags'][q] |= 1

    q, = np.where(flags['sva1_gold_flags'] & 2)
    d['sva1_flags'][q] |= 2

    q, = np.where((flags['sva1_gold_flags'] >> 2) != 0)
    d['sva1_flags'][q] |= 4

    q, = np.where(flags['sva1_spte_flags'] != 0)
    d['sva1_flags'][q] |= 8

    q, = np.where(flags['sva1_gold_mag_flags'] != 0)
    d['sva1_flags'][q] |= 16

    fitsio.write(config['flatcats_info'], d, clobber=True)
    del flags
    del imflags
    del ngflags
    del dtot

    ###############################
    # ngmix file
    # first have to match to psf and s2n
    ng = fitsio.read(config['ngmix'])
    psf = fitsio.read(config['ngmix_psfs'])
    pids = psf['id']
    ngids = ng['coadd_objects_id']
    pinds, nginds = numpy_util.match(pids, ngids)
    assert np.all(psf['flags'][pinds] == 0)
    ng['psfrec_t'][nginds] = psf['psfrec_T'][pinds]
    ng['psfrec_e_1'][nginds] = psf['psfrec_e'][pinds, 0]
    ng['psfrec_e_2'][nginds] = psf['psfrec_e'][pinds, 1]
    ng = ng[nginds]
    del psf, pids, ngids, pinds, nginds

    # now match to new s2n measures
    ngs2n = fitsio.read(config['ngmix_s2n'])
    s2nids = ngs2n['id']
    ngids = ng['coadd_objects_id']
    s2ninds, nginds = numpy_util.match(s2nids, ngids)
    ng = ng[nginds]
    ngs2n = ngs2n[s2ninds]
    del s2ninds, nginds, s2nids, ngids

    # make sure everything is in the ngmix file
    q, = np.where(d['ngmix_flags'] == 0)
    flagged_ids = set(d['coadd_objects_id'][q])
    ngids = set(ng['coadd_objects_id'])
    assert flagged_ids <= ngids

    # match to total file
    fids = d['coadd_objects_id']
    ngids = ng['coadd_objects_id']
    finds, nginds = numpy_util.match(fids, ngids)
    assert len(finds) == len(d)

    # extra tags
    dlist = ng.dtype.descr
    dlist.append(('exp_log10sb_i', 'f8'))
    dlist.append(('exp_w', 'f8'))
    dlist.append(('exp_e_sens_avg', 'f8'))
    dlist.append(('exp_T_r', 'f8'))
    dlist.append(('exp_s2n_r', 'f8'))
    dlist.append(('exp_T_s2n_r', 'f8'))
    dnew = []
    for dt in dlist:
        if '_t' in dt[0]:
            name = dt[0]
            name = name.replace('_t', '_T')
            dnew.append((name, dt[1]))
        elif dt[0] not in [
                'exp_e_1', 'exp_e_2', 'exp_e_cov_1_1', 'exp_e_cov_1_2',
                'exp_e_cov_2_2'
        ]:
            dnew.append(dt)
    dlist = dnew
    for tag in [
            'exp_e_1', 'exp_e_2', 'exp_e_cov_1_1', 'exp_e_cov_1_2',
            'exp_e_cov_2_1', 'exp_e_cov_2_2'
    ]:
        dlist.append((tag, 'f8'))

    ng_final = np.zeros(len(d), dtype=dlist)
    ng_final['coadd_objects_id'] = d['coadd_objects_id']
    for tag in ng.dtype.names:
        if tag != 'coadd_objects_id':
            if '_t' in tag:
                final_tag = tag.replace('_t', '_T')
            else:
                final_tag = tag
            ng_final[final_tag][finds] = ng[tag][nginds]
        else:
            assert np.array_equal(ng_final[tag][finds], ng[tag][nginds])
    del ng

    ng_final['exp_log10sb_i'][finds] = np.log10(
        np.abs(ng_final['exp_flux_i'][finds] / ng_final['exp_T'][finds]))
    ng_final['exp_e_sens_avg'] = (ng_final['exp_e_sens_1'] +
                                  ng_final['exp_e_sens_2']) / 2.0

    # s2n measures
    for tag in ['exp_T_r', 'exp_s2n_r', 'exp_T_s2n_r']:
        ng_final[tag][finds] = ngs2n[tag][nginds]
    del ngs2n

    # swap e1 signs
    for tag in ['exp_e_1', 'exp_e_cov_1_2', 'psfrec_e_1']:
        ng_final[tag] *= -1.0

    # fianl tag comp that depends on psf sign
    ng_final['exp_e_cov_2_1'] = ng_final['exp_e_cov_1_2']
    ng_final['exp_w'] = 1.0 / (2.0 * 0.22 * 0.22 + ng_final['exp_e_cov_1_1'] +
                               ng_final['exp_e_cov_2_2'])

    # unblind
    for tag in ['exp_e_1', 'exp_e_2']:
        ng_final[tag] = ng_final[tag] / unblind.get_factor()

    fitsio.write(config['flatcats_ngmix'], ng_final, clobber=True)
    del ng_final

    #####################
    # im3shape file
    # find stuff in catalog
    im = fitsio.read(config['im3shape'])

    # make sure everything is in the im3shape file
    q, = np.where(d['im3shape_flags'] == 0)
    flagged_ids = set(d['coadd_objects_id'][q])
    imids = set(im['coadd_objects_id'])
    assert flagged_ids <= imids

    fids = d['coadd_objects_id']
    imids = im['coadd_objects_id']
    finds, iminds = numpy_util.match(fids, imids)

    # fill data
    dlist = im.dtype.descr
    im_final = np.zeros(len(d), dtype=dlist)
    im_final['coadd_objects_id'] = d['coadd_objects_id']
    for tag in im.dtype.names:
        if tag != 'coadd_objects_id':
            im_final[tag][finds] = im[tag][iminds]
        else:
            assert np.array_equal(im_final[tag][finds], im[tag][iminds])

    # clip the weights
    np.clip(im_final['w'], 0.0, 0.24**(-2.0), im_final['w'])

    # unblind
    for tag in ['e1', 'e2']:
        im_final[tag] = im_final[tag] / unblind.get_factor()

    fitsio.write(config['flatcats_im3shape'], im_final, clobber=True)
Exemplo n.º 2
0
def make_im3shape(config, info):
    """Make the im3shape catalog file
    """

    # Setup the dtype:
    dt = np.dtype([
        ('coadd_objects_id', 'i8'),
        ('e_1', 'f8'),
        ('e_2', 'f8'),
        ('nbc_m', 'f8'),
        ('nbc_c1', 'f8'),
        ('nbc_c2', 'f8'),
        ('w', 'f8'),
        ('error_flag', 'i4'),
        ('info_flag', 'i4'),
        ('snr_w', 'f4'),
        ('snr_r', 'f4'),
        ('flux_r', 'f4'),
        ('radius', 'f4'),
        ('is_bulge', 'i4'),
        ('mean_rgpp_rp', 'f4'),
        ('mean_psf_e1', 'f8'),
        ('mean_psf_e2', 'f8'),
        ('mean_psf_fwhm', 'f4'),
        ('ra_shift', 'f4'),
        ('dec_shift', 'f4'),
        ('chi2', 'f4'),
        ('likelihood', 'f4'),
        ('stamp_size', 'i4'),
        ('n_exposure', 'i4') ])

    # Read the source catalog
    imcat = fitsio.read(config['im3shape'])

    # im3shape objects are sorted, but has some that are not in info catalog.
    im, dim = numpy_util.match(imcat['coadd_objects_id'], info['COADD_OBJECTS_ID'])
    #assert np.all(imcat['coadd_objects_id'] == sorted(imcat['coadd_objects_id']))

    # Set reasonable defaults for float fields:
    data = np.zeros(len(info), dtype=dt)
    for col, t in dt.descr:
        if 'f' in t:
            data[col] = -9999

    # The coadd_objects_id column can match for all rows
    data['coadd_objects_id'] = info['COADD_OBJECTS_ID']

    # Check that the overlap with imcat is correct
    assert np.all(data['coadd_objects_id'][dim] == imcat['coadd_objects_id'][im])

    # Default error_flag is NO_ATTEMPT
    data['error_flag'] = 2**30
    data['info_flag'] = 2**25

    print 'unique error_flags in original catalog = ',np.unique(imcat['error_flag'])
    print 'unique error_flags that match info = ',np.unique(imcat['error_flag'][im])

    # Copy in the columns from the source catalog that keep the same name:
    for col in ['nbc_m', 'nbc_c1', 'nbc_c2', 'w', 'error_flag', 'info_flag',
                'radius', 'mean_rgpp_rp', 'mean_psf_fwhm', 'likelihood',
                'stamp_size', 'n_exposure']:
        data[col][dim] = imcat[col][im]

    print 'unique error_flags in final = ',np.unique(data['error_flag'][dim])

    # Some get a new name:
    data['e_1'][dim] = imcat['e1'][im]
    data['e_2'][dim] = imcat['e2'][im]
    data['snr_w'][dim] = imcat['snr'][im]
    data['snr_r'][dim] = imcat['round_snr'][im]
    data['mean_psf_e1'][dim] = imcat['mean_psf_e1_sky'][im]
    data['mean_psf_e2'][dim] = imcat['mean_psf_e2_sky'][im]
    data['ra_shift'][dim] = imcat['ra_as'][im]
    data['dec_shift'][dim] = imcat['dec_as'][im]
    data['chi2'][dim] = imcat['chi2_pixel'][im]

    # Do a calculation to get the flux from separate bulge/disc fluxes.
    data['is_bulge'][dim] = imcat['bulge_flux'][im] != 0.
    # Only one is non-zero:
    bulge = imcat['bulge_flux'][im] != 0.
    disc = imcat['disc_flux'][im] != 0.
    print 'sum isbulge = ',np.sum(data['is_bulge'][dim])
    print 'sum bulge_flux > 0. = ',np.sum(imcat['bulge_flux'][im] != 0.)
    print 'sum disc_flux > 0. = ',np.sum(imcat['disc_flux'][im] != 0.)
    assert np.all((imcat['bulge_flux'][im] != 0) != (imcat['disc_flux'][im] != 0))
    data['flux_r'][dim] = imcat['mean_flux'][im] * (imcat['bulge_flux'][im]+imcat['disc_flux'][im])

    # clip the weights
    data['w'][dim] = np.clip(data['w'][dim], 0.0, 0.24**(-2.0))

    # unblind
    for tag in ['e_1','e_2']:
        data[tag][dim] /= unblind.get_factor()
    
    # Before writing out, make all the names upper case
    data.dtype.names = tuple([name.upper() for name in data.dtype.names])

    fitsio.write(config['release_im3shape'],data,clobber=True)
Exemplo n.º 3
0
def make_ngmix(config, info):
    """Make the ngmix catalog file
    """

    # Setup the dtype:
    dt = np.dtype([
        ('coadd_objects_id', 'i8'),
        ('e_1', 'f8'),
        ('e_2', 'f8'),
        ('sens_avg', 'f8'),
        ('w', 'f8'),
        ('e_cov_1_1', 'f8'),
        ('e_cov_1_2', 'f8'),
        ('e_cov_2_1', 'f8'),
        ('e_cov_2_2', 'f8'),
        ('error_flag', 'i4'),
        ('snr_w', 'f4'),
        ('snr_r', 'f4'),
        ('flux_i', 'f4'),
        ('mag_i', 'f4'),
        ('t', 'f4'),
        ('t_err', 'f4'),
        ('t_r', 'f4'),
        ('snr_t', 'f4'),
        ('snr_t_r', 'f4'),
        ('log10_sb_i', 'f4'),
        ('mean_psf_e1', 'f8'),
        ('mean_psf_e2', 'f8'),
        ('mean_psf_t', 'f4'),
        ('sens_1', 'f4'),
        ('sens_2', 'f4'),
        ('arate', 'f4'),
        ('stamp_size', 'i4'),
        ('mask_frac', 'f4') ])

    # Read the source catalogs
    ngcat = fitsio.read(config['ngmix'])

    # ngmix catalogs are not sorted, so need to use match function
    # Also, ngmix has some object that are not in info catalog.
    ng, dng = numpy_util.match(ngcat['coadd_objects_id'], info['COADD_OBJECTS_ID'])

    # Set reasonable defaults for float fields:
    data = np.zeros(len(info), dtype=dt)
    for col, t in dt.descr:
        if 'f' in t:
            data[col] = -9999

    # The coadd_objects_id column can match for all rows
    data['coadd_objects_id'] = info['COADD_OBJECTS_ID']

    # Check that the overlap with ngcat is correct
    assert np.all(data['coadd_objects_id'][dng] == ngcat['coadd_objects_id'][ng])

    # Copy in the columns from the source catalog that keep the same name:
    for col in ['mask_frac']:
        data[col][dng] = ngcat[col][ng]

    # Some columns just need to drop the 'exp_' prefix:
    for col in ['e_1', 'e_2', 'flux_i', 'mag_i', 'arate', 't', 't_err',
                'e_cov_1_1', 'e_cov_1_2', 'e_cov_2_2']:
        data[col][dng] = ngcat['exp_' + col][ng]

    # Some need to be renamed:
    data['snr_w'][dng] = ngcat['exp_s2n_w'][ng]
    data['snr_t'][dng] = ngcat['exp_t_s2n'][ng]
    data['sens_1'][dng] = ngcat['exp_e_sens_1'][ng]
    data['sens_2'][dng] = ngcat['exp_e_sens_2'][ng]
    data['stamp_size'][dng] = ngcat['box_size'][ng]

    # Combine the flags we have so far:
    print 'flags range from %d to %d'%(
        np.min(ngcat['flags'][ngcat['flags']>0]),
        np.max(ngcat['flags']))
    print 'exp_flags range from %d to %d'%(
        np.min(ngcat['exp_flags'][ngcat['exp_flags']>0]),
        np.max(ngcat['exp_flags']))
    data['error_flag'] = 2**30
    data['error_flag'][dng] = ngcat['flags'][ng]
    data['error_flag'][dng] |= ngcat['exp_flags'][ng]

    # Calculate mean sensitivity
    data['sens_avg'][dng] = (ngcat['exp_e_sens_1'][ng] + ngcat['exp_e_sens_2'][ng]) / 2.

    # Calculate the recommended weight.
    data['w'][dng] = 1.0/(2.0*0.22*0.22 + ngcat['exp_e_cov_1_1'][ng] + ngcat['exp_e_cov_2_2'][ng])

    # Calculate log10(sb)
    data['log10_sb_i'][dng] = np.log10(np.abs(ngcat['exp_flux_i'][ng]/ngcat['exp_t'][ng]))

    # swap e1 signs
    for tag in ['e_1', 'e_cov_1_2', 'e_cov_2_1']:
        data[tag][dng] *= -1.0
    data['e_cov_2_1'][dng] = data['e_cov_1_2'][dng]

    # unblind
    for tag in ['e_1','e_2']:
        data[tag][dng] /= unblind.get_factor()

    # Bring in the round columns from the s2n catalog
    scat = fitsio.read(config['ngmix_s2n'])
    sc, dsc = numpy_util.match(scat['id'], info['COADD_OBJECTS_ID'])
    data['snr_r'][dsc] = scat['exp_s2n_r'][sc]
    data['t_r'][dsc] = scat['exp_T_r'][sc]
    data['snr_t_r'][dsc] = scat['exp_T_s2n_r'][sc]

    # Check that round_flags are consisten with current error_flag
    print 'round_flags range from %d to %d'%(
        np.min(scat['round_flags'][scat['round_flags']>0]),
        np.max(scat['round_flags']))
    print 'exp_round_flags range from %d to %d'%(
        np.min(scat['exp_round_flags'][scat['exp_round_flags']>0]),
        np.max(scat['exp_round_flags'][scat['exp_round_flags']<2**30]))
    assert np.all(data['error_flag'][dsc][ scat['round_flags'][sc] > 0 ] > 0)
    assert np.all(data['error_flag'][dsc][ scat['exp_round_flags'][sc] == 2**30 ] > 0)

    # Combine the round flags into the error_flag column
    data['error_flag'][dsc] |= scat['round_flags'][sc]
    data['error_flag'][dsc] |= scat['exp_round_flags'][sc]

    # Bring in the mean psf information from the psf catalog
    pcat = fitsio.read(config['ngmix_psfs'])
    pc, dpc = numpy_util.match(pcat['id'], info['COADD_OBJECTS_ID'])
    data['mean_psf_e1'][dpc] = pcat['psfrec_e'][pc,0]
    data['mean_psf_e2'][dpc] = pcat['psfrec_e'][pc,1]
    data['mean_psf_t'][dpc] = pcat['psfrec_T'][pc]

    # swap e1 signs
    for tag in ['mean_psf_e1']:
        data[tag][dng] *= -1.0

    # Before writing out, make all the names upper case
    data.dtype.names = tuple([name.upper() for name in data.dtype.names])

    fitsio.write(config['release_ngmix'],data,clobber=True)
Exemplo n.º 4
0
def make_im3shape(config, info):
    """Make the im3shape catalog file
    """

    # Setup the dtype:
    dt = np.dtype([('coadd_objects_id', 'i8'), ('e_1', 'f8'), ('e_2', 'f8'),
                   ('nbc_m', 'f8'), ('nbc_c1', 'f8'), ('nbc_c2', 'f8'),
                   ('w', 'f8'), ('error_flag', 'i4'), ('info_flag', 'i4'),
                   ('snr_w', 'f4'), ('snr_r', 'f4'), ('flux_r', 'f4'),
                   ('radius', 'f4'), ('is_bulge', 'i4'),
                   ('mean_rgpp_rp', 'f4'), ('mean_psf_e1', 'f8'),
                   ('mean_psf_e2', 'f8'), ('mean_psf_fwhm', 'f4'),
                   ('ra_shift', 'f4'), ('dec_shift', 'f4'), ('chi2', 'f4'),
                   ('likelihood', 'f4'), ('stamp_size', 'i4'),
                   ('n_exposure', 'i4')])

    # Read the source catalog
    imcat = fitsio.read(config['im3shape'])

    # im3shape objects are sorted, but has some that are not in info catalog.
    im, dim = numpy_util.match(imcat['coadd_objects_id'],
                               info['COADD_OBJECTS_ID'])
    #assert np.all(imcat['coadd_objects_id'] == sorted(imcat['coadd_objects_id']))

    # Set reasonable defaults for float fields:
    data = np.zeros(len(info), dtype=dt)
    for col, t in dt.descr:
        if 'f' in t:
            data[col] = -9999

    # The coadd_objects_id column can match for all rows
    data['coadd_objects_id'] = info['COADD_OBJECTS_ID']

    # Check that the overlap with imcat is correct
    assert np.all(
        data['coadd_objects_id'][dim] == imcat['coadd_objects_id'][im])

    # Default error_flag is NO_ATTEMPT
    data['error_flag'] = 2**30
    data['info_flag'] = 2**25

    print 'unique error_flags in original catalog = ', np.unique(
        imcat['error_flag'])
    print 'unique error_flags that match info = ', np.unique(
        imcat['error_flag'][im])

    # Copy in the columns from the source catalog that keep the same name:
    for col in [
            'nbc_m', 'nbc_c1', 'nbc_c2', 'w', 'error_flag', 'info_flag',
            'radius', 'mean_rgpp_rp', 'mean_psf_fwhm', 'likelihood',
            'stamp_size', 'n_exposure'
    ]:
        data[col][dim] = imcat[col][im]

    print 'unique error_flags in final = ', np.unique(data['error_flag'][dim])

    # Some get a new name:
    data['e_1'][dim] = imcat['e1'][im]
    data['e_2'][dim] = imcat['e2'][im]
    data['snr_w'][dim] = imcat['snr'][im]
    data['snr_r'][dim] = imcat['round_snr'][im]
    data['mean_psf_e1'][dim] = imcat['mean_psf_e1_sky'][im]
    data['mean_psf_e2'][dim] = imcat['mean_psf_e2_sky'][im]
    data['ra_shift'][dim] = imcat['ra_as'][im]
    data['dec_shift'][dim] = imcat['dec_as'][im]
    data['chi2'][dim] = imcat['chi2_pixel'][im]

    # Do a calculation to get the flux from separate bulge/disc fluxes.
    data['is_bulge'][dim] = imcat['bulge_flux'][im] != 0.
    # Only one is non-zero:
    bulge = imcat['bulge_flux'][im] != 0.
    disc = imcat['disc_flux'][im] != 0.
    print 'sum isbulge = ', np.sum(data['is_bulge'][dim])
    print 'sum bulge_flux > 0. = ', np.sum(imcat['bulge_flux'][im] != 0.)
    print 'sum disc_flux > 0. = ', np.sum(imcat['disc_flux'][im] != 0.)
    assert np.all(
        (imcat['bulge_flux'][im] != 0) != (imcat['disc_flux'][im] != 0))
    data['flux_r'][dim] = imcat['mean_flux'][im] * (imcat['bulge_flux'][im] +
                                                    imcat['disc_flux'][im])

    # clip the weights
    data['w'][dim] = np.clip(data['w'][dim], 0.0, 0.24**(-2.0))

    # unblind
    for tag in ['e_1', 'e_2']:
        data[tag][dim] /= unblind.get_factor()

    # Before writing out, make all the names upper case
    data.dtype.names = tuple([name.upper() for name in data.dtype.names])

    fitsio.write(config['release_im3shape'], data, clobber=True)
Exemplo n.º 5
0
def make_ngmix(config, info):
    """Make the ngmix catalog file
    """

    # Setup the dtype:
    dt = np.dtype([('coadd_objects_id', 'i8'), ('e_1', 'f8'), ('e_2', 'f8'),
                   ('sens_avg', 'f8'), ('w', 'f8'), ('e_cov_1_1', 'f8'),
                   ('e_cov_1_2', 'f8'),
                   ('e_cov_2_1', 'f8'), ('e_cov_2_2', 'f8'),
                   ('error_flag', 'i4'), ('snr_w', 'f4'), ('snr_r', 'f4'),
                   ('flux_i', 'f4'), ('mag_i', 'f4'), ('t', 'f4'),
                   ('t_err', 'f4'), ('t_r', 'f4'), ('snr_t', 'f4'),
                   ('snr_t_r', 'f4'), ('log10_sb_i', 'f4'),
                   ('mean_psf_e1', 'f8'), ('mean_psf_e2', 'f8'),
                   ('mean_psf_t', 'f4'), ('sens_1', 'f4'), ('sens_2', 'f4'),
                   ('arate', 'f4'), ('stamp_size', 'i4'), ('mask_frac', 'f4')])

    # Read the source catalogs
    ngcat = fitsio.read(config['ngmix'])

    # ngmix catalogs are not sorted, so need to use match function
    # Also, ngmix has some object that are not in info catalog.
    ng, dng = numpy_util.match(ngcat['coadd_objects_id'],
                               info['COADD_OBJECTS_ID'])

    # Set reasonable defaults for float fields:
    data = np.zeros(len(info), dtype=dt)
    for col, t in dt.descr:
        if 'f' in t:
            data[col] = -9999

    # The coadd_objects_id column can match for all rows
    data['coadd_objects_id'] = info['COADD_OBJECTS_ID']

    # Check that the overlap with ngcat is correct
    assert np.all(
        data['coadd_objects_id'][dng] == ngcat['coadd_objects_id'][ng])

    # Copy in the columns from the source catalog that keep the same name:
    for col in ['mask_frac']:
        data[col][dng] = ngcat[col][ng]

    # Some columns just need to drop the 'exp_' prefix:
    for col in [
            'e_1', 'e_2', 'flux_i', 'mag_i', 'arate', 't', 't_err',
            'e_cov_1_1', 'e_cov_1_2', 'e_cov_2_2'
    ]:
        data[col][dng] = ngcat['exp_' + col][ng]

    # Some need to be renamed:
    data['snr_w'][dng] = ngcat['exp_s2n_w'][ng]
    data['snr_t'][dng] = ngcat['exp_t_s2n'][ng]
    data['sens_1'][dng] = ngcat['exp_e_sens_1'][ng]
    data['sens_2'][dng] = ngcat['exp_e_sens_2'][ng]
    data['stamp_size'][dng] = ngcat['box_size'][ng]

    # Combine the flags we have so far:
    print 'flags range from %d to %d' % (np.min(
        ngcat['flags'][ngcat['flags'] > 0]), np.max(ngcat['flags']))
    print 'exp_flags range from %d to %d' % (np.min(
        ngcat['exp_flags'][ngcat['exp_flags'] > 0]), np.max(
            ngcat['exp_flags']))
    data['error_flag'] = 2**30
    data['error_flag'][dng] = ngcat['flags'][ng]
    data['error_flag'][dng] |= ngcat['exp_flags'][ng]

    # Calculate mean sensitivity
    data['sens_avg'][dng] = (ngcat['exp_e_sens_1'][ng] +
                             ngcat['exp_e_sens_2'][ng]) / 2.

    # Calculate the recommended weight.
    data['w'][dng] = 1.0 / (2.0 * 0.22 * 0.22 + ngcat['exp_e_cov_1_1'][ng] +
                            ngcat['exp_e_cov_2_2'][ng])

    # Calculate log10(sb)
    data['log10_sb_i'][dng] = np.log10(
        np.abs(ngcat['exp_flux_i'][ng] / ngcat['exp_t'][ng]))

    # swap e1 signs
    for tag in ['e_1', 'e_cov_1_2', 'e_cov_2_1']:
        data[tag][dng] *= -1.0
    data['e_cov_2_1'][dng] = data['e_cov_1_2'][dng]

    # unblind
    for tag in ['e_1', 'e_2']:
        data[tag][dng] /= unblind.get_factor()

    # Bring in the round columns from the s2n catalog
    scat = fitsio.read(config['ngmix_s2n'])
    sc, dsc = numpy_util.match(scat['id'], info['COADD_OBJECTS_ID'])
    data['snr_r'][dsc] = scat['exp_s2n_r'][sc]
    data['t_r'][dsc] = scat['exp_T_r'][sc]
    data['snr_t_r'][dsc] = scat['exp_T_s2n_r'][sc]

    # Check that round_flags are consisten with current error_flag
    print 'round_flags range from %d to %d' % (np.min(
        scat['round_flags'][scat['round_flags'] > 0]),
                                               np.max(scat['round_flags']))
    print 'exp_round_flags range from %d to %d' % (
        np.min(scat['exp_round_flags'][scat['exp_round_flags'] > 0]),
        np.max(scat['exp_round_flags'][scat['exp_round_flags'] < 2**30]))
    assert np.all(data['error_flag'][dsc][scat['round_flags'][sc] > 0] > 0)
    assert np.all(
        data['error_flag'][dsc][scat['exp_round_flags'][sc] == 2**30] > 0)

    # Combine the round flags into the error_flag column
    data['error_flag'][dsc] |= scat['round_flags'][sc]
    data['error_flag'][dsc] |= scat['exp_round_flags'][sc]

    # Bring in the mean psf information from the psf catalog
    pcat = fitsio.read(config['ngmix_psfs'])
    pc, dpc = numpy_util.match(pcat['id'], info['COADD_OBJECTS_ID'])
    data['mean_psf_e1'][dpc] = pcat['psfrec_e'][pc, 0]
    data['mean_psf_e2'][dpc] = pcat['psfrec_e'][pc, 1]
    data['mean_psf_t'][dpc] = pcat['psfrec_T'][pc]

    # swap e1 signs
    for tag in ['mean_psf_e1']:
        data[tag][dng] *= -1.0

    # Before writing out, make all the names upper case
    data.dtype.names = tuple([name.upper() for name in data.dtype.names])

    fitsio.write(config['release_ngmix'], data, clobber=True)
Exemplo n.º 6
0
def main(config):
    ###############################
    # info file

    # do the flags
    flags = fitsio.read(config['gold_flags'])
    imflags = fitsio.read(config['im3shape_flags'])
    ngflags = fitsio.read(config['ngmix_flags'])    
    if False:
        q, = np.where((flags['sva1_gold_flags'] <= 3) &
                      (flags['sva1_spte_flags'] == 0) &
                      (flags['sva1_gold_mag_flags'] == 0) &
                      ((imflags['im3shape_flags'] == 0) | 
                       (ngflags['ngmix_flags'] == 0)))
    else:
        q, = np.where((flags['sva1_gold_flags'] <= 3) &
                      (flags['sva1_spte_flags'] == 0) &
                      (flags['sva1_gold_mag_flags'] == 0))
    flags = flags[q]
    imflags = imflags[q]
    ngflags = ngflags[q]
    
    # print number of sources
    for code,cflags,tag in zip(['ngmix','im3shape'],[ngflags,imflags],['ngmix_flags','im3shape_flags']):
        q, = np.where((flags['sva1_gold_flags'] == 0) & (cflags[tag] == 0))
        print "%s:" % code,len(q)
    
    # read in the photo-z bins - match
    dtot = []
    dz = []
    if 'photoz_binning_im3shape' in config:
        dim = np.load(config['photoz_binning_im3shape']).astype(int)        
        for i in xrange(len(dim)):
            dtot.append(tuple(dim[i]))
        dim = np.load(config['photoz_binning_im3shape']).astype(float)
        for i in xrange(len(dim)):
            dz.append(tuple(dim[i])[-1])        
        del dim
    if 'photoz_binning_ngmix' in config:
        dng = np.load(config['photoz_binning_ngmix']).astype(int)
        for i in xrange(len(dng)):
            dtot.append(tuple(dng[i]))    
        dng = np.load(config['photoz_binning_ngmix']).astype(float)
        for i in xrange(len(dng)):
            dz.append(tuple(dng[i])[-1])        
        del dng
    if 'photoz_binning_extra' in config:
        de = np.load(config['photoz_binning_extra']).astype(int)
        for i in xrange(len(de)):
            dtot.append(tuple(de[i]))
        de = np.load(config['photoz_binning_extra']).astype(float)
        for i in xrange(len(de)):
            dz.append(tuple(de[i])[-1])        
        del de

    if len(dtot) > 0:
        dtot = np.array(dtot,dtype=[('coadd_objects_id','i8'),('photoz_bin','i4'),('mean_photoz','f8')])    
        dz = np.array(dz,dtype='f8')
        dtot['mean_photoz'][:] = dz[:]
        u, indices = np.unique(dtot['coadd_objects_id'], return_index=True)
        dtot = dtot[indices]
        del u,indices

        # make sure all in file
        fids = set(flags['coadd_objects_id'])
        pzids = set(dtot['coadd_objects_id'])
        if not fids <= pzids:
            print "not all gals got photoz binning indexes!"
            if True:
                fids = flags['coadd_objects_id']
                pzids = dtot['coadd_objects_id']    
                finds,pzinds = numpy_util.match(fids,pzids)
                finds = set(finds)
                tinds = set(np.arange(len(flags)))
                minds = tinds - finds
                minds = np.array(list(minds))
                #print set(flags['coadd_objects_id'][minds])
                print '# of missing gals is',len(minds)
                #assert False

        fids = flags['coadd_objects_id']
        pzids = dtot['coadd_objects_id']    
        finds,pzinds = numpy_util.match(fids,pzids)
    
    # make final catalog
    ftags = ['mag_auto_g','mag_auto_r','mag_auto_i','mag_auto_z',
             'ra','dec','mean_photoz']
    itags = ['coadd_objects_id','sva1_flags','im3shape_flags','ngmix_flags',
             'sva1_gold_flags','sva1_spte_flags','sva1_gold_mag_flags',
             'photoz_bin']
    dlist = []
    for ftag in ftags:
        dlist.append((ftag,'f8'))
    for itag in itags:
        dlist.append((itag,'i8'))
    d = np.zeros(len(flags),dtype=dlist)
    d['photoz_bin'][:] = -99
    d['mean_photoz'][:] = -99.0
    for tag in flags.dtype.names:
        d[tag] = flags[tag]
    for tag in imflags.dtype.names:
        d[tag] = imflags[tag]
    for tag in ngflags.dtype.names:
        d[tag] = ngflags[tag]
    if len(dtot) > 0:
        d['photoz_bin'][finds] = dtot['photoz_bin'][pzinds]    
        d['mean_photoz'][finds] = dtot['mean_photoz'][pzinds]    
            
    q, = np.where(flags['sva1_gold_flags'] & 1)
    d['sva1_flags'][q] |= 1
    
    q, = np.where(flags['sva1_gold_flags'] & 2)
    d['sva1_flags'][q] |= 2

    q, = np.where((flags['sva1_gold_flags'] >> 2) != 0)
    d['sva1_flags'][q] |= 4
    
    q, = np.where(flags['sva1_spte_flags'] != 0)
    d['sva1_flags'][q] |= 8
    
    q, = np.where(flags['sva1_gold_mag_flags'] != 0)
    d['sva1_flags'][q] |= 16
    
    fitsio.write(config['flatcats_info'],d,clobber=True)
    del flags
    del imflags
    del ngflags
    del dtot
    
    ###############################
    # ngmix file
    # first have to match to psf and s2n
    ng = fitsio.read(config['ngmix'])
    psf = fitsio.read(config['ngmix_psfs'])
    pids = psf['id']
    ngids = ng['coadd_objects_id']
    pinds,nginds = numpy_util.match(pids,ngids)
    assert np.all(psf['flags'][pinds] == 0)    
    ng['psfrec_t'][nginds] = psf['psfrec_T'][pinds]
    ng['psfrec_e_1'][nginds] = psf['psfrec_e'][pinds,0]
    ng['psfrec_e_2'][nginds] = psf['psfrec_e'][pinds,1]
    ng = ng[nginds]    
    del psf,pids,ngids,pinds,nginds
    
    # now match to new s2n measures
    ngs2n = fitsio.read(config['ngmix_s2n'])
    s2nids = ngs2n['id']
    ngids = ng['coadd_objects_id']
    s2ninds,nginds = numpy_util.match(s2nids,ngids)
    ng = ng[nginds]
    ngs2n = ngs2n[s2ninds]
    del s2ninds,nginds,s2nids,ngids
    
    # make sure everything is in the ngmix file
    q, = np.where(d['ngmix_flags'] == 0)
    flagged_ids = set(d['coadd_objects_id'][q])
    ngids = set(ng['coadd_objects_id'])
    assert flagged_ids <= ngids    
    
    # match to total file
    fids = d['coadd_objects_id']
    ngids = ng['coadd_objects_id']
    finds,nginds = numpy_util.match(fids,ngids)
    assert len(finds) == len(d)
    
    # extra tags
    dlist = ng.dtype.descr
    dlist.append(('exp_log10sb_i','f8'))
    dlist.append(('exp_w','f8'))
    dlist.append(('exp_e_sens_avg','f8'))
    dlist.append(('exp_T_r','f8'))
    dlist.append(('exp_s2n_r','f8'))
    dlist.append(('exp_T_s2n_r','f8'))
    dnew = []
    for dt in dlist:
        if '_t' in dt[0]:
            name = dt[0]
            name = name.replace('_t','_T')
            dnew.append((name,dt[1]))
        elif dt[0] not in ['exp_e_1','exp_e_2','exp_e_cov_1_1','exp_e_cov_1_2','exp_e_cov_2_2']:
            dnew.append(dt)
    dlist = dnew
    for tag in ['exp_e_1','exp_e_2','exp_e_cov_1_1','exp_e_cov_1_2','exp_e_cov_2_1','exp_e_cov_2_2']:
        dlist.append((tag,'f8'))

    ng_final = np.zeros(len(d),dtype=dlist)
    ng_final['coadd_objects_id'] = d['coadd_objects_id']
    for tag in ng.dtype.names:
        if tag != 'coadd_objects_id':
            if '_t' in tag:
                final_tag = tag.replace('_t','_T')
            else:
                final_tag = tag
            ng_final[final_tag][finds] = ng[tag][nginds]
        else:
            assert np.array_equal(ng_final[tag][finds],ng[tag][nginds])
    del ng
    
    ng_final['exp_log10sb_i'][finds] = np.log10(np.abs(ng_final['exp_flux_i'][finds]/ng_final['exp_T'][finds]))
    ng_final['exp_e_sens_avg'] = (ng_final['exp_e_sens_1'] + ng_final['exp_e_sens_2'])/2.0
    
    # s2n measures
    for tag in ['exp_T_r','exp_s2n_r','exp_T_s2n_r']:
        ng_final[tag][finds] = ngs2n[tag][nginds]
    del ngs2n
    
    # swap e1 signs
    for tag in ['exp_e_1','exp_e_cov_1_2','psfrec_e_1']:
        ng_final[tag] *= -1.0
    
    # fianl tag comp that depends on psf sign
    ng_final['exp_e_cov_2_1'] = ng_final['exp_e_cov_1_2']
    ng_final['exp_w'] = 1.0/(2.0*0.22*0.22 + ng_final['exp_e_cov_1_1'] + ng_final['exp_e_cov_2_2'])
    
    # unblind
    for tag in ['exp_e_1','exp_e_2']:
        ng_final[tag] = ng_final[tag]/unblind.get_factor()
    
    fitsio.write(config['flatcats_ngmix'],ng_final,clobber=True)
    del ng_final
    
    #####################
    # im3shape file
    # find stuff in catalog
    im = fitsio.read(config['im3shape'])
    
    # make sure everything is in the im3shape file
    q, = np.where(d['im3shape_flags'] == 0)
    flagged_ids = set(d['coadd_objects_id'][q])
    imids = set(im['coadd_objects_id'])
    assert flagged_ids <= imids
    
    fids = d['coadd_objects_id']
    imids = im['coadd_objects_id']
    finds,iminds = numpy_util.match(fids,imids)
    
    # fill data
    dlist = im.dtype.descr
    im_final = np.zeros(len(d),dtype=dlist)
    im_final['coadd_objects_id'] = d['coadd_objects_id']
    for tag in im.dtype.names:
        if tag != 'coadd_objects_id':
            im_final[tag][finds] = im[tag][iminds]
        else:
            assert np.array_equal(im_final[tag][finds],im[tag][iminds])
        
    # clip the weights
    np.clip(im_final['w'], 0.0, 0.24**(-2.0), im_final['w'])
            
    # unblind
    for tag in ['e1','e2']:
        im_final[tag] = im_final[tag]/unblind.get_factor()
    
    fitsio.write(config['flatcats_im3shape'],im_final,clobber=True)