예제 #1
0
def getbrickfiles(brickname=None):

    survey = LegacySurveyData()
    brickinfo = survey.get_brick_by_name(brickname)
    brickwcs = wcs_for_brick(brickinfo)
    ccdinfo = survey.ccds_touching_wcs(brickwcs)
    nccd = len(ccdinfo)

    calibdir = survey.get_calib_dir()
    imagedir = survey.survey_dir

    # Construct image file names and the calibration file names.
    expnum = ccdinfo.expnum
    ccdname = ccdinfo.ccdname

    psffiles = list()
    skyfiles = list()
    imagefiles = list()
    for ccd in ccdinfo:
        info = survey.get_image_object(ccd)
        for attr in ['imgfn', 'dqfn', 'wtfn']:
            fn = getattr(info, attr).replace(imagedir+'/', '')
            #if '160108_073601' in fn:
            #    pdb.set_trace()
            imagefiles.append(fn)
        psffiles.append(info.psffn.replace(calibdir, 'calib'))
        skyfiles.append(info.splineskyfn.replace(calibdir, 'calib'))
        
    #for ii in range(nccd):
        #exp = '{0:08d}'.format(expnum[ii])
        #rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits')
        #psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile))
        #skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile))
        #imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii]))))

    #print(np.array(imagefiles))
    #print(np.array(psffiles))
    #print(np.array(skyfiles))
    return imagefiles, psffiles, skyfiles
예제 #2
0
def getbrickfiles(brickname=None):

    survey = LegacySurveyData()
    brickinfo = survey.get_brick_by_name(brickname)
    brickwcs = wcs_for_brick(brickinfo)
    ccdinfo = survey.ccds_touching_wcs(brickwcs)
    nccd = len(ccdinfo)

    calibdir = survey.get_calib_dir()
    imagedir = survey.survey_dir

    # Construct image file names and the calibration file names.
    expnum = ccdinfo.expnum
    ccdname = ccdinfo.ccdname

    psffiles = list()
    skyfiles = list()
    imagefiles = list()
    for ccd in ccdinfo:
        info = survey.get_image_object(ccd)
        for attr in ['imgfn', 'dqfn', 'wtfn']:
            fn = getattr(info, attr).replace(imagedir + '/', '')
            #if '160108_073601' in fn:
            #    pdb.set_trace()
            imagefiles.append(fn)
        psffiles.append(info.psffn.replace(calibdir, 'calib'))
        skyfiles.append(info.splineskyfn.replace(calibdir, 'calib'))

    #for ii in range(nccd):
    #exp = '{0:08d}'.format(expnum[ii])
    #rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits')
    #psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile))
    #skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile))
    #imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii]))))

    #print(np.array(imagefiles))
    #print(np.array(psffiles))
    #print(np.array(skyfiles))
    return imagefiles, psffiles, skyfiles
예제 #3
0
파일: ps1cat.py 프로젝트: findaz/legacypipe
    def __init__(
        self,
        expnum=None,
        ccdname=None,
        ccdwcs=None,
        pattern='/project/projectdirs/cosmo/work/ps1/cats/chunks-qz-star-v3/ps1-%(hp)05d.fits'
    ):
        """Read PS1 or gaia sources for an exposure number + CCD name or CCD WCS

        Args:
            expnum, ccdname: select catalogue with these
            ccdwcs: or select catalogue with this
            pattern: absolute path and wildcard for PS1 or Gaia catalogues
                dr: /project/projectdirs/cosmo/work/
                PS1: ${dr}/ps1/cats/chunks-qz-star-v3/ps1-%(hp)05d.fits
                PS1-Gaia: ${dr}/gaia/chunks-ps1-gaia/chunk-%(hp)05d.fits
        """
        assert ('ps1' in pattern or 'gaia' in pattern)
        #assert(ps1_or_gaia in ['ps1','ps1_gaia'])
        #if ps1_or_gaia == 'ps1':
        #  # PS1 "qz" directory
        #  # e.g. /project/projectdirs/cosmo/work/ps1/cats/chunks-qz-star-v2
        #  self.catdir= os.getenv('PS1CAT_DIR')
        #elif ps1_or_gaia == 'ps1_gaia':
        #  # PS1-Gaia "qz" matches-only directory
        #  # e.g. /project/projectdirs/cosmo/work/gaia/chunks-ps1-gaia
        #  self.catdir= os.getenv('PS1_GAIA_MATCHES')
        #fnpattern = os.path.join(self.catdir, prefix + '-%(hp)05d.fits')
        super(ps1cat, self).__init__(pattern)

        if ccdwcs is None:
            from legacypipe.survey import LegacySurveyData
            survey = LegacySurveyData()
            ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
            im = survey.get_image_object(ccd)
            self.ccdwcs = im.get_wcs()
        else:
            self.ccdwcs = ccdwcs
예제 #4
0
    def __init__(self, expnum=None, ccdname=None, ccdwcs=None):
        """Initialize the class with either the exposure number *and* CCD name, or
        directly with the WCS of the CCD of interest.

        """
        # GAIA and PS1 info, gaia for astrometry, ps1 for photometry
        self.gaiadir = os.getenv('GAIACAT_DIR')
        # PS1 only
        self.ps1dir = os.getenv('PS1CAT_DIR')  # PS1 only
        if self.ps1dir is None:
            raise ValueError('Need PS1CAT_DIR environment variable to be set.')
        if self.gaiadir is None:
            print(
                'WARNING: GAIACAT_DIR environment variable not set: using Pan-STARRS1 for astrometry'
            )
        self.nside = 32
        if ccdwcs is None:
            from legacypipe.survey import LegacySurveyData
            survey = LegacySurveyData()
            ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
            im = survey.get_image_object(ccd)
            self.ccdwcs = im.get_wcs()
        else:
            self.ccdwcs = ccdwcs
예제 #5
0
    def __init__(self, expnum=None, ccdname=None, ccdwcs=None):
        """Read PS1 or gaia sources for an exposure number + CCD name or CCD WCS

        Args:
            expnum, ccdname: select catalogue with these
            ccdwcs: or select catalogue with this

        """
        self.ps1catdir = os.getenv('PS1CAT_DIR')
        if self.ps1catdir is None:
            raise ValueError(
                'You must have the PS1CAT_DIR environment variable set to point to healpixed PS1 catalogs'
            )
        fnpattern = os.path.join(self.ps1catdir, 'ps1-%(hp)05d.fits')
        super(ps1cat, self).__init__(fnpattern)

        if ccdwcs is None:
            from legacypipe.survey import LegacySurveyData
            survey = LegacySurveyData()
            ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
            im = survey.get_image_object(ccd)
            self.ccdwcs = im.get_wcs()
        else:
            self.ccdwcs = ccdwcs
예제 #6
0
def main():
    """Main program.
    """
    import argparse

    parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).")
    parser.add_argument('--calibs', action='store_true',
                      help='Output CCDs that need to be calibrated.')

    parser.add_argument('--nper', type=int, default=None,
                      help='Batch N calibs per line')

    parser.add_argument('--forced', action='store_true',
                      help='Output forced-photometry commands')

    parser.add_argument('--lsb', action='store_true',
                      help='Output Low-Surface-Brightness commands')

    parser.add_argument('--touching', action='store_true',
                      help='Cut to only CCDs touching selected bricks')
    parser.add_argument('--near', action='store_true',
                      help='Quick cut to only CCDs near selected bricks')

    parser.add_argument('--check', action='store_true',
                      help='Check which calibrations actually need to run.')
    parser.add_argument('--check-coadd', action='store_true',
                      help='Check which caoadds actually need to run.')
    parser.add_argument('--out', help='Output filename for calibs, default %(default)s',
                      default='jobs')
    parser.add_argument('--command', action='store_true',
                      help='Write out full command-line to run calib')
    parser.add_argument('--opt', help='With --command, extra options to add')
    
    parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
    parser.add_argument('--mindec', type=float, help='Minimum Dec to run')

    parser.add_argument('--region', help='Region to select')

    parser.add_argument('--bricks', help='Set bricks.fits file to load')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')
    parser.add_argument('--ignore_cuts', action='store_true',default=False,help='no photometric or blacklist cuts')
    parser.add_argument('--save_to_fits', action='store_true',default=False,help='save cut brick,ccd to fits table')
    parser.add_argument('--name', action='store',default='dr3',help='save with this suffix, e.g. refers to ccds table')

    parser.add_argument('--delete-sky', action='store_true',
                      help='Delete any existing sky calibration files')
    parser.add_argument('--delete-pvastrom', action='store_true',
                      help='Delete any existing PV WCS calibration files')

    parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')

    parser.add_argument('--brickq', type=int, default=None,
                        help='Queue only bricks with the given "brickq" value [0 to 3]')

    parser.add_argument('--brickq-deps', action='store_true', default=False,
                        help='Queue bricks directly using qdo API, setting brickq dependencies')
    parser.add_argument('--queue', default='bricks',
                        help='With --brickq-deps, the QDO queue name to use')
    
    opt = parser.parse_args()

    survey = LegacySurveyData()
    if opt.bricks is not None:
        B = fits_table(opt.bricks)
        log('Read', len(B), 'from', opt.bricks)
    else:
        B = survey.get_bricks()

    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        log('Read', len(T), 'from', opt.ccds)
    else:
        T = survey.get_ccds()
        log(len(T), 'CCDs')
    T.index = np.arange(len(T))

    if opt.ignore_cuts == False:
        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
    print(len(T), 'CCDs remain')

    # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
    # plt.clf()
    # plt.hist(counts, counts.max()+1)
    # plt.savefig('bricks.png')
    # B.cut(I[counts >= 9])
    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # #plt.scatter(B.ra[I], B.dec[I], c=counts)
    # plt.savefig('bricks2.png')


    # DES Stripe82
    #rlo,rhi = 350.,360.
    # rlo,rhi = 300., 10.
    # dlo,dhi = -6., 4.
    # TINY bit
    #rlo,rhi = 350.,351.1
    #dlo,dhi = 0., 1.1

    # EDR+
    # 860 bricks
    # ~10,000 CCDs
    #rlo,rhi = 239,246
    #dlo,dhi =   5, 13

    # DR1
    #rlo,rhi = 0, 360
    # part 1
    #dlo,dhi = 25, 40
    # part 2
    #dlo,dhi = 20,25
    # part 3
    #dlo,dhi = 15,20
    # part 4
    #dlo,dhi = 10,15
    # part 5
    #dlo,dhi = 5,10
    # the rest
    #dlo,dhi = -11, 5
    #dlo,dhi = 15,25.5

    dlo,dhi = -25, 40
    rlo,rhi = 0, 360

    # Arjun says 3x3 coverage area is roughly
    # RA=240-252 DEC=6-12 (but not completely rectangular)

    # COSMOS
    #rlo,rhi = 148.9, 151.2
    #dlo,dhi = 0.9, 3.5

    # A nice well-behaved region (EDR2/3)
    # rlo,rhi = 243.6, 244.6
    # dlo,dhi = 8.1, 8.6

    # 56 bricks, ~725 CCDs
    #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
    # 240 bricks, ~3000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
    # 535 bricks, ~7000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))


    if opt.region in ['test1', 'test2', 'test3', 'test4']:
        nm = dict(test1='2446p115', # weird stuff around bright star
                  test2='1183p292', # faint sources around bright galaxy
                  test3='3503p005', # DES
                  test4='1163p277', # Pollux
                  )[opt.region]

        B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
        log('Cut to', len(B), 'bricks')
        log(B.ra, B.dec)
        dlo,dhi = -90,90
        rlo,rhi = 0, 360

    elif opt.region == 'edr':
        # EDR:
        # 535 bricks, ~7000 CCDs
        rlo,rhi = 240,245
        dlo,dhi =   5, 12

    elif opt.region == 'edrplus':
        rlo,rhi = 235,248
        dlo,dhi =   5, 15

    elif opt.region == 'edr-south':
        rlo,rhi = 240,245
        dlo,dhi =   5, 10

    elif opt.region == 'cosmos1':
        # 16 bricks in the core of the COSMOS field.
        rlo,rhi = 149.75, 150.75
        dlo,dhi = 1.6, 2.6

    elif opt.region == 'pristine':
        # Stream?
        rlo,rhi = 240,250
        dlo,dhi = 10,15

    elif opt.region == 'des':
        dlo, dhi = -6., 4.
        rlo, rhi = 317., 7.

        T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
        log('Cut to', len(T), 'CCDs with "CPDES82" in filename')

    elif opt.region == 'subdes':
        rlo,rhi = 320., 360.
        dlo,dhi = -1.25, 1.25

    elif opt.region == 'northwest':
        rlo,rhi = 240,360
        dlo,dhi = 20,40
    elif opt.region == 'north':
        rlo,rhi = 120,240
        dlo,dhi = 20,40
    elif opt.region == 'northeast':
        rlo,rhi = 0,120
        dlo,dhi = 20,40
    elif opt.region == 'southwest':
        rlo,rhi = 240,360
        dlo,dhi = -20,0
    elif opt.region == 'south':
        rlo,rhi = 120,240
        dlo,dhi = -20,0
    elif opt.region == 'southeast':
        rlo,rhi = 0,120
        dlo,dhi = -20,0
    elif opt.region == 'southsoutheast':
        rlo,rhi = 0,120
        dlo,dhi = -20,-10
    elif opt.region == 'midwest':
        rlo,rhi = 240,360
        dlo,dhi = 0,20
    elif opt.region == 'middle':
        rlo,rhi = 120,240
        dlo,dhi = 0,20
    elif opt.region == 'mideast':
        rlo,rhi = 0,120
        dlo,dhi = 0,20

    elif opt.region == 'grz':
        # Bricks with grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
        log('Cut to', len(B), 'bricks with grz coverage')

    elif opt.region == 'nogrz':
        # Bricks without grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
        log('Cut to', len(B), 'bricks withOUT grz coverage')
    elif opt.region == 'deep2':
        rlo,rhi = 250,260
        dlo,dhi = 30,35

    elif opt.region == 'deep2f3':
        rlo,rhi = 351.25, 353.75
        dlo,dhi = 0, 0.5

    elif opt.region == 'virgo':
        rlo,rhi = 185,190
        dlo,dhi =  10, 15

    elif opt.region == 'virgo2':
        rlo,rhi = 182,192
        dlo,dhi =   8, 18

    elif opt.region == 'lsb':
        rlo,rhi = 147.2, 147.8
        dlo,dhi = -0.4, 0.4

    elif opt.region == 'eboss-elg':
        # RA -45 to +45
        # Dec -5 to +7
        rlo,rhi = 315., 45.
        dlo,dhi = -5., 7.

    elif opt.region == 'eboss-ngc':
        # NGC ELGs
        # RA 115 to 175
        # Dec 15 to  30
        rlo,rhi = 115., 175.
        dlo,dhi =  15.,  30.

    elif opt.region == 'mzls':
        dlo,dhi = 30., 90.
    elif opt.region == 'dr4-bootes':
        # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched 
        #dlo,dhi = 34., 35.
        #rlo,rhi = 209.5, 210.5
        dlo,dhi = 33., 36.
        rlo,rhi = 216.5, 219.5

        
    if opt.mindec is not None:
        dlo = opt.mindec
    if opt.maxdec is not None:
        dhi = opt.maxdec

    if rlo < rhi:
        B.cut((B.ra >= rlo) * (B.ra <= rhi) *
              (B.dec >= dlo) * (B.dec <= dhi))
    else: # RA wrap
        B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) *
              (B.dec >= dlo) * (B.dec <= dhi))
    log(len(B), 'bricks in range')
    for name in B.get('brickname'):
        print(name)
    B.writeto('bricks-cut.fits')

    I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize)
    keep = np.zeros(len(B), bool)
    for i in I:
        keep[i] = True
    B.cut(keep)
    log('Cut to', len(B), 'bricks near CCDs')

    plt.clf()
    plt.plot(B.ra, B.dec, 'b.')
    plt.title('DR3 bricks')
    plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
    plt.savefig('bricks.png')

    if opt.brickq is not None:
        B.cut(B.brickq == opt.brickq)
        log('Cut to', len(B), 'with brickq =', opt.brickq)
    
    if opt.touching:
        keep = np.zeros(len(T), bool)
        for j in J:
            keep[j] = True
        T.cut(keep)
        log('Cut to', len(T), 'CCDs near bricks')

    # Aside -- how many near DR1=1 CCDs?
    if False:
        T2 = D.get_ccds()
        log(len(T2), 'CCDs')
        T2.cut(T2.dr1 == 1)
        log(len(T2), 'CCDs marked DR1=1')
        log(len(B), 'bricks in range')
        I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize)
        keep = np.zeros(len(B), bool)
        for i in I:
            keep[i] = True
        B2 = B[keep]
        log('Total of', len(B2), 'bricks near CCDs with DR1=1')
        for band in 'grz':
            Tb = T2[T2.filter == band]
            log(len(Tb), 'in filter', band)
            I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, survey.bricksize)
            good = np.zeros(len(B2), np.uint8)
            for i in I:
                good[i] = 1
            B2.set('has_' + band, good)

        B2.writeto('survey-bricks-in-dr1.fits')
        sys.exit(0)

    # sort by dec decreasing
    #B.cut(np.argsort(-B.dec))
    # RA increasing
    B.cut(np.argsort(B.ra))

    for b in B:
        if opt.check:
            fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue
        if opt.check_coadd:
            fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue

        print(b.brickname)

    if opt.save_to_fits:
        assert(opt.touching)
        # Write cut tables to file
        for tab,typ in zip([B,T],['bricks','ccds']):
            fn='%s-%s-cut.fits' % (typ,opt.name)
            if os.path.exists(fn):
                os.remove(fn)
            tab.writeto(fn)
            print('Wrote %s' % fn)
        # Write text files listing ccd and filename names
        nm1,nm2= 'ccds-%s.txt'% opt.name,'filenames-%s.txt' % opt.name
        if os.path.exists(nm1):
            os.remove(nm1)
        if os.path.exists(nm2):
            os.remove(nm2)
        f1,f2=open(nm1,'w'),open(nm2,'w')
        fns= list(set(T.get('image_filename')))
        for fn in fns:
            f2.write('%s\n' % fn.strip())
        for ti in T:
            f1.write('%s\n' % ti.get('image_filename').strip())
        f1.close()
        f2.close()
        print('Wrote *-names.txt')
    

    if opt.brickq_deps:
        import qdo
        from legacypipe.survey import on_bricks_dependencies

        #... find Queue...
        q = qdo.connect(opt.queue, create_ok=True)
        print('Connected to QDO queue', opt.queue, q)
        brick_to_task = dict()

        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
        print(len(T), 'CCDs remaining')

        T.wra = T.ra + (T.ra > 180) * -360
        wra = rlo - 360
        plt.clf()
        plt.plot(T.wra, T.dec, 'b.')
        ax = [wra, rhi, dlo, dhi]
        plt.axis(ax)
        plt.title('CCDs')
        plt.savefig('q-ccds.png')

        B.wra = B.ra + (B.ra > 180) * -360

        # this slight overestimate (for DECam images) is fine
        radius = 0.3
        Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius,
                            indexlist=True)
        ikeep = []
        for ib,(b,Iccd) in enumerate(zip(B, Iccds)):
            if Iccd is None or len(Iccd) == 0:
                print('No matched CCDs to brick', b.brickname)
                continue
            wcs = wcs_for_brick(b)
            cI = ccds_touching_wcs(wcs, T[np.array(Iccd)])
            print(len(cI), 'CCDs touching brick', b.brickname)
            if len(cI) == 0:
                continue
            ikeep.append(ib)
        B.cut(np.array(ikeep))
        print('Cut to', len(B), 'bricks touched by CCDs')
        
        for brickq in range(4):
            I = np.flatnonzero(B.brickq == brickq)
            print(len(I), 'bricks with brickq =', brickq)

            J = np.flatnonzero(B.brickq < brickq)
            preB = B[J]
            reqs = []
            if brickq > 0:
                for b in B[I]:
                    # find brick dependencies
                    brickdeps = on_bricks_dependencies(b, survey, bricks=preB)
                    # convert to task ids
                    taskdeps = [brick_to_task.get(b.brickname,None) for b in brickdeps]
                    # If we dropped a dependency brick from a previous brickq because
                    # of no overlapping CCDs, it won't appear in the brick_to_task map.
                    taskdeps = [t for t in taskdeps if t is not None]
                    reqs.append(taskdeps)

            plt.clf()
            plt.plot(B.wra, B.dec, '.', color='0.5')
            plt.plot(B.wra[I], B.dec[I], 'b.')
            plt.axis(ax)
            plt.title('Bricks: brickq=%i' % brickq)
            plt.savefig('q-bricks-%i.png' % brickq)
            
            # submit to qdo queue
            print('Queuing', len(B[I]), 'bricks')
            if brickq == 0:
                reqs = None
            else:
                assert(len(I) == len(reqs))
            taskids = q.add_multiple(B.brickname[I], requires=reqs)
            assert(len(taskids) == len(I))
            print('Queued', len(taskids), 'bricks')
            brick_to_task.update(dict(zip(B.brickname[I], taskids)))
        
    if not (opt.calibs or opt.forced or opt.lsb):
        sys.exit(0)

    bands = 'grz'
    log('Filters:', np.unique(T.filter))
    T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
    log('Cut to', len(T), 'CCDs in filters', bands)

    if opt.touching:
        allI = set()
        for b in B:
            wcs = wcs_for_brick(b)
            I = ccds_touching_wcs(wcs, T)
            log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
            if len(I) == 0:
                continue
            allI.update(I)
        allI = list(allI)
        allI.sort()
    elif opt.near:
        # Roughly brick radius + DECam image radius
        radius = 0.35
        allI,nil,nil = match_radec(T.ra, T.dec, B.ra, B.dec, radius, nearest=True)
    else:
        allI = np.arange(len(T))

    if opt.write_ccds:
        T[allI].writeto(opt.write_ccds)
        log('Wrote', opt.write_ccds)

    ## Be careful here -- T has been cut; we want to write out T.index.
    ## 'allI' contains indices into T.

    if opt.forced:
        log('Writing forced-photometry commands to', opt.out)
        f = open(opt.out,'w')
        log('Total of', len(allI), 'CCDs')
        for j,i in enumerate(allI):
            expstr = '%08i' % T.expnum[i]
            outfn = os.path.join('forced', expstr[:5], expstr,
                                 'decam-%s-%s-forced.fits' %
                                 (expstr, T.ccdname[i]))
            imgfn = os.path.join(survey.survey_dir, 'images',
                                 T.image_filename[i].strip())
            if (not os.path.exists(imgfn) and
                imgfn.endswith('.fz') and
                os.path.exists(imgfn[:-3])):
                imgfn = imgfn[:-3]

            #f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' %
            #        (imgfn, T.image_hdu[i], outfn))

            f.write('python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n' %
                    (T.expnum[i], T.ccdname[i], outfn))
            
        f.close()
        log('Wrote', opt.out)
        sys.exit(0)

    if opt.lsb:
        log('Writing LSB commands to', opt.out)
        f = open(opt.out,'w')
        log('Total of', len(allI), 'CCDs')
        for j,i in enumerate(allI):
            exp = T.expnum[i]
            ext = T.ccdname[i].strip()
            outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
            f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext))
        f.close()
        log('Wrote', opt.out)
        sys.exit(0)


    log('Writing calibs to', opt.out)
    f = open(opt.out,'w')
    log('Total of', len(allI), 'CCDs')

    batch = []

    def write_batch(f, batch, cmd):
        if cmd is None:
            cmd = ''
        f.write(cmd + ' '.join(batch) + '\n')

    cmd = None
    if opt.command:
        cmd = 'python legacypipe/run-calib.py '
        if opt.opt is not None:
            cmd += opt.opt + ' '
        
    for j,i in enumerate(allI):

        if opt.delete_sky or opt.delete_pvastrom:
            log(j+1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if opt.delete_sky and os.path.exists(im.skyfn):
                log('  deleting:', im.skyfn)
                os.unlink(im.skyfn)
            if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
                log('  deleting:', im.pvwcsfn)
                os.unlink(im.pvwcsfn)

        if opt.check:
            log(j+1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if not im.run_calibs(im, just_check=True):
                log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done')
                continue

        if opt.command:
            s = '%i-%s' % (T.expnum[i], T.ccdname[i])
            prefix = 'python legacypipe/run-calib.py ' + opt.opt
            #('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
            #     (T.expnum[i], T.ccdname[i]))
        else:
            s = '%i' % T.index[i]
            prefix = ''
            
        if j < 10:
            print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i],
                  'filename', T.image_filename[i])
            
        if not opt.nper:
            f.write(prefix + s + '\n')
        else:
            batch.append(s)
            if len(batch) >= opt.nper:
                write_batch(f, batch, cmd)
                batch = []

        if opt.check:
            f.flush()

    if len(batch):
        write_batch(f, batch, cmd)

    f.close()
    log('Wrote', opt.out)
    return 0
예제 #7
0
def main():
    survey = LegacySurveyData()
    ccds = survey.get_ccds()
    print(len(ccds), 'CCDs')

    expnums = np.unique(ccds.expnum)
    print(len(expnums), 'unique exposures')

    for expnum in expnums:

        expnumstr = '%08i' % expnum
        skyoutfn = os.path.join('splinesky', expnumstr[:5], 'decam-%s.fits' % expnumstr)
        psfoutfn = os.path.join('psfex', expnumstr[:5], 'decam-%s.fits' % expnumstr)

        if os.path.exists(skyoutfn) and os.path.exists(psfoutfn):
            print('Exposure', expnum, 'is done already')
            continue

        C = ccds[ccds.expnum == expnum]
        print(len(C), 'CCDs in expnum', expnum)

        psfex = []
        psfhdrvals = []

        splinesky = []
        skyhdrvals = []

        for ccd in C:
            im = survey.get_image_object(ccd)

            fn = im.splineskyfn
            if os.path.exists(fn):
                T = fits_table(fn)
                splinesky.append(T)
                # print(fn)
                # T.about()
                hdr = fitsio.read_header(fn)
                skyhdrvals.append([hdr[k] for k in [
                            'SKY', 'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname])
            else:
                print('File not found:', fn)

            fn = im.psffn
            if os.path.exists(fn):
                T = fits_table(fn)
                hdr = fitsio.read_header(fn, ext=1)

                keys = ['LOADED', 'ACCEPTED', 'CHI2', 'POLNAXIS', 
                        'POLNGRP', 'PSF_FWHM', 'PSF_SAMP', 'PSFNAXIS',
                        'PSFAXIS1', 'PSFAXIS2', 'PSFAXIS3',]

                if hdr['POLNAXIS'] == 0:
                    # No polynomials.  Fake it.
                    T.polgrp1 = np.array([0])
                    T.polgrp2 = np.array([0])
                    T.polname1 = np.array(['fake'])
                    T.polname2 = np.array(['fake'])
                    T.polzero1 = np.array([0])
                    T.polzero2 = np.array([0])
                    T.polscal1 = np.array([1])
                    T.polscal2 = np.array([1])
                    T.poldeg1 = np.array([0])
                    T.poldeg2 = np.array([0])
                else:
                    keys.extend([
                            'POLGRP1', 'POLNAME1', 'POLZERO1', 'POLSCAL1',
                            'POLGRP2', 'POLNAME2', 'POLZERO2', 'POLSCAL2',
                            'POLDEG1'])

                for k in keys:
                    T.set(k.lower(), np.array([hdr[k]]))
                psfex.append(T)
                #print(fn)
                #T.about()
    
                hdr = fitsio.read_header(fn)
                psfhdrvals.append([hdr.get(k,'') for k in [
                    'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname])
            else:
                print('File not found:', fn)

        if len(psfex):
            padded = pad_arrays([p.psf_mask[0] for p in psfex])
            cols = psfex[0].columns()
            cols.remove('psf_mask')
            T = merge_tables(psfex, columns=cols)
            T.psf_mask = np.concatenate([[p] for p in padded])
            T.legpipev = np.array([h[0] for h in psfhdrvals])
            T.plver    = np.array([h[1] for h in psfhdrvals])
            T.expnum   = np.array([h[2] for h in psfhdrvals])
            T.ccdname  = np.array([h[3] for h in psfhdrvals])
            fn = psfoutfn
            trymakedirs(fn, dir=True)
            T.writeto(fn)
            print('Wrote', fn)

        if len(splinesky):
            T = fits_table()
            T.gridw = np.array([t.gridvals[0].shape[1] for t in splinesky])
            T.gridh = np.array([t.gridvals[0].shape[0] for t in splinesky])

            padded = pad_arrays([t.gridvals[0] for t in splinesky])
            T.gridvals = np.concatenate([[p] for p in padded])
            padded = pad_arrays([t.xgrid[0] for t in splinesky])
            T.xgrid = np.concatenate([[p] for p in padded])
            padded = pad_arrays([t.xgrid[0] for t in splinesky])
            T.ygrid = np.concatenate([[p] for p in padded])
    
            cols = splinesky[0].columns()
            print('Columns:', cols)
            for c in ['gridvals', 'xgrid', 'ygrid']:
                cols.remove(c)

            T.add_columns_from(merge_tables(splinesky, columns=cols))
            T.skyclass = np.array([h[0] for h in skyhdrvals])
            T.legpipev = np.array([h[1] for h in skyhdrvals])
            T.plver    = np.array([h[2] for h in skyhdrvals])
            T.expnum   = np.array([h[3] for h in skyhdrvals])
            T.ccdname  = np.array([h[4] for h in skyhdrvals])
            fn = skyoutfn
            trymakedirs(fn, dir=True)
            T.writeto(fn)
            print('Wrote', fn)
예제 #8
0
def main(survey=None, opt=None):
    '''Driver function for forced photometry of individual Legacy
    Survey images.
    '''
    if opt is None:
        parser = get_parser()
        opt = parser.parse_args()

    Time.add_measurement(MemMeas)
    t0 = Time()

    if os.path.exists(opt.outfn):
        print('Ouput file exists:', opt.outfn)
        sys.exit(0)

    if opt.derivs and opt.agn:
        print('Sorry, can\'t do --derivs AND --agn')
        sys.exit(0)

    if not opt.forced:
        opt.apphot = True

    zoomslice = None
    if opt.zoom is not None:
        (x0, x1, y0, y1) = opt.zoom
        zoomslice = (slice(y0, y1), slice(x0, x1))

    ps = None
    if opt.plots is not None:
        import pylab as plt
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence(opt.plots)

    # Try parsing filename as exposure number.
    try:
        expnum = int(opt.expnum)
        filename = None
    except:
        # make this 'None' for survey.find_ccds()
        expnum = None
        filename = opt.expnum

    # Try parsing HDU number
    try:
        hdu = int(opt.ccdname)
        ccdname = None
    except:
        hdu = -1
        ccdname = opt.ccdname

    if survey is None:
        survey = LegacySurveyData()

    catsurvey = survey
    if opt.catalog_dir is not None:
        catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir)

    if filename is not None and hdu >= 0:
        # FIXME -- try looking up in CCDs file?
        # Read metadata from file
        print('Warning: faking metadata from file contents')
        T = exposure_metadata([filename], hdus=[hdu])
        print('Metadata:')
        T.about()

        if not 'ccdzpt' in T.columns():
            phdr = fitsio.read_header(filename)
            T.ccdzpt = np.array([phdr['MAGZERO']])
            print('WARNING: using header MAGZERO')
            T.ccdraoff = np.array([0.])
            T.ccddecoff = np.array([0.])
            print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.')

    else:
        # Read metadata from survey-ccds.fits table
        T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
        print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
        if hdu >= 0:
            T.cut(T.image_hdu == hdu)
            print(len(T), 'with HDU', hdu)
        if filename is not None:
            T.cut(np.array([f.strip() == filename for f in T.image_filename]))
            print(len(T), 'with filename', filename)
        if opt.camera is not None:
            T.cut(T.camera == opt.camera)
            print(len(T), 'with camera', opt.camera)
        assert (len(T) == 1)

    ccd = T[0]
    im = survey.get_image_object(ccd)

    if opt.do_calib:
        #from legacypipe.survey import run_calibs
        #kwa = dict(splinesky=True)
        #run_calibs((im, kwa))
        im.run_calibs(splinesky=True)

    tim = im.get_tractor_image(slc=zoomslice,
                               pixPsf=True,
                               splinesky=True,
                               constant_invvar=opt.constant_invvar,
                               hybridPsf=opt.hybrid_psf,
                               normalizePsf=opt.normalize_psf)
    print('Got tim:', tim)

    print('Read image:', Time() - t0)

    if opt.catfn in ['DR1', 'DR2', 'DR3', 'DR5', 'DR']:

        margin = 20
        TT = []
        chipwcs = tim.subwcs
        bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
        for b in bricks:
            # there is some overlap with this brick... read the catalog.
            fn = catsurvey.find_file('tractor', brick=b.brickname)
            if not os.path.exists(fn):
                print('WARNING: catalog', fn, 'does not exist.  Skipping!')
                continue
            print('Reading', fn)
            T = fits_table(fn)
            ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
            W, H = chipwcs.get_width(), chipwcs.get_height()
            I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) *
                               (yy >= -margin) * (yy <= (H + margin)))
            T.cut(I)
            print('Cut to', len(T), 'sources within image + margin')
            # print('Brick_primary:', np.unique(T.brick_primary))
            T.cut(T.brick_primary)
            print('Cut to', len(T), 'on brick_primary')
            for col in ['out_of_bounds', 'left_blob']:
                if col in T.get_columns():
                    T.cut(T.get(col) == False)
                    print('Cut to', len(T), 'on', col)
            if len(T):
                TT.append(T)
        if len(TT) == 0:
            print('No sources to photometer.')
            return 0
        T = merge_tables(TT, columns='fillzero')
        T._header = TT[0]._header
        del TT
        print('Total of', len(T), 'catalog sources')

        # Fix up various failure modes:
        # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
        # -> convert to EXP
        I = np.flatnonzero(
            np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r)))
                      for t in T]))
        if len(I):
            print('Converting', len(I), 'bogus COMP galaxies to EXP')
            for i in I:
                T.type[i] = 'EXP'

        # Same thing with the exp component.
        # -> convert to DEV
        I = np.flatnonzero(
            np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r)))
                      for t in T]))
        if len(I):
            print('Converting', len(I), 'bogus COMP galaxies to DEV')
            for i in I:
                T.type[i] = 'DEV'

        if opt.write_cat:
            T.writeto(opt.write_cat)
            print('Wrote catalog to', opt.write_cat)

    else:
        T = fits_table(opt.catfn)

    surveydir = survey.get_survey_dir()
    del survey

    kwargs = {}
    cols = T.get_columns()
    if 'flux_r' in cols and not 'decam_flux_r' in cols:
        kwargs.update(fluxPrefix='')
    cat = read_fits_catalog(T, **kwargs)
    # Replace the brightness (which will be a NanoMaggies with g,r,z)
    # with a NanoMaggies with this image's band only.
    for src in cat:
        src.brightness = NanoMaggies(**{tim.band: 1.})

    print('Read catalog:', Time() - t0)

    print('Forced photom...')
    F = run_forced_phot(cat,
                        tim,
                        ceres=opt.ceres,
                        derivs=opt.derivs,
                        fixed_also=True,
                        agn=opt.agn,
                        do_forced=opt.forced,
                        do_apphot=opt.apphot,
                        ps=ps)
    t0 = Time()

    F.release = T.release
    F.brickid = T.brickid
    F.brickname = T.brickname
    F.objid = T.objid

    F.camera = np.array([ccd.camera] * len(F))
    F.expnum = np.array([im.expnum] * len(F)).astype(np.int32)
    F.ccdname = np.array([im.ccdname] * len(F))

    # "Denormalizing"
    F.filter = np.array([tim.band] * len(T))
    F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
    F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32)
    F.ra = T.ra
    F.dec = T.dec

    ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
    F.x = (x - 1).astype(np.float32)
    F.y = (y - 1).astype(np.float32)

    h, w = tim.shape
    F.mask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h - 1),
                    np.clip(np.round(F.x).astype(int), 0, w - 1)]

    program_name = sys.argv[0]
    version_hdr = get_version_header(program_name, surveydir)
    filename = getattr(ccd, 'image_filename')
    if filename is None:
        # HACK -- print only two directory names + filename of CPFILE.
        fname = os.path.basename(im.imgfn)
        d = os.path.dirname(im.imgfn)
        d1 = os.path.basename(d)
        d = os.path.dirname(d)
        d2 = os.path.basename(d)
        filename = os.path.join(d2, d1, fname)
        print('Trimmed filename to', filename)
    version_hdr.add_record(
        dict(name='CPFILE', value=filename, comment='CP file'))
    version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
    version_hdr.add_record(
        dict(name='CAMERA', value=ccd.camera, comment='Camera'))
    version_hdr.add_record(
        dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
    version_hdr.add_record(
        dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
    version_hdr.add_record(
        dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
    version_hdr.add_record(
        dict(name='EXPOSURE',
             value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname),
             comment='Name of this image'))

    keys = [
        'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
    ]
    for key in keys:
        if key in tim.primhdr:
            version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))

    hdr = fitsio.FITSHDR()
    units = {
        'exptime': 'sec',
        'flux': 'nanomaggy',
        'flux_ivar': '1/nanomaggy^2'
    }
    columns = F.get_columns()
    for i, col in enumerate(columns):
        if col in units:
            hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))

    outdir = os.path.dirname(opt.outfn)
    if len(outdir):
        trymakedirs(outdir)
    fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
    F.writeto(opt.outfn, header=hdr, append=True)
    print('Wrote', opt.outfn)

    if opt.save_model or opt.save_data:
        hdr = fitsio.FITSHDR()
        tim.getWcs().wcs.add_to_header(hdr)
    if opt.save_model:
        print('Getting model image...')
        tr = Tractor([tim], cat)
        mod = tr.getModelImage(tim)
        fitsio.write(opt.save_model, mod, header=hdr, clobber=True)
        print('Wrote', opt.save_model)
    if opt.save_data:
        fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
        print('Wrote', opt.save_data)

    print('Finished forced phot:', Time() - t0)
    return 0
예제 #9
0
pixscale = 0.262
cd = pixscale / 3600.
targetwcs = Tan(ra, dec, W / 2., H / 2., -cd, 0., 0., cd, float(W), float(H))

rr, dd = targetwcs.pixelxy2radec([1, W, W, 1, 1], [1, 1, H, H, 1])
targetrd = np.vstack((rr, dd)).T

ccds = survey.ccds_touching_wcs(targetwcs)
print(len(ccds), 'CCDs touching WCS')

print('MJDs', ccds.mjd_obs)
ccds.writeto('test-ccds.fits')

ccd1 = ccds[(ccds.expnum == expnum) * (ccds.ccdname == ccdname)]
print('CCD1:', ccd1)
im1 = survey.get_image_object(ccd1[0])
print('Im:', im1)

#wcs = im1.get_wcs()
#x0,x1,y0,y1,slc = im1.get_image_extent(wcs=wcs, radecpoly=targetrd)

tim1 = im1.get_tractor_image(radecpoly=targetrd,
                             pixPsf=True,
                             hybridPsf=True,
                             normalizePsf=True,
                             splinesky=True)
print('Tim', tim1)

##
tims = [tim1]
ccd1.ccd_x0 = np.array([tim.x0 for tim in tims]).astype(np.int16)
예제 #10
0
    psf = psf.drawImage(scale=pixscale, nx=W + 1, ny=H + 1)

    ##  psf           = tractor.GaussianMixturePSF(1., 0., 0., v, v, 0.)
    psf = tractor.psf.PixelizedPSF(psf.array)

    return psf


if __name__ == '__main__':
    print('Welcome to montelg src.')

    survey = LegacySurveyData()

    ccds = survey.get_annotated_ccds()
    ccds = survey.cleanup_ccds_table(ccds)
    im = survey.get_image_object(ccds[0])

    bands = ['g', 'r', 'z']

    red = dict(g=2.5, r=1., i=0.4, z=0.4)

    os.environ[
        'LEGACY_SURVEY_DIR'] = '/project/projectdirs/cosmo/data/legacysurvey/dr8/'

    ##  Source.
    ra, dec = 40., 10.

    gre = 0.40  # [arcsec].
    gmag = 23.0
    gflux = 10.**(-0.4 * (gmag - 22.5))  # [Nanomaggies].
    ##  gflux         = exptime * 10.**((zpt - gmag) / 2.5)               # [Total counts on the image].
예제 #11
0
def psf_residuals(expnum,
                  ccdname,
                  stampsize=35,
                  nstar=30,
                  magrange=(13, 17),
                  verbose=0,
                  splinesky=False):

    # Set the debugging level.
    if verbose == 0:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG
    logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)

    pngprefix = 'qapsf-{}-{}'.format(expnum, ccdname)

    # Gather all the info we need about this CCD.
    survey = LegacySurveyData()
    ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
    band = ccd.filter
    ps1band = dict(g=0, r=1, i=2, z=3, Y=4)
    print('Band {}'.format(band))

    #scales = dict(g=0.0066, r=0.01, z=0.025)
    #vmin, vmax = np.arcsinh(-1), np.arcsinh(100)
    #print(scales[band])

    im = survey.get_image_object(ccd)
    iminfo = im.get_image_info()
    H, W = iminfo['dims']

    wcs = im.get_wcs()

    # Choose a uniformly selected subset of PS1 stars on this CCD.
    ps1 = ps1cat(ccdwcs=wcs)
    cat = ps1.get_stars(band=band, magrange=magrange)

    rand = np.random.RandomState(seed=expnum * ccd.ccdnum)
    these = rand.choice(len(cat) - 1, nstar, replace=False)
    #these = rand.random_integers(0,len(cat)-1,nstar)
    cat = cat[these]
    cat = cat[np.argsort(cat.median[:, ps1band[band]])]  # sort by magnitude
    #print(cat.nmag_ok)

    get_tim_kwargs = dict(pixPsf=True, splinesky=splinesky)

    # Make a QAplot of the positions of all the stars.
    tim = im.get_tractor_image(**get_tim_kwargs)
    img = tim.getImage()
    #img = tim.getImage()/scales[band]

    fig = plt.figure(figsize=(5, 10))
    ax = fig.gca()
    ax.get_xaxis().get_major_formatter().set_useOffset(False)
    #ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest',
    #          origin='lower',vmin=vmax,vmax=vmax)

    ax.imshow(img, **tim.ima)
    ax.axis('off')
    ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.format(
        band, expnum, ccdname, ccd.airmass, ccd.seeing))

    for istar, ps1star in enumerate(cat):
        ra, dec = (ps1star.ra, ps1star.dec)
        ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
        ax.text(xpos,
                ypos,
                '{:2d}'.format(istar + 1),
                color='red',
                horizontalalignment='left')
        circ = plt.Circle((xpos, ypos), radius=30, color='g', fill=False, lw=1)
        ax.add_patch(circ)

    #radec = wcs.radec_bounds()
    #ax.scatter(cat.ra,cat.dec)
    #ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998])
    #ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015])
    #ax.set_xlabel('$RA\ (deg)$',fontsize=18)
    #ax.set_ylabel('$Dec\ (deg)$',fontsize=18)
    fig.savefig(pngprefix + '-ccd.png', bbox_inches='tight')

    # Initialize the many-stamp QAplot
    ncols = 3
    nrows = np.ceil(nstar / ncols).astype('int')

    inchperstamp = 2.0
    fig = plt.figure(figsize=(inchperstamp * 3 * ncols, inchperstamp * nrows))
    irow = 0
    icol = 0

    for istar, ps1star in enumerate(cat):
        ra, dec = (ps1star.ra, ps1star.dec)
        mag = ps1star.median[ps1band[band]]  # r-band

        ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
        ix, iy = int(xpos), int(ypos)

        # create a little tractor Image object around the star
        slc = (slice(max(iy - stampsize, 0), min(iy + stampsize + 1, H)),
               slice(max(ix - stampsize, 0), min(ix + stampsize + 1, W)))

        # The PSF model 'const2Psf' is the one used in DR1: a 2-component
        # Gaussian fit to PsfEx instantiated in the image center.
        tim = im.get_tractor_image(slc=slc, **get_tim_kwargs)
        stamp = tim.getImage()
        ivarstamp = tim.getInvvar()

        # Initialize a tractor PointSource from PS1 measurements
        flux = NanoMaggies.magToNanomaggies(mag)
        star = PointSource(RaDecPos(ra, dec), NanoMaggies(**{band: flux}))

        # Fit just the source RA,Dec,flux.
        tractor = Tractor([tim], [star])
        tractor.freezeParam('images')

        print('2-component MOG:', tim.psf)
        tractor.printThawedParams()

        for step in range(50):
            dlnp, X, alpha = tractor.optimize()
            if dlnp < 0.1:
                break
        print('Fit:', star)
        model_mog = tractor.getModelImage(0)
        chi2_mog = -2.0 * tractor.getLogLikelihood()
        mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0]

        # Now change the PSF model to a pixelized PSF model from PsfEx instantiated
        # at this place in the image.
        psf = PixelizedPsfEx(im.psffn)
        tim.psf = psf.constantPsfAt(xpos, ypos)

        #print('PSF model:', tim.psf)
        #tractor.printThawedParams()
        for step in range(50):
            dlnp, X, alpha = tractor.optimize()
            if dlnp < 0.1:
                break

        print('Fit:', star)
        model_psfex = tractor.getModelImage(0)
        chi2_psfex = -2.0 * tractor.getLogLikelihood()
        mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0]

        #mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95])
        sig = np.std((stamp - model_psfex)[ivarstamp > 0])
        mn, mx = [-2.0 * sig, 5 * sig]

        # Generate a QAplot.
        if (istar > 0) and (istar % (ncols) == 0):
            irow = irow + 1
        icol = 3 * istar - 3 * ncols * irow
        #print(istar, irow, icol, icol+1, icol+2)

        ax1 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol),
                               aspect='equal')
        ax1.axis('off')
        #ax1.imshow(stamp, **tim.ima)
        ax1.imshow(stamp,
                   cmap='gray',
                   interpolation='nearest',
                   origin='lower',
                   vmin=mn,
                   vmax=mx)
        ax1.text(0.1,
                 0.9,
                 '{:2d}'.format(istar + 1),
                 color='white',
                 horizontalalignment='left',
                 verticalalignment='top',
                 transform=ax1.transAxes)

        ax2 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 1),
                               aspect='equal')
        ax2.axis('off')
        #ax2.imshow(stamp-model_mog, **tim.ima)
        ax2.imshow(stamp - model_mog,
                   cmap='gray',
                   interpolation='nearest',
                   origin='lower',
                   vmin=mn,
                   vmax=mx)
        ax2.text(0.1,
                 0.9,
                 'MoG',
                 color='white',
                 horizontalalignment='left',
                 verticalalignment='top',
                 transform=ax2.transAxes)
        ax2.text(0.08,
                 0.08,
                 '{:.3f}'.format(mag_mog),
                 color='white',
                 horizontalalignment='left',
                 verticalalignment='bottom',
                 transform=ax2.transAxes)

        #ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14)
        #ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex))

        ax3 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 2),
                               aspect='equal')
        ax3.axis('off')
        #ax3.imshow(stamp-model_psfex, **tim.ima)
        ax3.imshow(stamp - model_psfex,
                   cmap='gray',
                   interpolation='nearest',
                   origin='lower',
                   vmin=mn,
                   vmax=mx)
        ax3.text(0.1,
                 0.9,
                 'PSFEx',
                 color='white',
                 horizontalalignment='left',
                 verticalalignment='top',
                 transform=ax3.transAxes)
        ax3.text(0.08,
                 0.08,
                 '{:.3f}'.format(mag_psfex),
                 color='white',
                 horizontalalignment='left',
                 verticalalignment='bottom',
                 transform=ax3.transAxes)

        if istar == (nstar - 1):
            break
    fig.savefig(pngprefix + '-stargrid.png', bbox_inches='tight')
예제 #12
0
def main():
    import argparse
    parser = argparse.ArgumentParser()

    parser.add_argument(
        '--dr',
        '--drdir',
        dest='drdir',
        default='/project/projectdirs/cosmo/data/legacysurvey/dr5',
        help='Directory containing data release w/ tar-gzipped calibs')
    parser.add_argument(
        '-b',
        '--brick',
        help='Brick name to run; required unless --radec is given')
    parser.add_argument(
        '--radec',
        nargs=2,
        help='RA,Dec center for a custom location (not a brick)')
    parser.add_argument('--pixscale',
                        type=float,
                        default=0.262,
                        help='Pixel scale of the output coadds (arcsec/pixel)')
    parser.add_argument('-W',
                        '--width',
                        type=int,
                        default=3600,
                        help='Target image width, default %(default)i')
    parser.add_argument('-H',
                        '--height',
                        type=int,
                        default=3600,
                        help='Target image height, default %(default)i')
    parser.add_argument(
        '--zoom',
        type=int,
        nargs=4,
        help='Set target image extent (default "0 3600 0 3600")')
    parser.add_argument('--no-psf',
                        dest='do_psf',
                        default=True,
                        action='store_false',
                        help='Do not extract PsfEx files')
    parser.add_argument('--no-sky',
                        dest='do_sky',
                        default=True,
                        action='store_false',
                        help='Do not extract SplineSky files')

    opt = parser.parse_args()
    if opt.brick is None and opt.radec is None:
        parser.print_help()
        return -1
    optdict = vars(opt)

    drdir = opt.drdir
    W = opt.width
    H = opt.height
    pixscale = opt.pixscale
    target_extent = opt.zoom

    do_psf = opt.do_psf
    do_sky = opt.do_sky

    #brickname = '1501p020'

    custom = (opt.radec is not None)
    #ra,dec = 216.03, 34.86
    if custom:
        ra, dec = opt.radec  #27.30, -10.43
        ra = float(ra)
        dec = float(dec)
        #W,H = 1000,1000
        #W,H = 1500,1500
        brickname = 'custom_%.3f_%.3f' % (ra, dec)
        #do_sky = False

    survey = LegacySurveyData()

    if custom:
        from legacypipe.survey import BrickDuck
        # Custom brick; create a fake 'brick' object
        brick = BrickDuck(ra, dec, brickname)
    else:
        brickname = opt.brick
        brick = survey.get_brick_by_name(brickname)

    # Get WCS object describing brick
    targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale)
    if target_extent is not None:
        (x0, x1, y0, y1) = target_extent
        W = x1 - x0
        H = y1 - y0
        targetwcs = targetwcs.get_subimage(x0, y0, W, H)

    # Find CCDs
    ccds = survey.ccds_touching_wcs(targetwcs, ccdrad=None)
    if ccds is None:
        raise NothingToDoError('No CCDs touching brick')
    print(len(ccds), 'CCDs touching target WCS')

    for ccd in ccds:
        im = survey.get_image_object(ccd)
        print('CCD', im)

        expnum = '%08i' % im.expnum

        if do_psf:
            if os.path.exists(im.psffn) or os.path.exists(im.merged_psffn):
                print('PSF file exists')
            else:
                print('Need PSF', im.psffn, im.merged_psffn)

                tarfn = os.path.join(
                    drdir, 'calib', im.camera, 'psfex-merged',
                    'legacysurvey_dr5_calib_decam_psfex-merged_%s.tar.gz' %
                    expnum[:5])
                print(tarfn)
                if os.path.exists(tarfn):
                    outfn = '%s/%s-%s.fits' % (expnum[:5], im.camera, expnum)
                    cmd = 'cd %s/%s/psfex-merged && tar xvzf %s %s' % (
                        survey.get_calib_dir(), im.camera, tarfn, outfn)
                    print(cmd)
                    os.system(cmd)

        if do_sky:
            if os.path.exists(im.splineskyfn) or os.path.exists(
                    im.merged_splineskyfn):
                print('Sky file exists')
            else:
                print('Need sky', im.splineskyfn, im.merged_splineskyfn)

                tarfn = os.path.join(
                    drdir, 'calib', im.camera, 'splinesky-merged',
                    'legacysurvey_dr5_calib_decam_splinesky-merged_%s.tar.gz' %
                    expnum[:5])
                print(tarfn)
                if os.path.exists(tarfn):
                    outfn = '%s/%s-%s.fits' % (expnum[:5], im.camera, expnum)
                    cmd = 'cd %s/%s/splinesky-merged && tar xvzf %s %s' % (
                        survey.get_calib_dir(), im.camera, tarfn, outfn)
                    print(cmd)
                    os.system(cmd)
예제 #13
0
from legacyanalysis.ps1cat import ps1cat
from legacypipe.survey import LegacySurveyData

from tractor import Image, PointSource, PixPos, NanoMaggies, Tractor

ps = PlotSequence('rewcs')

expnum, ccdname = 431109, 'N14'
cat = ps1cat(expnum=expnum, ccdname=ccdname)
stars = cat.get_stars()
print len(stars), 'stars'

survey = LegacySurveyData()
ccd = survey.find_ccds(expnum=expnum,ccdname=ccdname)[0]
im = survey.get_image_object(ccd)
wcs = im.get_wcs()
tim = im.get_tractor_image(pixPsf=True, splinesky=True)

margin = 15
ok,stars.xx,stars.yy = wcs.radec2pixelxy(stars.ra, stars.dec) 
stars.xx -= 1.
stars.yy -= 1.
W,H = wcs.get_width(), wcs.get_height()
stars.ix = np.round(stars.xx).astype(int)
stars.iy = np.round(stars.yy).astype(int)
stars.cut((stars.ix >= margin) * (stars.ix < (W-margin)) *
          (stars.iy >= margin) * (stars.iy < (H-margin)))

plt.clf()
plt.subplots_adjust(left=0.05, right=0.95, bottom=0.05, top=0.95,
예제 #14
0
def rbmain():
    from legacypipe.catalog import read_fits_catalog
    from legacypipe.survey import LegacySurveyData, wcs_for_brick
    from tractor.galaxy import DevGalaxy
    from tractor import PointSource, Catalog
    from tractor import GaussianMixturePSF
    from legacypipe.survey import BrickDuck
    from legacypipe.forced_photom import main as forced_main
    from astrometry.util.file import trymakedirs
    import shutil

    ceres = 'ceres' in sys.argv
    psfex = 'psfex' in sys.argv

    for v in [
            'UNWISE_COADDS_TIMERESOLVED_DIR', 'SKY_TEMPLATE_DIR',
            'LARGEGALAXIES_CAT', 'GAIA_CAT_DIR', 'TYCHO2_KD_DIR'
    ]:
        if v in os.environ:
            del os.environ[v]

    oldargs = sys.argv
    sys.argv = [sys.argv[0]]
    main()
    sys.argv = oldargs

    # Test create_kdtree and (reading CCD kd-tree)!
    indir = os.path.join(os.path.dirname(__file__), 'testcase6')
    with tempfile.TemporaryDirectory() as surveydir:
        files = [
            'calib', 'gaia', 'images', 'survey-bricks.fits.gz',
            'tycho2.kd.fits'
        ]
        for fn in files:
            src = os.path.join(indir, fn)
            dst = os.path.join(surveydir, fn)
            #trymakedirs(dst, dir=True)
            print('Copy', src, dst)
            if os.path.isfile(src):
                shutil.copy(src, dst)
            else:
                shutil.copytree(src, dst)

        from legacypipe.create_kdtrees import create_kdtree
        infn = os.path.join(indir, 'survey-ccds-1.fits.gz')
        outfn = os.path.join(surveydir, 'survey-ccds-1.kd.fits')
        create_kdtree(infn, outfn, False)

        os.environ['TYCHO2_KD_DIR'] = surveydir
        outdir = 'out-testcase6-kd'
        main(args=[
            '--brick', '1102p240', '--zoom', '500', '600', '650', '750',
            '--force-all', '--no-write', '--no-wise', '--no-gaia',
            '--survey-dir', surveydir, '--outdir', outdir
        ])
        fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
        assert (os.path.exists(fn))
        T = fits_table(fn)
        assert (len(T) == 2)
        # Since there is a Tycho-2 star in the blob, forced to be PSF.
        assert (T.type[0].strip() == 'PSF')
        assert (T.type[1].strip() == 'PSF')
        # There is a Tycho-2 star in the blob.
        I = np.flatnonzero(T.ref_cat == 'T2')
        assert (len(I) == 1)
        assert (T.ref_id[I][0] == 1909016711)

        cat = read_fits_catalog(T)
        assert (len(cat) == 2)
        assert (isinstance(cat[0], PointSource))
        assert (isinstance(cat[1], PointSource))
        cat, ivs = read_fits_catalog(T, invvars=True)
        assert (len(cat) == 2)
        assert (isinstance(cat[0], PointSource))
        assert (isinstance(cat[1], PointSource))
        cat2 = Catalog(*cat)
        assert (len(ivs) == len(cat2.getParams()))

        # test --fit-on-coadds
        outdir = 'out-testcase6-coadds'
        main(args=[
            '--brick', '1102p240', '--zoom', '500', '600', '650', '750',
            '--force-all', '--no-write', '--no-wise', '--no-gaia',
            '--survey-dir', surveydir, '--fit-on-coadds', '--outdir', outdir
        ])

        fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
        assert (os.path.exists(fn))
        T = fits_table(fn)
        assert (len(T) == 2)
        # Since there is a Tycho-2 star in the blob, forced to be PSF.
        assert (T.type[0].strip() == 'PSF')
        assert (T.type[1].strip() == 'PSF')
        # There is a Tycho-2 star in the blob.
        I = np.flatnonzero(T.ref_cat == 'T2')
        assert (len(I) == 1)
        assert (T.ref_id[I][0] == 1909016711)
        del os.environ['TYCHO2_KD_DIR']

        # test --skip-coadds
        r = main(args=[
            '--brick', '1102p240', '--zoom', '500', '600', '650', '750',
            '--force-all', '--no-write', '--no-wise', '--no-gaia',
            '--survey-dir', surveydir, '--outdir', outdir, '--skip-coadd'
        ])
        assert (r == 0)

        # test --skip
        r = main(args=[
            '--brick', '1102p240', '--zoom', '500', '600', '650', '750',
            '--force-all', '--no-write', '--no-wise', '--no-gaia',
            '--survey-dir', surveydir, '--outdir', outdir, '--skip'
        ])
        assert (r == 0)

        # NothingToDoError (neighbouring brick)
        r = main(args=[
            '--brick', '1102p240', '--zoom', '0', '100', '0', '100',
            '--force-all', '--no-write', '--no-wise', '--no-gaia',
            '--survey-dir', surveydir, '--outdir', outdir
        ])
        assert (r == 0)

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase9')

    # Test for some get_tractor_image kwargs
    survey = LegacySurveyData(surveydir)
    fakebrick = BrickDuck(9.1228, 3.3975, 'quack')
    wcs = wcs_for_brick(fakebrick, W=100, H=100)
    ccds = survey.ccds_touching_wcs(wcs)
    ccd = ccds[0]
    im = survey.get_image_object(ccd)
    H, W = wcs.shape
    targetrd = np.array([
        wcs.pixelxy2radec(x, y)
        for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
    ])
    tim = im.get_tractor_image(radecpoly=targetrd)
    assert (tim.getImage() is not None)
    assert (tim.getInvError() is not None)
    assert (tim.dq is not None)
    tim2 = im.get_tractor_image(radecpoly=targetrd, pixels=False)
    assert (np.all(tim2.getImage() == 0.))
    tim4 = im.get_tractor_image(radecpoly=targetrd, invvar=False)
    u = np.unique(tim4.inverr)
    assert (len(u) == 1)
    u = u[0]
    target = tim4.zpscale / tim4.sig1
    assert (np.abs(u / target - 1.) < 0.001)
    tim3 = im.get_tractor_image(radecpoly=targetrd, invvar=False, dq=False)
    assert (not hasattr(tim3, 'dq'))
    tim5 = im.get_tractor_image(radecpoly=targetrd, gaussPsf=True)
    print(tim5.getPsf())
    assert (isinstance(tim5.getPsf(), GaussianMixturePSF))

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase12')
    os.environ['TYCHO2_KD_DIR'] = surveydir
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'
    os.environ['UNWISE_MODEL_SKY_DIR'] = os.path.join(surveydir, 'images',
                                                      'unwise-mod')
    #python legacypipe/runbrick.py --radec  --width 100 --height 100 --outdir dup5b --survey-dir test/testcase12 --force-all --no-wise
    unwdir = os.path.join(surveydir, 'images', 'unwise')
    main(args=[
        '--radec', '346.684', '12.791', '--width', '100', '--height', '100',
        '--no-wise-ceres', '--unwise-dir', unwdir, '--survey-dir', surveydir,
        '--outdir', 'out-testcase12', '--skip-coadd', '--force-all'
    ])

    # --plots for stage_wise_forced
    main(args=[
        '--radec', '346.684', '12.791', '--width', '100', '--height', '100',
        '--no-wise-ceres', '--unwise-dir', unwdir, '--survey-dir', surveydir,
        '--outdir', 'out-testcase12', '--stage', 'wise_forced', '--plots'
    ])
    del os.environ['GAIA_CAT_DIR']
    del os.environ['GAIA_CAT_VER']
    del os.environ['TYCHO2_KD_DIR']
    del os.environ['UNWISE_MODEL_SKY_DIR']

    M = fitsio.read(
        'out-testcase12/coadd/cus/custom-346684p12791/legacysurvey-custom-346684p12791-maskbits.fits.fz'
    )
    # Count masked & unmasked bits (the cluster splits this 100x100 field)
    from collections import Counter
    c = Counter(M.ravel())
    from legacypipe.bits import MASKBITS
    assert (c[0] >= 4000)
    assert (c[MASKBITS['CLUSTER']] >= 4000)

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase9')
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'
    os.environ['LARGEGALAXIES_CAT'] = os.path.join(surveydir,
                                                   'sga-sub.kd.fits')
    main(args=[
        '--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
        '--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir',
        surveydir, '--outdir', 'out-testcase9', '--skip', '--force-all',
        '--ps', 'tc9-ps.fits', '--ps-t0',
        str(int(time.time()))
    ])
    # (omit --force-all --no-write... reading from pickles below!)

    # Test with --apodize
    main(args=[
        '--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
        '--old-calibs-ok', '--no-wise', '--force-all', '--no-write',
        '--survey-dir', surveydir, '--outdir', 'out-testcase9-ap', '--apodize'
    ])

    main(args=[
        '--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
        '--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir',
        surveydir, '--outdir', 'out-testcase9', '--plots', '--stage', 'halos'
    ])

    main(args=[
        '--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
        '--old-calibs-ok', '--no-wise-ceres', '--no-wise', '--survey-dir',
        surveydir, '--outdir', 'out-testcase9-coadds', '--stage',
        'image_coadds', '--blob-image'
    ])

    T = fits_table(
        'out-testcase9/tractor/cus/tractor-custom-009122p03397.fits')
    assert (len(T) == 4)
    # Gaia star becomes a DUP!
    assert (np.sum([t == 'DUP' for t in T.type]) == 1)
    # LSLGA galaxy exists!
    Igal = np.flatnonzero([r == 'L3' for r in T.ref_cat])
    assert (len(Igal) == 1)
    assert (np.all(T.ref_id[Igal] > 0))
    assert (T.type[Igal[0]] == 'SER')

    # --brick and --zoom rather than --radec --width --height
    main(args=[
        '--survey-dir', surveydir, '--outdir', 'out-testcase9b', '--zoom',
        '1950', '2050', '340', '440', '--brick', '0091p035', '--force-all'
    ])

    # test forced phot??
    shutil.copy('test/testcase9/survey-bricks.fits.gz', 'out-testcase9b')

    forced_main(args=[
        '--survey-dir', surveydir, '--no-ceres', '--catalog-dir',
        'out-testcase9b', '372546', 'N26', 'forced1.fits'
    ])
    assert (os.path.exists('forced1.fits'))
    _ = fits_table('forced1.fits')
    # ... more tests...!

    forced_main(args=[
        '--survey-dir', surveydir, '--no-ceres', '--catalog-dir',
        'out-testcase9b', '--derivs', '--threads', '2', '--apphot', '372547',
        'N26', 'forced2.fits'
    ])
    assert (os.path.exists('forced2.fits'))
    _ = fits_table('forced2.fits')

    forced_main(args=[
        '--survey-dir', surveydir, '--no-ceres', '--catalog-dir',
        'out-testcase9b', '--agn', '257266', 'S21', 'forced3.fits'
    ])
    assert (os.path.exists('forced3.fits'))
    _ = fits_table('forced3.fits')

    if ceres:
        forced_main(args=[
            '--survey-dir', surveydir, '--catalog-dir', 'out-testcase9b',
            '--derivs', '--threads', '2', '--apphot', '372546', 'N26',
            'forced4.fits'
        ])
        assert (os.path.exists('forced4.fits'))
        _ = fits_table('forced4.fits')

    # Test cache_dir
    with tempfile.TemporaryDirectory() as cachedir, \
        tempfile.TemporaryDirectory() as tempsurveydir:
        files = []
        for dirpath, _, filenames in os.walk(surveydir):
            for fn in filenames:
                path = os.path.join(dirpath, fn)
                relpath = os.path.relpath(path, surveydir)
                files.append(relpath)
        # cache or no?
        files.sort()
        files_cache = files[::2]
        files_nocache = files[1::2]
        # Survey-ccds *must* be in nocache.
        fn = 'survey-ccds-1.kd.fits'
        if fn in files_cache:
            files_cache.remove(fn)
            files_nocache.append(fn)

        for fn in files_cache:
            src = os.path.join(surveydir, fn)
            dst = os.path.join(cachedir, fn)
            trymakedirs(dst, dir=True)
            print('Copy', src, dst)
            shutil.copy(src, dst)

        for fn in files_nocache:
            src = os.path.join(surveydir, fn)
            dst = os.path.join(tempsurveydir, fn)
            trymakedirs(dst, dir=True)
            print('Copy', src, dst)
            shutil.copy(src, dst)

        main(args=[
            '--radec', '9.1228', '3.3975', '--width', '100', '--height', '100',
            '--no-wise', '--survey-dir', tempsurveydir, '--cache-dir',
            cachedir, '--outdir', 'out-testcase9cache', '--force-all'
        ])

    del os.environ['GAIA_CAT_DIR']
    del os.environ['GAIA_CAT_VER']
    del os.environ['LARGEGALAXIES_CAT']

    # if ceres:
    #     surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
    #     main(args=['--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
    #                '--no-wise', '--force-all', '--no-write', '--ceres',
    #                '--survey-dir', surveydir,
    #                '--outdir', 'out-testcase3-ceres',
    #                '--no-depth-cut'])

    # MzLS + BASS data
    # python legacypipe/runbrick.py --run north --brick 1773p595 --zoom 1300 1500 700 900 --survey-dir dr9-north -s coadds
    # fitscopy coadd/177/1773p595/legacysurvey-1773p595-ccds.fits"[#row<3 || #row==12]" cx.fits
    # python legacyanalysis/create_testcase.py cx.fits test/mzlsbass2 1773p595 --survey-dir dr9-north/ --fpack
    surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass2')
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir2, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'
    main(args=[
        '--brick', '1773p595', '--zoom', '1300', '1500', '700', '900',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir2,
        '--outdir', 'out-mzlsbass2'
    ])

    T = fits_table('out-mzlsbass2/tractor/177/tractor-1773p595.fits')
    assert (np.sum(T.ref_cat == 'G2') == 3)
    assert (np.sum(T.ref_id > 0) == 3)

    # Test --max-blobsize, --checkpoint, --bail-out

    outdir = 'out-mzlsbass2b'
    chk = 'checkpoint-mzb2b.p'
    if os.path.exists(chk):
        os.unlink(chk)
    main(args=[
        '--brick', '1773p595', '--zoom', '1300', '1500', '700', '900',
        '--no-wise', '--force-all', '--stage', 'fitblobs', '--write-stage',
        'srcs', '--survey-dir', surveydir2, '--outdir', outdir, '--checkpoint',
        chk, '--nblobs', '3'
    ])
    # err... --max-blobsize does not result in bailed-out blobs masked,
    # because it treats large blobs as *completed*...
    #'--max-blobsize', '3000',

    outdir = 'out-mzlsbass2c'
    main(args=[
        '--brick', '1773p595', '--zoom', '1300', '1500', '700', '900',
        '--no-wise', '--force-all', '--survey-dir', surveydir2, '--outdir',
        outdir, '--bail-out', '--checkpoint', chk, '--no-write'
    ])

    del os.environ['GAIA_CAT_DIR']
    del os.environ['GAIA_CAT_VER']

    M = fitsio.read(
        os.path.join(outdir, 'coadd', '177', '1773p595',
                     'legacysurvey-1773p595-maskbits.fits.fz'))
    assert (np.sum((M & MASKBITS['BAILOUT']) > 0) >= 1000)

    # Test RexGalaxy

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
    outdir = 'out-testcase6-rex'
    the_args = [
        '--brick',
        '1102p240',
        '--zoom',
        '500',
        '600',
        '650',
        '750',
        '--force-all',
        '--no-write',
        '--no-wise',
        '--skip-calibs',
        #'--rex', #'--plots',
        '--survey-dir',
        surveydir,
        '--outdir',
        outdir
    ]
    print('python legacypipe/runbrick.py', ' '.join(the_args))
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'
    main(args=the_args)
    fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 2)
    print('Types:', T.type)
    # Since there is a Tycho-2 star in the blob, forced to be PSF.
    assert (T.type[0].strip() == 'PSF')
    cmd = (
        '(cd %s && sha256sum -c %s)' %
        (outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum')))
    print(cmd)
    rtn = os.system(cmd)
    assert (rtn == 0)

    # Test with a Tycho-2 star in the blob.

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
    outdir = 'out-testcase6'
    main(args=[
        '--brick', '1102p240', '--zoom', '500', '600', '650', '750',
        '--force-all', '--no-write', '--no-wise', '--survey-dir', surveydir,
        '--outdir', outdir
    ])
    fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 2)
    print('Types:', T.type)
    # Since there is a Tycho-2 star in the blob, forced to be PSF.
    assert (T.type[0].strip() == 'PSF')
    del os.environ['GAIA_CAT_DIR']
    del os.environ['GAIA_CAT_VER']

    # Test that we can run splinesky calib if required...

    from legacypipe.decam import DecamImage
    DecamImage.splinesky_boxsize = 128

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')

    survey = LegacySurveyData(surveydir)
    # get brick by id
    brickid = 473357
    brick = survey.get_brick(brickid)
    assert (brick.brickname == '1867p255')
    assert (brick.brickid == brickid)

    outdir = 'out-testcase4'
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'

    fn = os.path.join(surveydir, 'calib', 'sky-single', 'decam', 'CP',
                      'V4.8.2', 'CP20170315', 'c4d_170316_062107_ooi_z_ls9',
                      'c4d_170316_062107_ooi_z_ls9-N2-splinesky.fits')
    if os.path.exists(fn):
        os.unlink(fn)

    main(args=[
        '--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
        '--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
        os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
        os.path.join(surveydir, 'images',
                     'unwise-tr'), '--blob-image', '--no-hybrid-psf',
        '--survey-dir', surveydir, '--outdir', outdir, '-v', '--no-wise-ceres'
    ])
    print('Checking for calib file', fn)
    assert (os.path.exists(fn))

    # Test with blob-masking when creating sky calib.
    os.unlink(fn)

    main(args=[
        '--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
        '--force-all', '--no-write', '--coadd-bw', '--blob-mask-dir',
        surveydir, '--survey-dir', surveydir, '--stage', 'image_coadds',
        '--outdir', 'out-testcase4b', '--plots'
    ])
    print('Checking for calib file', fn)
    assert (os.path.exists(fn))

    if ceres:
        main(args=[
            '--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
            '--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
            os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
            os.path.join(surveydir, 'images', 'unwise-tr'), '--survey-dir',
            surveydir, '--outdir', outdir
        ])
    if psfex:
        # Check that we can regenerate PsfEx files if necessary.
        fn = os.path.join(surveydir, 'calib', 'psfex', 'decam', 'CP', 'V4.8.2',
                          'CP20170315',
                          'c4d_170316_062107_ooi_z_ls9-psfex.fits')
        if os.path.exists(fn):
            os.unlink(fn)

        main(args=[
            '--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
            '--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
            os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
            os.path.join(surveydir, 'images', 'unwise-tr'), '--blob-image',
            '--survey-dir', surveydir, '--outdir', outdir, '-v'
        ])
        print('After generating PsfEx calib:')
        os.system('find %s' % (os.path.join(surveydir, 'calib')))

    # Wrap-around, hybrid PSF
    surveydir = os.path.join(os.path.dirname(__file__), 'testcase8')
    outdir = 'out-testcase8'
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'

    main(args=[
        '--brick',
        '1209p050',
        '--zoom',
        '720',
        '1095',
        '3220',
        '3500',
        '--force-all',
        '--no-write',
        '--no-wise',  #'--plots',
        '--survey-dir',
        surveydir,
        '--outdir',
        outdir
    ])

    # Test with a Tycho-2 star + another saturated star in the blob.

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase7')
    outdir = 'out-testcase7'
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'
    main(args=[
        '--brick',
        '1102p240',
        '--zoom',
        '250',
        '350',
        '1550',
        '1650',
        '--force-all',
        '--no-write',
        '--no-wise',  #'--plots',
        '--survey-dir',
        surveydir,
        '--outdir',
        outdir
    ])
    del os.environ['GAIA_CAT_DIR']
    del os.environ['GAIA_CAT_VER']
    fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 4)

    # Check skipping blobs outside the brick's unique area.
    # (this now doesn't detect any sources at all, reasonably)
    # surveydir = os.path.join(os.path.dirname(__file__), 'testcase5')
    # outdir = 'out-testcase5'
    #
    # fn = os.path.join(outdir, 'tractor', '186', 'tractor-1867p255.fits')
    # if os.path.exists(fn):
    #     os.unlink(fn)
    #
    # main(args=['--brick', '1867p255', '--zoom', '0', '150', '0', '150',
    #            '--force-all', '--no-write', '--coadd-bw',
    #            '--survey-dir', surveydir,
    #            '--early-coadds',
    #            '--outdir', outdir] + extra_args)
    #
    # assert(os.path.exists(fn))
    # T = fits_table(fn)
    # assert(len(T) == 1)

    # Custom RA,Dec; blob ra,dec.
    surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
    outdir = 'out-testcase4b'
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    # Catalog written with one entry (--blobradec)
    fn = os.path.join(outdir, 'tractor', 'cus',
                      'tractor-custom-186743p25461.fits')
    if os.path.exists(fn):
        os.unlink(fn)
    main(args=[
        '--radec', '186.743965', '25.461788', '--width', '250', '--height',
        '250', '--force-all', '--no-write', '--no-wise', '--blobradec',
        '186.740369', '25.453855', '--survey-dir', surveydir, '--outdir',
        outdir
    ])

    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 1)

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
    outdir = 'out-testcase3'
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia')
    os.environ['GAIA_CAT_VER'] = '2'
    checkpoint_fn = os.path.join(outdir, 'checkpoint.pickle')
    if os.path.exists(checkpoint_fn):
        os.unlink(checkpoint_fn)
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', outdir, '--checkpoint', checkpoint_fn,
        '--checkpoint-period', '1', '--threads', '2'
    ])

    # Read catalog into Tractor sources to test read_fits_catalog
    survey = LegacySurveyData(survey_dir=outdir)
    fn = survey.find_file('tractor', brick='2447p120')
    print('Checking', fn)
    T = fits_table(fn)
    cat = read_fits_catalog(T)
    print('Read catalog:', cat)

    assert (len(cat) == 2)
    src = cat[1]
    print('Source0', src)
    from tractor.sersic import SersicGalaxy
    assert (type(src) in [DevGalaxy, SersicGalaxy])
    assert (np.abs(src.pos.ra - 244.77973) < 0.00001)
    assert (np.abs(src.pos.dec - 12.07234) < 0.00002)
    src = cat[0]
    print('Source1', src)
    assert (type(src) == PointSource)
    assert (np.abs(src.pos.ra - 244.77828) < 0.00001)
    assert (np.abs(src.pos.dec - 12.07250) < 0.00001)

    # Check that we can run again, using that checkpoint file.
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', outdir, '--checkpoint', checkpoint_fn,
        '--checkpoint-period', '1', '--threads', '2'
    ])
    # Assert...... something?

    # Test --checkpoint without --threads
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', outdir, '--checkpoint', checkpoint_fn,
        '--checkpoint-period', '1'
    ])

    # From Kaylan's Bootes pre-DR4 run
    # surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass3')
    # main(args=['--brick', '2173p350', '--zoom', '100', '200', '100', '200',
    #            '--no-wise', '--force-all', '--no-write',
    #            '--survey-dir', surveydir2,
    #            '--outdir', 'out-mzlsbass3'] + extra_args)

    # With plots!
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', 'out-testcase3', '--plots', '--nblobs', '1'
    ])
예제 #15
0
def main(survey=None, opt=None):
    '''Driver function for forced photometry of individual DECam images.
    '''
    if opt is None:
        parser = get_parser()
        opt = parser.parse_args()

    Time.add_measurement(MemMeas)
    t0 = Time()

    if os.path.exists(opt.outfn):
        print('Ouput file exists:', opt.outfn)
        sys.exit(0)

    if not opt.forced:
        opt.apphot = True

    zoomslice = None
    if opt.zoom is not None:
        (x0,x1,y0,y1) = opt.zoom
        zoomslice = (slice(y0,y1), slice(x0,x1))

    ps = None
    if opt.plots is not None:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence(opt.plots)

    # Try parsing filename as exposure number.
    try:
        expnum = int(opt.filename)
        opt.filename = None
    except:
        # make this 'None' for survey.find_ccds()
        expnum = None

    # Try parsing HDU number
    try:
        opt.hdu = int(opt.hdu)
        ccdname = None
    except:
        ccdname = opt.hdu
        opt.hdu = -1

    if survey is None:
        survey = LegacySurveyData()

    if opt.filename is not None and opt.hdu >= 0:
        # Read metadata from file
        T = exposure_metadata([opt.filename], hdus=[opt.hdu])
        print('Metadata:')
        T.about()
    else:
        # Read metadata from survey-ccds.fits table
        T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
        print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
        if opt.hdu >= 0:
            T.cut(T.image_hdu == opt.hdu)
            print(len(T), 'with HDU', opt.hdu)
        if opt.filename is not None:
            T.cut(np.array([f.strip() == opt.filename for f in T.image_filename]))
            print(len(T), 'with filename', opt.filename)
        assert(len(T) == 1)

    ccd = T[0]
    im = survey.get_image_object(ccd)
    tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True,
                               constant_invvar=opt.constant_invvar)
    print('Got tim:', tim)

    print('Read image:', Time()-t0)

    if opt.catfn in ['DR1', 'DR2', 'DR3']:

        margin = 20
        TT = []
        chipwcs = tim.subwcs
        bricks = bricks_touching_wcs(chipwcs, survey=survey)
        for b in bricks:
            # there is some overlap with this brick... read the catalog.
            fn = survey.find_file('tractor', brick=b.brickname)
            if not os.path.exists(fn):
                print('WARNING: catalog', fn, 'does not exist.  Skipping!')
                continue
            print('Reading', fn)
            T = fits_table(fn)
            ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec)
            W,H = chipwcs.get_width(), chipwcs.get_height()
            I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) *
                               (yy >= -margin) * (yy <= (H+margin)))
            T.cut(I)
            print('Cut to', len(T), 'sources within image + margin')
            # print('Brick_primary:', np.unique(T.brick_primary))
            T.cut(T.brick_primary)
            print('Cut to', len(T), 'on brick_primary')
            T.cut((T.out_of_bounds == False) * (T.left_blob == False))
            print('Cut to', len(T), 'on out_of_bounds and left_blob')
            if len(T):
                TT.append(T)
        if len(TT) == 0:
            print('No sources to photometer.')
            return 0
        T = merge_tables(TT, columns='fillzero')
        T._header = TT[0]._header
        del TT

        # Fix up various failure modes:
        # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
        # -> convert to EXP
        I = np.flatnonzero(np.array([((t.type == 'COMP') and
                                      (not np.isfinite(t.shapedev_r)))
                                     for t in T]))
        if len(I):
            print('Converting', len(I), 'bogus COMP galaxies to EXP')
            for i in I:
                T.type[i] = 'EXP'

        # Same thing with the exp component.
        # -> convert to DEV
        I = np.flatnonzero(np.array([((t.type == 'COMP') and
                                      (not np.isfinite(t.shapeexp_r)))
                                     for t in T]))
        if len(I):
            print('Converting', len(I), 'bogus COMP galaxies to DEV')
            for i in I:
                T.type[i] = 'DEV'

        if opt.write_cat:
            T.writeto(opt.write_cat)
            print('Wrote catalog to', opt.write_cat)

    else:
        T = fits_table(opt.catfn)

    surveydir = survey.get_survey_dir()
    del survey
        
    cat = read_fits_catalog(T)
    # print('Got cat:', cat)

    print('Read catalog:', Time()-t0)

    print('Forced photom...')
    opti = None
    forced_kwargs = {}
    if opt.ceres:
        from tractor.ceres_optimizer import CeresOptimizer
        B = 8
        opti = CeresOptimizer(BW=B, BH=B)
        #forced_kwargs.update(verbose=True)

    for src in cat:
        # Limit sizes of huge models
        from tractor.galaxy import ProfileGalaxy
        if isinstance(src, ProfileGalaxy):
            px,py = tim.wcs.positionToPixel(src.getPosition())
            h = src._getUnitFluxPatchSize(tim, px, py, tim.modelMinval)
            MAXHALF = 128
            if h > MAXHALF:
                print('halfsize', h,'for',src,'-> setting to',MAXHALF)
                src.halfsize = MAXHALF
        
    tr = Tractor([tim], cat, optimizer=opti)
    tr.freezeParam('images')
    for src in cat:
        src.freezeAllBut('brightness')
        src.getBrightness().freezeAllBut(tim.band)
    disable_galaxy_cache()
        
    F = fits_table()
    F.brickid   = T.brickid
    F.brickname = T.brickname
    F.objid     = T.objid

    F.filter  = np.array([tim.band]               * len(T))
    F.mjd     = np.array([tim.primhdr['MJD-OBS']] * len(T))
    F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32)

    ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
    F.x = (x-1).astype(np.float32)
    F.y = (y-1).astype(np.float32)

    if opt.forced:
        if opt.plots is None:
            forced_kwargs.update(wantims=False)

        R = tr.optimize_forced_photometry(variance=True, fitstats=True,
                                          shared_params=False, priors=False, **forced_kwargs)

        if opt.plots:
            (data,mod,ie,chi,roi) = R.ims1[0]

            ima = tim.ima
            imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5)
            plt.clf()
            plt.imshow(data, **ima)
            plt.title('Data: %s' % tim.name)
            ps.savefig()

            plt.clf()
            plt.imshow(mod, **ima)
            plt.title('Model: %s' % tim.name)
            ps.savefig()

            plt.clf()
            plt.imshow(chi, **imchi)
            plt.title('Chi: %s' % tim.name)
            ps.savefig()

        F.flux = np.array([src.getBrightness().getFlux(tim.band)
                           for src in cat]).astype(np.float32)
        F.flux_ivar = R.IV.astype(np.float32)

        F.fracflux = R.fitstats.profracflux.astype(np.float32)
        F.rchi2    = R.fitstats.prochi2    .astype(np.float32)

        print('Forced photom:', Time()-t0)

        
    if opt.apphot:
        import photutils

        img = tim.getImage()
        ie = tim.getInvError()
        with np.errstate(divide='ignore'):
            imsigma = 1. / ie
        imsigma[ie == 0] = 0.

        apimg = []
        apimgerr = []

        # Aperture photometry locations
        xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
        apxy = xxyy - 1.

        apertures = apertures_arcsec / tim.wcs.pixel_scale()
        print('Apertures:', apertures, 'pixels')

        for rad in apertures:
            aper = photutils.CircularAperture(apxy, rad)
            p = photutils.aperture_photometry(img, aper, error=imsigma)
            apimg.append(p.field('aperture_sum'))
            apimgerr.append(p.field('aperture_sum_err'))
        ap = np.vstack(apimg).T
        ap[np.logical_not(np.isfinite(ap))] = 0.
        F.apflux = ap.astype(np.float32)
        ap = 1./(np.vstack(apimgerr).T)**2
        ap[np.logical_not(np.isfinite(ap))] = 0.
        F.apflux_ivar = ap.astype(np.float32)
        print('Aperture photom:', Time()-t0)

    program_name = sys.argv[0]
    version_hdr = get_version_header(program_name, surveydir)
    filename = getattr(ccd, 'image_filename')
    if filename is None:
        # HACK -- print only two directory names + filename of CPFILE.
        fname = os.path.basename(im.imgfn)
        d = os.path.dirname(im.imgfn)
        d1 = os.path.basename(d)
        d = os.path.dirname(d)
        d2 = os.path.basename(d)
        filename = os.path.join(d2, d1, fname)
        print('Trimmed filename to', filename)
    version_hdr.add_record(dict(name='CPFILE', value=filename, comment='CP file'))
    version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
    version_hdr.add_record(dict(name='CAMERA', value=ccd.camera, comment='Camera'))
    version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
    version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
    version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
    version_hdr.add_record(dict(name='EXPOSURE',
                                value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname),
                                comment='Name of this image'))

    keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV',
            'INSTRUME']
    for key in keys:
        if key in tim.primhdr:
            version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))

    hdr = fitsio.FITSHDR()
    units = {'exptime':'sec', 'flux':'nanomaggy', 'flux_ivar':'1/nanomaggy^2'}
    columns = F.get_columns()
    for i,col in enumerate(columns):
        if col in units:
            hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col]))

    outdir = os.path.dirname(opt.outfn)
    if len(outdir):
        trymakedirs(outdir)
    fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
    F.writeto(opt.outfn, header=hdr, append=True)
    print('Wrote', opt.outfn)
    
    if opt.save_model or opt.save_data:
        hdr = fitsio.FITSHDR()
        tim.getWcs().wcs.add_to_header(hdr)
    if opt.save_model:
        print('Getting model image...')
        mod = tr.getModelImage(tim)
        fitsio.write(opt.save_model, mod, header=hdr, clobber=True)
        print('Wrote', opt.save_model)
    if opt.save_data:
        fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
        print('Wrote', opt.save_data)
    
    print('Finished forced phot:', Time()-t0)
    return 0
예제 #16
0
def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--catalog', help='Catalog to render')
    parser.add_argument('--brick-coadd', help='Produce a coadd of the images overlapping the given brickname.')
    parser.add_argument('--ccds', help='Use this table of CCDs')
    parser.add_argument('--brick-wcs', help='File containing a WCS header describing the coadd WCS to render.')
    parser.add_argument('--brick-wcs-ext', type=int, help='FITS file extension containing a WCS header describing the coadd WCS to render.')
    parser.add_argument('--outlier-mask-brick', help='Comma-separated list of bricknames from which outlier masks should be read.')
    parser.add_argument('--out', help='Filename pattern ("BAND" will be replaced by band name) of output images.')
    parser.add_argument('--resid', help='Filename pattern ("BAND" will be replaced by band name) of residual images.')
    parser.add_argument('--jpeg', help='Write RGB image to this filename')
    parser.add_argument('--resid-jpeg', help='Write RGB residual image to this filename')
    opt = parser.parse_args()

    if opt.catalog is None:
        print('Need catalog!')
        return -1
    cat = fits_table(opt.catalog)
    
    if opt.ccds is None:
        if opt.brick_coadd is None:
            print('Need brick catalog!')
            return -1
        brickname = opt.brick_coadd

    survey = LegacySurveyData()

    if opt.brick_wcs is None:
        print('FIXME')
        return -1
    else:
        wcs = Tan(opt.brick_wcs, opt.brick_wcs_ext)
        
    tcat = read_fits_catalog(cat)

    if opt.ccds:
        ccdfn = opt.ccds
    else:
        ccdfn = survey.find_file('ccds-table', brick=brickname)
    print('Reading', ccdfn)
    ccds = fits_table(ccdfn)

    H,W = wcs.shape
    targetrd = np.array([wcs.pixelxy2radec(x,y) for x,y in
                         [(1,1),(W,1),(W,H),(1,H),(1,1)]])

    tims = []
    for ccd in ccds:
        im = survey.get_image_object(ccd)
        #slc = slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1)
        #tim = im.get_tractor_image(slc=slc)
        tim = im.get_tractor_image(radecpoly=targetrd)
        print('Read', tim)
        tims.append(tim)

    if opt.outlier_mask_brick is not None:
        bricks = opt.outlier_mask_brick.split(',')
        for b in bricks:
            print('Reading outlier mask for brick', b,
                  ':', survey.find_file('outliers_mask', brick=b, output=False))
            ok = read_outlier_mask_file(survey, tims, b,
                                        subimage=True, output=False)

        
    tr = Tractor(tims, tcat)
    mods = list(tr.getModelImages())

    bands = 'grz'

    def write_model(band, cowimg=None, cowmod=None, **kwargs):
        if cowmod is None:
            print('No model for', band)
            return
        outfn = opt.out.replace('BAND', band)
        fitsio.write(outfn, cowmod, clobber=True)
        print('Wrote model for', band, 'to', outfn)
        if opt.resid:
            outfn = opt.resid.replace('BAND', band)
            fitsio.write(outfn, cowimg - cowmod, clobber=True)
            print('Wrote resid for', band, 'to', outfn)

    C = make_coadds(tims, bands, wcs, mods=mods,
                    callback=write_model)
    if opt.jpeg:
        from legacypipe.survey import get_rgb
        import pylab as plt
        plt.imsave(opt.jpeg, get_rgb(C.comods, bands), origin='lower')
    if opt.resid_jpeg:
        from legacypipe.survey import get_rgb
        import pylab as plt
        plt.imsave(opt.resid_jpeg,
                   get_rgb([im-mod for im,mod in zip(C.coimgs, C.comods)], bands),
                   origin='lower')
예제 #17
0
def main(outfn='ccds-annotated.fits', ccds=None):
    survey = LegacySurveyData(ccds=ccds)
    if ccds is None:
        ccds = survey.get_ccds()

    # File from the "observing" svn repo:
    # https://desi.lbl.gov/svn/decam/code/observing/trunk
    tiles = fits_table('decam-tiles_obstatus.fits')

    I = survey.photometric_ccds(ccds)
    ccds.photometric = np.zeros(len(ccds), bool)
    ccds.photometric[I] = True

    I = survey.apply_blacklist(ccds)
    ccds.blacklist_ok = np.zeros(len(ccds), bool)
    ccds.blacklist_ok[I] = True

    ccds.good_region = np.empty((len(ccds), 4), np.int16)
    ccds.good_region[:,:] = -1

    ccds.ra0  = np.zeros(len(ccds), np.float64)
    ccds.dec0 = np.zeros(len(ccds), np.float64)
    ccds.ra1  = np.zeros(len(ccds), np.float64)
    ccds.dec1 = np.zeros(len(ccds), np.float64)
    ccds.ra2  = np.zeros(len(ccds), np.float64)
    ccds.dec2 = np.zeros(len(ccds), np.float64)
    ccds.ra3  = np.zeros(len(ccds), np.float64)
    ccds.dec3 = np.zeros(len(ccds), np.float64)

    ccds.dra  = np.zeros(len(ccds), np.float32)
    ccds.ddec = np.zeros(len(ccds), np.float32)
    ccds.ra_center  = np.zeros(len(ccds), np.float64)
    ccds.dec_center = np.zeros(len(ccds), np.float64)

    ccds.sig1 = np.zeros(len(ccds), np.float32)

    ccds.meansky = np.zeros(len(ccds), np.float32)
    ccds.stdsky  = np.zeros(len(ccds), np.float32)
    ccds.maxsky  = np.zeros(len(ccds), np.float32)
    ccds.minsky  = np.zeros(len(ccds), np.float32)

    ccds.pixscale_mean = np.zeros(len(ccds), np.float32)
    ccds.pixscale_std  = np.zeros(len(ccds), np.float32)
    ccds.pixscale_max  = np.zeros(len(ccds), np.float32)
    ccds.pixscale_min  = np.zeros(len(ccds), np.float32)

    ccds.psfnorm_mean = np.zeros(len(ccds), np.float32)
    ccds.psfnorm_std  = np.zeros(len(ccds), np.float32)
    ccds.galnorm_mean = np.zeros(len(ccds), np.float32)
    ccds.galnorm_std  = np.zeros(len(ccds), np.float32)

    gaussgalnorm = np.zeros(len(ccds), np.float32)

    # 2nd moments
    ccds.psf_mx2 = np.zeros(len(ccds), np.float32)
    ccds.psf_my2 = np.zeros(len(ccds), np.float32)
    ccds.psf_mxy = np.zeros(len(ccds), np.float32)
    #
    ccds.psf_a = np.zeros(len(ccds), np.float32)
    ccds.psf_b = np.zeros(len(ccds), np.float32)
    ccds.psf_theta = np.zeros(len(ccds), np.float32)
    ccds.psf_ell   = np.zeros(len(ccds), np.float32)

    ccds.humidity = np.zeros(len(ccds), np.float32)
    ccds.outtemp  = np.zeros(len(ccds), np.float32)

    ccds.tileid   = np.zeros(len(ccds), np.int32)
    ccds.tilepass = np.zeros(len(ccds), np.uint8)
    ccds.tileebv  = np.zeros(len(ccds), np.float32)

    plvers = []

    for iccd,ccd in enumerate(ccds):
        im = survey.get_image_object(ccd)
        print('Reading CCD %i of %i:' % (iccd+1, len(ccds)), im)

        X = im.get_good_image_subregion()
        for i,x in enumerate(X):
            if x is not None:
                ccds.good_region[iccd,i] = x

        W,H = ccd.width, ccd.height

        psf = None
        wcs = None
        sky = None
        try:
            tim = im.get_tractor_image(pixPsf=True, splinesky=True, subsky=False,
                                       pixels=False, dq=False, invvar=False)

        except:
            import traceback
            traceback.print_exc()
            plvers.append('')
            continue

        if tim is None:
            plvers.append('')
            continue

        psf = tim.psf
        wcs = tim.wcs.wcs
        sky = tim.sky
        hdr = tim.primhdr

        # print('Got PSF', psf)
        # print('Got sky', type(sky))
        # print('Got WCS', wcs)

        ccds.humidity[iccd] = hdr.get('HUMIDITY')
        ccds.outtemp[iccd]  = hdr.get('OUTTEMP')

        ccds.sig1[iccd] = tim.sig1
        plvers.append(tim.plver)

        # parse 'DECaLS_15150_r' to get tile number
        obj = ccd.object.strip()
        words = obj.split('_')
        tile = None
        if len(words) == 3 and words[0] == 'DECaLS':
            try:
                tileid = int(words[1])
                tile = tiles[tileid - 1]
                if tile.tileid != tileid:
                    I = np.flatnonzero(tile.tileid == tileid)
                    tile = tiles[I[0]]
            except:
                pass

        if tile is not None:
            ccds.tileid  [iccd] = tile.tileid
            ccds.tilepass[iccd] = tile.get('pass')
            ccds.tileebv [iccd] = tile.ebv_med

        # Instantiate PSF on a grid
        S = 32
        xx = np.linspace(1+S, W-S, 5)
        yy = np.linspace(1+S, H-S, 5)
        xx,yy = np.meshgrid(xx, yy)
        psfnorms = []
        galnorms = []
        for x,y in zip(xx.ravel(), yy.ravel()):
            p = im.psf_norm(tim, x=x, y=y)
            g = im.galaxy_norm(tim, x=x, y=y)
            psfnorms.append(p)
            galnorms.append(g)
        ccds.psfnorm_mean[iccd] = np.mean(psfnorms)
        ccds.psfnorm_std [iccd] = np.std (psfnorms)
        ccds.galnorm_mean[iccd] = np.mean(galnorms)
        ccds.galnorm_std [iccd] = np.std (galnorms)

        # PSF in center of field
        cx,cy = (W+1)/2., (H+1)/2.
        p = psf.getPointSourcePatch(cx, cy).patch
        ph,pw = p.shape
        px,py = np.meshgrid(np.arange(pw), np.arange(ph))
        psum = np.sum(p)
        # print('psum', psum)
        p /= psum
        # centroids
        cenx = np.sum(p * px)
        ceny = np.sum(p * py)
        # print('cenx,ceny', cenx,ceny)
        # second moments
        x2 = np.sum(p * (px - cenx)**2)
        y2 = np.sum(p * (py - ceny)**2)
        xy = np.sum(p * (px - cenx)*(py - ceny))
        # semi-major/minor axes and position angle
        theta = np.rad2deg(np.arctan2(2 * xy, x2 - y2) / 2.)
        theta = np.abs(theta) * np.sign(xy)
        s = np.sqrt(((x2 - y2)/2.)**2 + xy**2)
        a = np.sqrt((x2 + y2) / 2. + s)
        b = np.sqrt((x2 + y2) / 2. - s)
        ell = 1. - b/a

        # print('PSF second moments', x2, y2, xy)
        # print('PSF position angle', theta)
        # print('PSF semi-axes', a, b)
        # print('PSF ellipticity', ell)

        ccds.psf_mx2[iccd] = x2
        ccds.psf_my2[iccd] = y2
        ccds.psf_mxy[iccd] = xy
        ccds.psf_a[iccd] = a
        ccds.psf_b[iccd] = b
        ccds.psf_theta[iccd] = theta
        ccds.psf_ell  [iccd] = ell

        print('Computing Gaussian approximate PSF quantities...')
        # Galaxy norm using Gaussian approximation of PSF.
        realpsf = tim.psf
        tim.psf = im.read_psf_model(0, 0, gaussPsf=True,
                                    psf_sigma=tim.psf_sigma)
        gaussgalnorm[iccd] = im.galaxy_norm(tim, x=cx, y=cy)
        tim.psf = realpsf
        
        # Sky -- evaluate on a grid (every ~10th pixel)
        skygrid = sky.evaluateGrid(np.linspace(0, ccd.width-1,  int(1+ccd.width/10)),
                                   np.linspace(0, ccd.height-1, int(1+ccd.height/10)))
        ccds.meansky[iccd] = np.mean(skygrid)
        ccds.stdsky[iccd]  = np.std(skygrid)
        ccds.maxsky[iccd]  = skygrid.max()
        ccds.minsky[iccd]  = skygrid.min()

        # WCS
        ccds.ra0[iccd],ccds.dec0[iccd] = wcs.pixelxy2radec(1, 1)
        ccds.ra1[iccd],ccds.dec1[iccd] = wcs.pixelxy2radec(1, H)
        ccds.ra2[iccd],ccds.dec2[iccd] = wcs.pixelxy2radec(W, H)
        ccds.ra3[iccd],ccds.dec3[iccd] = wcs.pixelxy2radec(W, 1)

        midx, midy = (W+1)/2., (H+1)/2.
        rc,dc  = wcs.pixelxy2radec(midx, midy)
        ra,dec = wcs.pixelxy2radec([1,W,midx,midx], [midy,midy,1,H])
        ccds.dra [iccd] = max(degrees_between(ra, dc+np.zeros_like(ra),
                                              rc, dc))
        ccds.ddec[iccd] = max(degrees_between(rc+np.zeros_like(dec), dec,
                                              rc, dc))
        ccds.ra_center [iccd] = rc
        ccds.dec_center[iccd] = dc

        # Compute scale change across the chip
        # how many pixels to step
        step = 10
        xx = np.linspace(1+step, W-step, 5)
        yy = np.linspace(1+step, H-step, 5)
        xx,yy = np.meshgrid(xx, yy)
        pixscale = []
        for x,y in zip(xx.ravel(), yy.ravel()):
            sx = [x-step, x-step, x+step, x+step, x-step]
            sy = [y-step, y+step, y+step, y-step, y-step]
            sr,sd = wcs.pixelxy2radec(sx, sy)
            rc,dc = wcs.pixelxy2radec(x, y)
            # project around a tiny little TAN WCS at (x,y), with 1" pixels
            locwcs = Tan(rc, dc, 0., 0., 1./3600, 0., 0., 1./3600, 1., 1.)
            ok,lx,ly = locwcs.radec2pixelxy(sr, sd)
            #print('local x,y:', lx, ly)
            A = polygon_area((lx, ly))
            pixscale.append(np.sqrt(A / (2*step)**2))
        # print('Pixel scales:', pixscale)
        ccds.pixscale_mean[iccd] = np.mean(pixscale)
        ccds.pixscale_min[iccd] = min(pixscale)
        ccds.pixscale_max[iccd] = max(pixscale)
        ccds.pixscale_std[iccd] = np.std(pixscale)

    ccds.plver = np.array(plvers)

    sfd = tractor.sfd.SFDMap()
    allbands = 'ugrizY'
    filts = ['%s %s' % ('DES', f) for f in allbands]
    wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4']
    ebv,ext = sfd.extinction(filts + wisebands, ccds.ra_center,
                             ccds.dec_center, get_ebv=True)
    ext = ext.astype(np.float32)
    ccds.ebv = ebv.astype(np.float32)
    ccds.decam_extinction = ext[:,:len(allbands)]
    ccds.wise_extinction = ext[:,len(allbands):]

    # Depth
    detsig1 = ccds.sig1 / ccds.psfnorm_mean
    depth = 5. * detsig1
    # that's flux in nanomaggies -- convert to mag
    ccds.psfdepth = -2.5 * (np.log10(depth) - 9)

    detsig1 = ccds.sig1 / ccds.galnorm_mean
    depth = 5. * detsig1
    # that's flux in nanomaggies -- convert to mag
    ccds.galdepth = -2.5 * (np.log10(depth) - 9)
    
    # Depth using Gaussian FWHM.
    psf_sigma = ccds.fwhm / 2.35
    gnorm = 1./(2. * np.sqrt(np.pi) * psf_sigma)
    detsig1 = ccds.sig1 / gnorm
    depth = 5. * detsig1
    # that's flux in nanomaggies -- convert to mag
    ccds.gausspsfdepth = -2.5 * (np.log10(depth) - 9)

    # Gaussian galaxy depth
    detsig1 = ccds.sig1 / gaussgalnorm
    depth = 5. * detsig1
    # that's flux in nanomaggies -- convert to mag
    ccds.gaussgaldepth = -2.5 * (np.log10(depth) - 9)

    ccds.writeto(outfn)
예제 #18
0
def main():
    import argparse

    parser = argparse.ArgumentParser(
        description='This script creates small self-contained data sets that '
        'are useful for test cases of the pipeline codes.')

    parser.add_argument('ccds', help='CCDs table describing region to grab')
    parser.add_argument('outdir', help='Output directory name')
    parser.add_argument('brick', help='Brick containing these images')

    parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image')
    parser.add_argument('--fpack', action='store_true', default=False)
    parser.add_argument('--pad', action='store_true', default=False,
                        help='Keep original image size, but zero out pixels outside ROI')
    
    args = parser.parse_args()

    C = fits_table(args.ccds)
    print(len(C), 'CCDs in', args.ccds)
    C.camera = np.array([c.strip() for c in C.camera])
    
    survey = LegacySurveyData()
    bricks = survey.get_bricks_readonly()
    outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])]
    assert(len(outbricks) == 1)
    
    outsurvey = LegacySurveyData(survey_dir = args.outdir)
    trymakedirs(args.outdir)
    outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz'))

    targetwcs = wcs_for_brick(outbricks[0])
    H,W = targetwcs.shape
    
    tycho = fits_table(os.path.join(survey.get_survey_dir(), 'tycho2.fits.gz'))
    print('Read', len(tycho), 'Tycho-2 stars')
    ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec)
    margin = 100
    tycho.cut(ok * (tx > -margin) * (tx < W+margin) *
              (ty > -margin) * (ty < H+margin))
    print('Cut to', len(tycho), 'Tycho-2 stars within brick')
    del ok,tx,ty
    tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz'))
    
    outccds = C.copy()
    for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1',
              'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1',
              'plver', 'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver',
              'psfplver' ]:
        outccds.delete_column(c)
    outccds.image_hdu[:] = 1

    # Convert to list to avoid truncating filenames
    outccds.image_filename = [fn for fn in outccds.image_filename]
    
    for iccd,ccd in enumerate(C):

        decam = (ccd.camera.strip() == 'decam')
        bok   = (ccd.camera.strip() == '90prime')

        im = survey.get_image_object(ccd)
        print('Got', im)
        slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1))
        tim = im.get_tractor_image(slc, pixPsf=True, splinesky=True,
                                   subsky=False, nanomaggies=False)
        print('Tim:', tim.shape)

        psf = tim.getPsf()
        print('PSF:', psf)
        psfex = psf.psfex
        print('PsfEx:', psfex)

        outim = outsurvey.get_image_object(ccd)
        print('Output image:', outim)

        print('Image filename:', outim.imgfn)
        trymakedirs(outim.imgfn, dir=True)

        imgdata = tim.getImage()
        dqdata = tim.dq
        if decam:
            # DECam-specific code remaps the DQ codes... re-read raw
            print('Reading data quality from', im.dqfn, 'hdu', im.hdu)
            dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice)
        ivdata = tim.getInvvar()

        if args.pad:
            # Create zero image of full size, copy in data.
            fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype)
            fullsize[slc] = imgdata
            imgdata = fullsize

            fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype)
            fullsize[slc] = dqdata
            dqdata = fullsize

            fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype)
            fullsize[slc] = ivdata
            ivdata = fullsize
            
        else:
            # Adjust the header WCS by x0,y0
            crpix1 = tim.hdr['CRPIX1']
            crpix2 = tim.hdr['CRPIX2']
            tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0
            tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0

        # Add image extension to filename
        # fitsio doesn't compress .fz by default, so drop .fz suffix
        
        outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname)
        if not args.fpack:
            outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits')

        # if bok:
        #     outim.whtfn  = outim.whtfn .replace('.wht.fits', '-%s.wht.fits' % im.ccdname)
        #     if not args.fpack:
        #         outim.whtfn  = outim.whtfn .replace('.fits.fz', '.fits')
        # else:
        if True:
            outim.wtfn  = outim.wtfn .replace('.fits', '-%s.fits' % im.ccdname)
            if not args.fpack:
                outim.wtfn  = outim.wtfn .replace('.fits.fz', '.fits')

        if outim.dqfn is not None:
            outim.dqfn  = outim.dqfn .replace('.fits', '-%s.fits' % im.ccdname)
            if not args.fpack:
                outim.dqfn  = outim.dqfn .replace('.fits.fz', '.fits')

        if bok:
            outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname)

        ccdfn = outim.imgfn
        ccdfn = ccdfn.replace(outsurvey.get_image_dir(),'')
        if ccdfn.startswith('/'):
            ccdfn = ccdfn[1:]
        outccds.image_filename[iccd] = ccdfn

        print('Changed output filenames to:')
        print(outim.imgfn)
        print(outim.dqfn)

        ofn = outim.imgfn
        if args.fpack:
            f,ofn = tempfile.mkstemp(suffix='.fits')
            os.close(f)
        fitsio.write(ofn, None, header=tim.primhdr, clobber=True)
        fitsio.write(ofn, imgdata, header=tim.hdr, extname=ccd.ccdname)

        if args.fpack:
            cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn)
            print('Running:', cmd)
            rtn = os.system(cmd)
            assert(rtn == 0)

        h,w = tim.shape
        if not args.pad:
            outccds.width[iccd] = w
            outccds.height[iccd] = h
            outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0
            outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0

        wcs = Tan(*[float(x) for x in
                    [ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2,
                     ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]])

        if args.pad:
            subwcs = wcs
        else:
            subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h)
            outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center()

        #if not bok:
        if True:
            print('Weight filename:', outim.wtfn)
            wfn = outim.wtfn
        # else:
        #     print('Weight filename:', outim.whtfn)
        #     wfn = outim.whtfn

        trymakedirs(wfn, dir=True)

        ofn = wfn
        if args.fpack:
            f,ofn = tempfile.mkstemp(suffix='.fits')
            os.close(f)

        fitsio.write(ofn, None, header=tim.primhdr, clobber=True)
        fitsio.write(ofn, ivdata, header=tim.hdr, extname=ccd.ccdname)

        if args.fpack:
            cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn)
            print('Running:', cmd)
            rtn = os.system(cmd)
            assert(rtn == 0)

        if outim.dqfn is not None:
            print('DQ filename', outim.dqfn)
            trymakedirs(outim.dqfn, dir=True)

            ofn = outim.dqfn
            if args.fpack:
                f,ofn = tempfile.mkstemp(suffix='.fits')
                os.close(f)

            fitsio.write(ofn, None, header=tim.primhdr, clobber=True)
            fitsio.write(ofn, dqdata, header=tim.hdr, extname=ccd.ccdname)

            if args.fpack:
                cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn)
                print('Running:', cmd)
                rtn = os.system(cmd)
                assert(rtn == 0)

        print('PSF filename:', outim.psffn)
        trymakedirs(outim.psffn, dir=True)
        psfex.writeto(outim.psffn)

        if not bok:
            print('Sky filename:', outim.splineskyfn)
            sky = tim.getSky()
            print('Sky:', sky)
            trymakedirs(outim.splineskyfn, dir=True)
            sky.write_fits(outim.splineskyfn)

    outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))

    # WISE
    if args.wise is not None:
        from wise.forcedphot import unwise_tiles_touching_wcs
        from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs,
                                 get_unwise_tractor_image, get_unwise_tile_dir)
        # Read WCS...
        print('Reading TAN wcs header from', args.wise)
        targetwcs = Tan(args.wise)
        tiles = unwise_tiles_touching_wcs(targetwcs)
        print('Cut to', len(tiles), 'unWISE tiles')
        H,W = targetwcs.shape
        r,d = targetwcs.pixelxy2radec(np.array([1,   W,   W/2, W/2]),
                                      np.array([H/2, H/2, 1,   H  ]))
        roiradec = [r[0], r[1], d[2], d[3]]

        unwise_dir = os.environ['UNWISE_COADDS_DIR']
        wise_out = os.path.join(args.outdir, 'images', 'unwise')
        print('Will write WISE outputs to', wise_out)

        unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR']
        wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr')
        print('Will write WISE time-resolved outputs to', wise_tr_out)

        W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_neo1-atlas.fits'))
        print('Read', len(W), 'time-resolved WISE coadd tiles')
        W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id]))
        print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth')

        # Write the time-resolved index subset.
        W.writeto(os.path.join(wise_tr_out, 'time_resolved_neo1-atlas.fits'))

        # this ought to be enough for anyone =)
        Nepochs = 5

        wisedata = []

        # full depth
        for band in [1,2,3,4]:
            wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band))

        # time-resolved
        for band in [1,2]:
            # W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2)
            bitmask = (1 << (band-1))
            for e in range(Nepochs):
                # Which tiles have images for this epoch?
                I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask)
                if len(I) == 0:
                    continue
                print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I])
                edir = os.path.join(unwise_tr_dir, 'e%03i' % e)
                eoutdir = os.path.join(wise_tr_out, 'e%03i' % e)
                wisedata.append((edir, eoutdir, tiles.coadd_id[I], band))

        wrote_masks = set()

        for indir, outdir, tiles, band in wisedata:
            for tile in tiles:
                wanyband = 'w'
                tim = get_unwise_tractor_image(indir, tile, band,
                                               bandname=wanyband, roiradecbox=roiradec)
                print('Got unWISE tim', tim)
                print(tim.shape)
                
                thisdir = get_unwise_tile_dir(outdir, tile)
                print('Directory for this WISE tile:', thisdir)
                base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band))
                print('Base filename:', base)

                masked = True
                mu = 'm' if masked else 'u'

                imfn = base + 'img-%s.fits'       % mu
                ivfn = base + 'invvar-%s.fits.gz' % mu
                nifn = base + 'n-%s.fits.gz'      % mu
                nufn = base + 'n-u.fits.gz'

                #print('WISE image header:', tim.hdr)

                # Adjust the header WCS by x0,y0
                wcs = tim.wcs.wcs
                tim.hdr['CRPIX1'] = wcs.crpix[0]
                tim.hdr['CRPIX2'] = wcs.crpix[1]

                H,W = tim.shape
                tim.hdr['IMAGEW'] = W
                tim.hdr['IMAGEH'] = H

                print('WCS:', wcs)
                print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2'])

                trymakedirs(imfn, dir=True)
                fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True)
                print('Wrote', imfn)
                fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True)
                print('Wrote', ivfn)
                fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True)
                print('Wrote', nifn)
                fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True)
                print('Wrote', nufn)

                if not (indir,tile) in wrote_masks:
                    print('Looking for mask file for', indir, tile)
                    # record that we tried this dir/tile combo
                    wrote_masks.add((indir,tile))
                    for idir in indir.split(':'):
                        tdir = get_unwise_tile_dir(idir, tile)
                        maskfn = 'unwise-%s-msk.fits.gz' % tile
                        fn = os.path.join(tdir, maskfn)
                        print('Mask file:', fn)
                        if os.path.exists(fn):
                            print('Reading', fn)
                            (x0,x1,y0,y1) = tim.roi
                            roislice = (slice(y0,y1), slice(x0,x1))
                            F = fitsio.FITS(fn)[0]
                            hdr = F.read_header()
                            M = F[roislice]
                            outfn = os.path.join(thisdir, maskfn)
                            fitsio.write(outfn, M, header=tim.hdr, clobber=True)
                            print('Wrote', outfn)
                            break

    outC = outsurvey.get_ccds_readonly()
    for iccd,ccd in enumerate(outC):
        outim = outsurvey.get_image_object(ccd)
        print('Got output image:', outim)
        otim = outim.get_tractor_image(pixPsf=True, splinesky=True)
        print('Got output tim:', otim)
예제 #19
0
def main():
    import argparse

    parser = argparse.ArgumentParser(
        description='This script creates small self-contained data sets that '
        'are useful for test cases of the pipeline codes.')

    parser.add_argument('ccds', help='CCDs table describing region to grab')
    parser.add_argument('outdir', help='Output directory name')
    parser.add_argument('brick', help='Brick containing these images')

    parser.add_argument('--survey-dir', type=str, default=None)
    parser.add_argument('--cache-dir', type=str, default=None,
                        help='Directory to search for cached files')
    parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image')
    parser.add_argument('--wise-wcs-hdu', help='For WISE outputs, the HDU to read the WCS from in the file given by --wise.', type=int, default=0)
    parser.add_argument('--fpack', action='store_true', default=False)
    parser.add_argument('--gzip', action='store_true', default=False)
    parser.add_argument('--pad', action='store_true', default=False,
                        help='Keep original image size, but zero out pixels outside ROI')
    
    args = parser.parse_args()

    v = 'SKY_TEMPLATE_DIR'
    if v in os.environ:
        del os.environ[v]

    C = fits_table(args.ccds)
    print(len(C), 'CCDs in', args.ccds)
    C.camera = np.array([c.strip() for c in C.camera])
    
    survey = LegacySurveyData(cache_dir=args.cache_dir, survey_dir=args.survey_dir)

    if ',' in args.brick:
        ra,dec = args.brick.split(',')
        ra = float(ra)
        dec = float(dec)
        fakebricks = fits_table()
        fakebricks.brickname = np.array([('custom-%06i%s%05i' %
                                          (int(1000*ra), 'm' if dec < 0 else 'p',
                                           int(1000*np.abs(dec))))])
        fakebricks.ra  = np.array([ra])
        fakebricks.dec = np.array([dec])
        bricks = fakebricks
        outbricks = bricks
    else:
        bricks = survey.get_bricks_readonly()
        outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])]
        assert(len(outbricks) == 1)

    outsurvey = LegacySurveyData(survey_dir = args.outdir)
    trymakedirs(args.outdir)
    outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz'))

    targetwcs = wcs_for_brick(outbricks[0])
    H,W = targetwcs.shape

    tycho2fn = survey.find_file('tycho2')
    kd = tree_open(tycho2fn, 'stars')
    radius = 1.
    rc,dc = targetwcs.radec_center()
    I = tree_search_radec(kd, rc, dc, radius)
    print(len(I), 'Tycho-2 stars within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (rc,dc))
    # Read only the rows within range.
    tycho = fits_table(tycho2fn, rows=I)
    del kd
    print('Read', len(tycho), 'Tycho-2 stars')
    ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec)
    #margin = 100
    #tycho.cut(ok * (tx > -margin) * (tx < W+margin) *
    #          (ty > -margin) * (ty < H+margin))
    print('Cut to', len(tycho), 'Tycho-2 stars within brick')
    del ok,tx,ty
    #tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz'))
    f,tfn = tempfile.mkstemp(suffix='.fits')
    os.close(f)
    tycho.writeto(tfn)
    outfn = os.path.join(args.outdir, 'tycho2.kd.fits')
    cmd = 'startree -i %s -o %s -P -k -n stars -T' % (tfn, outfn)
    print(cmd)
    rtn = os.system(cmd)
    assert(rtn == 0)
    os.unlink(tfn)

    from legacypipe.gaiacat import GaiaCatalog
    gcat = GaiaCatalog()
    # from ps1cat.py:
    wcs = targetwcs
    step=100.
    margin=10.
    # Grid the CCD in pixel space
    W,H = wcs.get_width(), wcs.get_height()
    xx,yy = np.meshgrid(
        np.linspace(1-margin, W+margin, 2+int((W+2*margin)/step)),
        np.linspace(1-margin, H+margin, 2+int((H+2*margin)/step)))
    # Convert to RA,Dec and then to unique healpixes
    ra,dec = wcs.pixelxy2radec(xx.ravel(), yy.ravel())
    healpixes = set()
    for r,d in zip(ra,dec):
        healpixes.add(gcat.healpix_for_radec(r, d))
    for hp in healpixes:
        hpcat = gcat.get_healpix_catalog(hp)
        ok,xx,yy = wcs.radec2pixelxy(hpcat.ra, hpcat.dec)
        onccd = np.flatnonzero((xx >= 1.-margin) * (xx <= W+margin) *
                               (yy >= 1.-margin) * (yy <= H+margin))
        hpcat.cut(onccd)
        if len(hpcat):
            outfn = os.path.join(args.outdir, 'gaia', 'chunk-%05d.fits' % hp)
            trymakedirs(os.path.join(args.outdir, 'gaia'))
            hpcat.writeto(outfn)
    
    outccds = C.copy()
    cols = outccds.get_columns()
    for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1',
              'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1',
              'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver',
              'psfplver' ]:
        if c in cols:
            outccds.delete_column(c)
    outccds.image_hdu[:] = 1

    # Convert to list to avoid truncating filenames
    outccds.image_filename = [fn for fn in outccds.image_filename]
    
    for iccd,ccd in enumerate(C):

        decam = (ccd.camera.strip() == 'decam')
        bok   = (ccd.camera.strip() == '90prime')

        im = survey.get_image_object(ccd)
        print('Got', im)
        if survey.cache_dir is not None:
            im.check_for_cached_files(survey)
        slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1))

        psfkwargs = dict(pixPsf=True, gaussPsf=False, hybridPsf=False,
                         normalizePsf=False)

        tim = im.get_tractor_image(slc, pixPsf=True,
                                   subsky=False, nanomaggies=False,
                                   no_remap_invvar=True, old_calibs_ok=True)
        print('Tim:', tim.shape)

        psfrow = psfhdr = None

        if args.pad:
            psf = im.read_psf_model(0, 0, w=im.width, h=im.height, **psfkwargs)
            psfex = psf.psfex
        else:
            psf = tim.getPsf()
            psfex = psf.psfex

            # Did the PSF model come from a merged file?
            for fn in [im.merged_psffn, im.psffn] + im.old_merged_psffns:
                if not os.path.exists(fn):
                    continue
                T = fits_table(fn)
                I, = np.nonzero((T.expnum == im.expnum) *
                                np.array([c.strip() == im.ccdname for c in T.ccdname]))
                if len(I) != 1:
                    continue
                psfrow = T[I]
                x0 = ccd.ccd_x0
                y0 = ccd.ccd_y0
                psfrow.polzero1[0] -= x0
                psfrow.polzero2[0] -= y0
                #psfhdr = fitsio.read_header(im.merged_psffn)
                break
        psfex.fwhm = tim.psf_fwhm

        #### HACK
        #psfrow = None
        assert(psfrow is not None)
        if psfrow is not None:
            print('PSF row:', psfrow)
        #else:
        #    print('PSF:', psf)
        #    print('PsfEx:', psfex)

        skyrow = skyhdr = None

        if args.pad:
            primhdr = fitsio.read_header(im.imgfn)
            imghdr = fitsio.read_header(im.imgfn, hdu=im.hdu)

            sky = im.read_sky_model(splinesky=True, primhdr=primhdr, imghdr=imghdr)
            #skyhdr = fitsio.read_header(im.splineskyfn)
            #msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True)
        else:
            sky = tim.getSky()

            # Did the sky model come from a merged file?
            #msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True)

        print('merged skyfn:', im.merged_skyfn)
        print('single skyfn:', im.skyfn)
        print('old merged skyfns:', im.old_merged_skyfns)

        for fn in [im.merged_skyfn, im.skyfn] + im.old_merged_skyfns:
            if not os.path.exists(fn):
                continue
            T = fits_table(fn)
            I, = np.nonzero((T.expnum == im.expnum) *
                            np.array([c.strip() == im.ccdname for c in T.ccdname]))
            skyrow = T[I]
            skyrow.x0[0] = ccd.ccd_x0
            skyrow.y0[0] = ccd.ccd_y0
            # s_med = skyrow.sky_med[0]
            # s_john = skyrow.sky_john[0]
            # skyhdr = fitsio.read_header(fn)

        assert(skyrow is not None)
        ### HACK
        #skyrow = None
                
        if skyrow is not None:
            print('Sky row:', skyrow)
        else:
            print('Sky:', sky)


        # Output filename format:
        fn = ccd.image_filename.strip()
        ccd.image_filename = os.path.join(os.path.dirname(fn),
                                          '%s.%s.fits' % (os.path.basename(fn).split('.')[0], ccd.ccdname.strip()))
        outim = outsurvey.get_image_object(ccd)
        print('Output image:', outim)
        
        print('Image filename:', outim.imgfn)
        trymakedirs(outim.imgfn, dir=True)

        imgdata = tim.getImage()
        ivdata = tim.getInvvar()

        # Since we remap DQ codes (always with Mosaic and Bok, sometimes with DECam),
        # re-read from the FITS file rather than using tim.dq.
        print('Reading data quality from', im.dqfn, 'hdu', im.hdu)
        dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice)

        print('Tim shape:', tim.shape, 'Slice', tim.slice)
        print('image shape:', imgdata.shape, 'iv', ivdata.shape, 'DQ', dqdata.shape)

        from collections import Counter
        dqvals = Counter(dqdata.ravel())
        print('DQ pixel counts:')
        for k,n in dqvals.most_common():
            print('  0x%x' % k, ':', n)

        if args.pad:
            # Create zero image of full size, copy in data.
            fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype)
            fullsize[slc] = imgdata
            imgdata = fullsize

            fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype)
            fullsize[slc] = dqdata
            dqdata = fullsize

            fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype)
            fullsize[slc] = ivdata
            ivdata = fullsize
            
        else:
            # Adjust the header WCS by x0,y0
            crpix1 = tim.hdr['CRPIX1']
            crpix2 = tim.hdr['CRPIX2']
            tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0
            tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0

        # Add image extension to filename
        # fitsio doesn't compress .fz by default, so drop .fz suffix
        
        #outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname)
        if not args.fpack:
            outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits')
        if args.gzip:
            outim.imgfn = outim.imgfn.replace('.fits', '.fits.gz')

        #outim.wtfn  = outim.wtfn.replace('.fits', '-%s.fits' % im.ccdname)
        if not args.fpack:
            outim.wtfn  = outim.wtfn.replace('.fits.fz', '.fits')
        if args.gzip:
            outim.wtfn = outim.wtfn.replace('.fits', '.fits.gz')

        if outim.dqfn is not None:
            #outim.dqfn  = outim.dqfn.replace('.fits', '-%s.fits' % im.ccdname)
            if not args.fpack:
                outim.dqfn  = outim.dqfn.replace('.fits.fz', '.fits')
            if args.gzip:
                outim.dqfn = outim.dqfn.replace('.fits', '.fits.gz')

        if bok:
            outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname)

        ccdfn = outim.imgfn
        ccdfn = ccdfn.replace(outsurvey.get_image_dir(), '')
        if ccdfn.startswith('/'):
            ccdfn = ccdfn[1:]
        outccds.image_filename[iccd] = ccdfn

        print('Changed output filenames to:')
        print(outim.imgfn)
        print(outim.dqfn)

        ofn = outim.imgfn
        if args.fpack:
            f,ofn = tempfile.mkstemp(suffix='.fits')
            os.close(f)
        fits = fitsio.FITS(ofn, 'rw', clobber=True)
        fits.write(None, header=tim.primhdr)
        fits.write(imgdata, header=tim.hdr, extname=ccd.ccdname)
        fits.close()

        if args.fpack:
            cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn)
            print('Running:', cmd)
            rtn = os.system(cmd)
            assert(rtn == 0)

        h,w = tim.shape
        if not args.pad:
            outccds.width[iccd] = w
            outccds.height[iccd] = h
            outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0
            outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0

        wcs = Tan(*[float(x) for x in
                    [ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2,
                     ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]])

        if args.pad:
            subwcs = wcs
        else:
            subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h)
            outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center()

        print('Weight filename:', outim.wtfn)
        wfn = outim.wtfn
        trymakedirs(wfn, dir=True)

        ofn = wfn
        if args.fpack:
            f,ofn = tempfile.mkstemp(suffix='.fits')
            os.close(f)

        fits = fitsio.FITS(ofn, 'rw', clobber=True)
        fits.write(None, header=tim.primhdr)
        fits.write(ivdata, header=tim.hdr, extname=ccd.ccdname)
        fits.close()

        if args.fpack:
            cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn)
            print('Running:', cmd)
            rtn = os.system(cmd)
            assert(rtn == 0)

        if outim.dqfn is not None:
            print('DQ filename', outim.dqfn)
            trymakedirs(outim.dqfn, dir=True)

            ofn = outim.dqfn
            if args.fpack:
                f,ofn = tempfile.mkstemp(suffix='.fits')
                os.close(f)

            fits = fitsio.FITS(ofn, 'rw', clobber=True)
            fits.write(None, header=tim.primhdr)
            fits.write(dqdata, header=tim.hdr, extname=ccd.ccdname)
            fits.close()

            if args.fpack:
                cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn)
                print('Running:', cmd)
                rtn = os.system(cmd)
                assert(rtn == 0)

        psfout = outim.psffn
        #if psfrow:
        #    psfout = outim.merged_psffn
        print('PSF output filename:', psfout)
        trymakedirs(psfout, dir=True)
        if psfrow:
            psfrow.writeto(psfout, primhdr=psfhdr)
        else:
            print('Writing PsfEx:', psfout)
            psfex.writeto(psfout)
            # update header
            F = fitsio.FITS(psfout, 'rw')
            F[0].write_keys([dict(name='EXPNUM', value=ccd.expnum),
                             dict(name='PLVER',  value=psf.plver),
                             dict(name='PROCDATE', value=psf.procdate),
                             dict(name='PLPROCID', value=psf.plprocid),])
            F.close()

        skyout = outim.skyfn
        #if skyrow:
        #    skyout = outim.merged_splineskyfn

        print('Sky output filename:', skyout)
        trymakedirs(skyout, dir=True)
        if skyrow is not None:
            skyrow.writeto(skyout, primhdr=skyhdr)
        else:
            primhdr = fitsio.FITSHDR()
            primhdr['PLVER'] = sky.plver
            primhdr['PLPROCID'] = sky.plprocid
            primhdr['PROCDATE'] = sky.procdate
            primhdr['EXPNUM'] = ccd.expnum
            primhdr['IMGDSUM'] = sky.datasum
            primhdr['S_MED'] = s_med
            primhdr['S_JOHN'] = s_john
            sky.write_fits(skyout, primhdr=primhdr)

        # HACK -- check result immediately.
        outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))
        outsurvey.ccds = None
        outC = outsurvey.get_ccds_readonly()
        occd = outC[iccd]
        outim = outsurvey.get_image_object(occd)
        print('Got output image:', outim)
        otim = outim.get_tractor_image(pixPsf=True,
                                       hybridPsf=True, old_calibs_ok=True)
        print('Got output tim:', otim)

    outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))

    # WISE
    if args.wise is not None:
        from wise.forcedphot import unwise_tiles_touching_wcs
        from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs,
                                 get_unwise_tractor_image, get_unwise_tile_dir)
        # Read WCS...
        print('Reading TAN wcs header from', args.wise, 'HDU', args.wise_wcs_hdu)
        targetwcs = Tan(args.wise, args.wise_wcs_hdu)
        tiles = unwise_tiles_touching_wcs(targetwcs)
        print('Cut to', len(tiles), 'unWISE tiles')
        H,W = targetwcs.shape
        r,d = targetwcs.pixelxy2radec(np.array([1,   W,   W/2, W/2]),
                                      np.array([H/2, H/2, 1,   H  ]))
        roiradec = [r[0], r[1], d[2], d[3]]

        unwise_dir = os.environ['UNWISE_COADDS_DIR']
        wise_out = os.path.join(args.outdir, 'images', 'unwise')
        print('Will write WISE outputs to', wise_out)

        unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR']
        wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr')
        print('Will write WISE time-resolved outputs to', wise_tr_out)
        trymakedirs(wise_tr_out)

        W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_atlas.fits'))
        print('Read', len(W), 'time-resolved WISE coadd tiles')
        W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id]))
        print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth')

        # Write the time-resolved index subset.
        W.writeto(os.path.join(wise_tr_out, 'time_resolved_atlas.fits'))

        # this ought to be enough for anyone =)
        _,Nepochs = W.epoch_bitmask.shape
        print('N epochs in time-resolved atlas:', Nepochs)

        wisedata = []

        # full depth
        for band in [1,2,3,4]:
            wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band, True))

        # time-resolved
        for band in [1,2]:
            # W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2)
            bitmask = (1 << (band-1))
            for e in range(Nepochs):
                # Which tiles have images for this epoch?
                I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask)
                if len(I) == 0:
                    continue
                print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I])
                edir = os.path.join(unwise_tr_dir, 'e%03i' % e)
                eoutdir = os.path.join(wise_tr_out, 'e%03i' % e)
                wisedata.append((edir, eoutdir, tiles.coadd_id[I], band, False))

        wrote_masks = set()

        model_dir = os.environ.get('UNWISE_MODEL_SKY_DIR')
        if model_dir is not None:
            model_dir_out = os.path.join(args.outdir, 'images', 'unwise-mod')
            trymakedirs(model_dir_out)

        for indir, outdir, tiles, band, fulldepth in wisedata:
            for tile in tiles:
                wanyband = 'w'
                tim = get_unwise_tractor_image(indir, tile, band,
                                               bandname=wanyband, roiradecbox=roiradec)
                print('Got unWISE tim', tim)
                print(tim.shape)

                if model_dir is not None and fulldepth and band in [1,2]:
                    print('ROI', tim.roi)
                    #0387p575.1.mod.fits
                    fn = '%s.%i.mod.fits' % (tile, band)
                    print('Filename', fn)
                    F = fitsio.FITS(os.path.join(model_dir, fn))
                    x0,x1,y0,y1 = tim.roi
                    slc = slice(y0,y1),slice(x0,x1)

                    phdr = F[0].read_header()

                    outfn = os.path.join(model_dir_out, fn)
                    for e,extname in [(1,'MODEL'), (2,'SKY')]:
                        pix = F[e][slc]
                        hdr = F[e].read_header()
                        crpix1 = hdr['CRPIX1']
                        crpix2 = hdr['CRPIX2']
                        hdr['CRPIX1'] -= x0
                        hdr['CRPIX2'] -= y0
                        #print('mod', mod)
                        #print('Model', mod.shape)
                        if e == 1:
                            fitsio.write(outfn, None, clobber=True, header=phdr)
                        fitsio.write(outfn, pix, header=hdr, extname=extname)
                    print('Wrote', outfn)

                thisdir = get_unwise_tile_dir(outdir, tile)
                print('Directory for this WISE tile:', thisdir)
                base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band))
                print('Base filename:', base)

                masked = True
                mu = 'm' if masked else 'u'

                imfn = base + 'img-%s.fits'       % mu
                ivfn = base + 'invvar-%s.fits.gz' % mu
                nifn = base + 'n-%s.fits.gz'      % mu
                nufn = base + 'n-u.fits.gz'

                #print('WISE image header:', tim.hdr)

                # Adjust the header WCS by x0,y0
                wcs = tim.wcs.wcs
                tim.hdr['CRPIX1'] = wcs.crpix[0]
                tim.hdr['CRPIX2'] = wcs.crpix[1]

                H,W = tim.shape
                tim.hdr['IMAGEW'] = W
                tim.hdr['IMAGEH'] = H

                print('WCS:', wcs)
                print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2'])

                trymakedirs(imfn, dir=True)
                fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True)
                print('Wrote', imfn)
                fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True)
                print('Wrote', ivfn)
                fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True)
                print('Wrote', nifn)
                fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True)
                print('Wrote', nufn)

                if not (indir,tile) in wrote_masks:
                    print('Looking for mask file for', indir, tile)
                    # record that we tried this dir/tile combo
                    wrote_masks.add((indir,tile))
                    for idir in indir.split(':'):
                        tdir = get_unwise_tile_dir(idir, tile)
                        maskfn = 'unwise-%s-msk.fits.gz' % tile
                        fn = os.path.join(tdir, maskfn)
                        print('Mask file:', fn)
                        if os.path.exists(fn):
                            print('Reading', fn)
                            (x0,x1,y0,y1) = tim.roi
                            roislice = (slice(y0,y1), slice(x0,x1))
                            F = fitsio.FITS(fn)[0]
                            hdr = F.read_header()
                            M = F[roislice]
                            outfn = os.path.join(thisdir, maskfn)
                            fitsio.write(outfn, M, header=tim.hdr, clobber=True)
                            print('Wrote', outfn)
                            break

    outC = outsurvey.get_ccds_readonly()
    for iccd,ccd in enumerate(outC):
        outim = outsurvey.get_image_object(ccd)
        print('Got output image:', outim)
        otim = outim.get_tractor_image(pixPsf=True,
                                       hybridPsf=True, old_calibs_ok=True)
        print('Got output tim:', otim)
예제 #20
0
def main():
    """Main program.
    """
    import argparse
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        '--force',
        action='store_true',
        help='Run calib processes even if files already exist?')
    parser.add_argument('--survey-dir', help='Override LEGACY_SURVEY_DIR')
    parser.add_argument(
        '--expnum',
        type=str,
        help='Cut to a single or set of exposures; comma-separated list')
    parser.add_argument('--extname',
                        '--ccdname',
                        help='Cut to a single extension/CCD name')

    parser.add_argument('--no-psf',
                        dest='psfex',
                        action='store_false',
                        help='Do not compute PsfEx calibs')
    parser.add_argument('--no-sky',
                        dest='sky',
                        action='store_false',
                        help='Do not compute sky models')
    parser.add_argument('--run-se',
                        action='store_true',
                        help='Run SourceExtractor')

    parser.add_argument('--no-splinesky',
                        dest='splinesky',
                        default=True,
                        action='store_false',
                        help='Use constant, not splinesky')
    parser.add_argument('--threads',
                        type=int,
                        help='Run multi-threaded',
                        default=None)
    parser.add_argument('--continue',
                        dest='cont',
                        default=False,
                        action='store_true',
                        help='Continue even if one file fails?')
    parser.add_argument('--plot-base',
                        help='Make plots with this base filename')

    parser.add_argument(
        '--blob-mask-dir',
        type=str,
        default=None,
        help=
        'The base directory to search for blob masks during sky model construction'
    )
    parser.add_argument('-v',
                        '--verbose',
                        dest='verbose',
                        action='count',
                        default=0,
                        help='Make more verbose')

    parser.add_argument('args', nargs=argparse.REMAINDER)
    opt = parser.parse_args()

    import logging
    if opt.verbose:
        lvl = logging.DEBUG
    else:
        lvl = logging.INFO
    logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
    # tractor logging is *soooo* chatty
    logging.getLogger('tractor.engine').setLevel(lvl + 10)

    survey = LegacySurveyData(survey_dir=opt.survey_dir)
    T = None
    if len(opt.args) == 0:
        if opt.expnum is not None:
            expnums = set([int(e) for e in opt.expnum.split(',')])
            T = merge_tables([
                survey.find_ccds(expnum=e, ccdname=opt.extname)
                for e in expnums
            ])
            print('Cut to', len(T), 'with expnum in', expnums, 'and extname',
                  opt.extname)
            opt.args = range(len(T))
        else:
            parser.print_help()
            return 0
    ps = None
    if opt.plot_base is not None:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence(opt.plot_base)

    survey_blob_mask = None
    if opt.blob_mask_dir is not None:
        survey_blob_mask = LegacySurveyData(opt.blob_mask_dir)

    args = []
    for a in opt.args:
        # Check for "expnum-ccdname" format.
        if '-' in str(a):
            words = a.split('-')
            assert (len(words) == 2)
            expnum = int(words[0])
            ccdname = words[1]

            T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
            if len(T) != 1:
                print('Found', len(I), 'CCDs for expnum', expnum, 'CCDname',
                      ccdname, ':', I)
                print('WARNING: skipping this expnum,ccdname')
                continue
            t = T[0]
        else:
            i = int(a)
            print('Index', i)
            t = T[i]

        im = survey.get_image_object(t)
        print('Running', im.name)

        kwargs = dict(psfex=opt.psfex,
                      sky=opt.sky,
                      ps=ps,
                      survey=survey,
                      survey_blob_mask=survey_blob_mask)
        if opt.force:
            kwargs.update(force=True)
        if opt.run_se:
            kwargs.update(se=True)
        if opt.splinesky:
            kwargs.update(splinesky=True)
        if opt.cont:
            kwargs.update(noraise=True)

        if opt.threads:
            args.append((im, kwargs))
        else:
            run_calibs((im, kwargs))

    if opt.threads:
        from astrometry.util.multiproc import multiproc
        mp = multiproc(opt.threads)
        mp.map(time_run_calibs, args)

    return 0
예제 #21
0
def main():
    """Main program.
    """
    import argparse
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        '--force',
        action='store_true',
        help='Run calib processes even if files already exist?')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')

    parser.add_argument(
        '--expnum',
        type=str,
        help='Cut to a single or set of exposures; comma-separated list')
    parser.add_argument('--extname',
                        '--ccdname',
                        help='Cut to a single extension/CCD name')

    parser.add_argument('--no-psf',
                        dest='psfex',
                        action='store_false',
                        help='Do not compute PsfEx calibs')
    parser.add_argument('--no-sky',
                        dest='sky',
                        action='store_false',
                        help='Do not compute sky models')
    parser.add_argument('--run-se',
                        action='store_true',
                        help='Run SourceExtractor')

    parser.add_argument('--splinesky',
                        action='store_true',
                        help='Spline sky, not constant')
    parser.add_argument('--threads',
                        type=int,
                        help='Run multi-threaded',
                        default=None)
    parser.add_argument('--continue',
                        dest='cont',
                        default=False,
                        action='store_true',
                        help='Continue even if one file fails?')

    parser.add_argument('args', nargs=argparse.REMAINDER)
    opt = parser.parse_args()

    survey = LegacySurveyData()
    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        T = survey.cleanup_ccds_table(T)

        print('Read', len(T), 'from', opt.ccds)
    else:
        T = survey.get_ccds()
        #print len(T), 'CCDs'

    if len(opt.args) == 0:
        if opt.expnum is not None:
            expnums = set([int(e) for e in opt.expnum.split(',')])
            T.cut(np.array([e in expnums for e in T.expnum]))
            print('Cut to', len(T), 'with expnum in', expnums)
        if opt.extname is not None:
            T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname]))
            print('Cut to', len(T), 'with extname =', opt.extname)

        opt.args = range(len(T))

    args = []
    for a in opt.args:
        # Check for "expnum-ccdname" format.
        if '-' in str(a):
            words = a.split('-')
            assert (len(words) == 2)
            expnum = int(words[0])
            ccdname = words[1]
            I = np.flatnonzero((T.expnum == expnum) * (T.ccdname == ccdname))
            if len(I) != 1:
                print('Found', len(I), 'CCDs for expnum', expnum, 'CCDname',
                      ccdname, ':', I)
                print('WARNING: skipping this expnum,ccdname')
                continue
            assert (len(I) == 1)
            t = T[I[0]]
        else:
            i = int(a)
            print('Index', i)
            t = T[i]

        #print('CCDnmatch', t.ccdnmatch)
        #if t.ccdnmatch < 20 and not opt.force:
        #    print('Skipping ccdnmatch = %i' % t.ccdnmatch)
        #    continue

        im = survey.get_image_object(t)
        print('Running', im.name)

        kwargs = dict(psfex=opt.psfex, sky=opt.sky)
        if opt.force:
            kwargs.update(force=True)
        if opt.run_se:
            kwargs.update(se=True)
        if opt.splinesky:
            kwargs.update(splinesky=True)

        if opt.cont:
            kwargs.update(noraise=True)

        if opt.threads:
            args.append((im, kwargs))
        else:
            run_calibs((im, kwargs))

    if opt.threads:
        from astrometry.util.multiproc import multiproc
        mp = multiproc(opt.threads)
        mp.map(run_calibs, args)

    return 0
예제 #22
0
def main():
    fn = '/global/cscratch1/sd/dstn/c4d_190730_024955_ori/c4d_190730_024955_ori.52.fits'
    survey_dir = '/global/cscratch1/sd/dstn/subtractor-survey-dir'

    imagedir = os.path.join(survey_dir, 'images')
    trymakedirs(imagedir)
    calibdir = os.path.join(survey_dir, 'calib')

    psfexdir = os.path.join(calibdir, 'decam', 'psfex-merged')
    trymakedirs(psfexdir)
    skydir = os.path.join(calibdir, 'decam', 'splinesky-merged')
    trymakedirs(skydir)

    basename = os.path.basename(fn)
    basename = basename.replace('.fits', '')

    # Output filenames for legacyzpts calibration/zeropoint files
    f, photfn = tempfile.mkstemp()
    os.close(f)
    surveyfn = os.path.join(survey_dir, 'survey-ccds-%s.fits.gz' % basename)
    annfn = os.path.join(survey_dir, 'annotated-%s.fits' % basename)
    mp = multiproc()

    survey = LegacySurveyData(survey_dir)
    # Use the subclass above to handle DECam images!
    survey.image_typemap.update(decam=GoldsteinDecamImage)

    # Grab the exposure number and CCD name
    hdr = fitsio.read_header(fn)
    expnum = hdr['EXPNUM']
    ccdname = hdr['EXTNAME'].strip()
    print('Exposure', expnum, 'CCD', ccdname)

    import logging
    lvl = logging.INFO
    logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
    # tractor logging is *soooo* chatty
    logging.getLogger('tractor.engine').setLevel(lvl + 10)

    if not os.path.exists(surveyfn):
        # Run calibrations and zeropoints
        runit(fn,
              photfn,
              surveyfn,
              annfn,
              mp,
              survey=survey,
              camera='decam',
              debug=False,
              choose_ccd=ccdname,
              splinesky=True,
              calibdir=calibdir,
              measureclass=GoldsteinDecamMeasurer)

    # Find catalog sources touching this CCD
    ccds = survey.find_ccds(expnum=expnum, ccdname=ccdname)
    assert (len(ccds) == 1)
    ccd = ccds[0]
    print('Got CCD', ccd)

    # Create Tractor image
    im = survey.get_image_object(ccd)
    print('Got image:', im)

    # Look at this sub-image, or the whole chip?
    #zoomslice=None
    zoomslice = (slice(0, 1000), slice(0, 1000))

    tim = im.get_tractor_image(slc=zoomslice,
                               pixPsf=True,
                               splinesky=True,
                               hybridPsf=True,
                               normalizePsf=True,
                               old_calibs_ok=True)
    print('Got tim:', tim)

    # Read catalog files touching this CCD
    catsurvey = LegacySurveyData(
        '/global/project/projectdirs/cosmo/work/legacysurvey/dr8/south')
    T = get_catalog_in_wcs(tim.subwcs, catsurvey)
    print('Got', len(T), 'DR8 catalog sources within CCD')

    # Gaia stars: move RA,Dec to the epoch of this image.
    I = np.flatnonzero(T.ref_epoch > 0)
    if len(I):
        from legacypipe.survey import radec_at_mjd
        print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd())
        ra, dec = radec_at_mjd(T.ra[I], T.dec[I], T.ref_epoch[I].astype(float),
                               T.pmra[I], T.pmdec[I], T.parallax[I],
                               tim.time.toMjd())
        T.ra[I] = ra
        T.dec[I] = dec

    # Create Tractor Source objects from the catalog
    cat = read_fits_catalog(T, bands=tim.band)
    print('Created', len(cat), 'source objects')

    # Render model image!
    tr = Tractor([tim], cat)
    mod = tr.getModelImage(0)

    # plots
    ima = dict(interpolation='nearest',
               origin='lower',
               vmin=-2 * tim.sig1,
               vmax=10 * tim.sig1,
               cmap='gray')
    plt.clf()
    plt.imshow(tim.getImage(), **ima)
    plt.title('Image')
    plt.savefig('img.jpg')

    plt.clf()
    plt.imshow(mod, **ima)
    plt.title('Model')
    plt.savefig('mod.jpg')

    plt.clf()
    plt.imshow(tim.getImage() - mod, **ima)
    plt.title('Residual')
    plt.savefig('res.jpg')
예제 #23
0
print(len(I1), 'oki images')
print(len(I2), 'non-oki images')
ccds.cut(
    np.hstack(
        zip(I1[np.random.permutation(len(I1))],
            I2[np.random.permutation(len(I2))])))
#ccds.cut(np.random.permutation(len(ccds)))

allsigs1 = []
allsigs2 = []
names = []

offsets = [1, 2, 3, 5, 10, 20, 40]

for ccd in ccds[:16]:
    im = survey.get_image_object(ccd)

    tim1 = im.get_tractor_image(gaussPsf=True, splinesky=True, dq=False)

    tim2 = im.get_tractor_image(gaussPsf=True,
                                splinesky=True,
                                dq=False,
                                nanomaggies=False,
                                subsky=False)
    tim = tim2

    name = tim.name
    if 'oki' in im.imgfn:
        name += ' oki'
    elif 'ooi' in im.imgfn:
        name += ' ooi'
예제 #24
0
def main():
    survey = LegacySurveyData()
    ccds = survey.get_ccds()
    print(len(ccds), 'CCDs')

    expnums = np.unique(ccds.expnum)
    print(len(expnums), 'unique exposures')

    for expnum in expnums:

        expnumstr = '%08i' % expnum
        skyoutfn = os.path.join('splinesky', expnumstr[:5], 'decam-%s.fits' % expnumstr)
        psfoutfn = os.path.join('psfex', expnumstr[:5], 'decam-%s.fits' % expnumstr)

        if os.path.exists(skyoutfn) and os.path.exists(psfoutfn):
            print('Exposure', expnum, 'is done already')
            continue

        C = ccds[ccds.expnum == expnum]
        print(len(C), 'CCDs in expnum', expnum)

        psfex = []
        psfhdrvals = []

        splinesky = []
        skyhdrvals = []

        for ccd in C:
            im = survey.get_image_object(ccd)

            fn = im.splineskyfn
            if os.path.exists(fn):
                print('Reading', fn)
                T = None
                try:
                    T = fits_table(fn)
                except KeyboardInterrupt:
                    raise
                except:
                    print('Failed to read file', fn, ':', sys.exc_info()[1])
                if T is not None:
                    splinesky.append(T)
                    # print(fn)
                    # T.about()
                    hdr = fitsio.read_header(fn)
                    skyhdrvals.append([hdr[k] for k in [
                                'SKY', 'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname])
            else:
                print('File not found:', fn)

            fn = im.psffn
            if os.path.exists(fn):
                T = fits_table(fn)
                hdr = fitsio.read_header(fn, ext=1)

                keys = ['LOADED', 'ACCEPTED', 'CHI2', 'POLNAXIS', 
                        'POLNGRP', 'PSF_FWHM', 'PSF_SAMP', 'PSFNAXIS',
                        'PSFAXIS1', 'PSFAXIS2', 'PSFAXIS3',]

                if hdr['POLNAXIS'] == 0:
                    # No polynomials.  Fake it.
                    T.polgrp1 = np.array([0])
                    T.polgrp2 = np.array([0])
                    T.polname1 = np.array(['fake'])
                    T.polname2 = np.array(['fake'])
                    T.polzero1 = np.array([0])
                    T.polzero2 = np.array([0])
                    T.polscal1 = np.array([1])
                    T.polscal2 = np.array([1])
                    T.poldeg1 = np.array([0])
                else:
                    keys.extend([
                            'POLGRP1', 'POLNAME1', 'POLZERO1', 'POLSCAL1',
                            'POLGRP2', 'POLNAME2', 'POLZERO2', 'POLSCAL2',
                            'POLDEG1'])

                for k in keys:
                    try:
                        v = hdr[k]
                    except:
                        print('Did not find key', k, 'in', fn)
                        sys.exit(-1)
                    T.set(k.lower(), np.array([hdr[k]]))
                psfex.append(T)
                #print(fn)
                #T.about()
    
                hdr = fitsio.read_header(fn)
                psfhdrvals.append([hdr.get(k,'') for k in [
                    'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname])
            else:
                print('File not found:', fn)

        if len(psfex):
            padded = pad_arrays([p.psf_mask[0] for p in psfex])
            cols = psfex[0].columns()
            cols.remove('psf_mask')
            T = merge_tables(psfex, columns=cols)
            T.psf_mask = np.concatenate([[p] for p in padded])
            T.legpipev = np.array([h[0] for h in psfhdrvals])
            T.plver    = np.array([h[1] for h in psfhdrvals])
            T.expnum   = np.array([h[2] for h in psfhdrvals])
            T.ccdname  = np.array([h[3] for h in psfhdrvals])
            fn = psfoutfn
            trymakedirs(fn, dir=True)
            T.writeto(fn)
            print('Wrote', fn)

        if len(splinesky):
            T = fits_table()
            T.gridw = np.array([t.gridvals[0].shape[1] for t in splinesky])
            T.gridh = np.array([t.gridvals[0].shape[0] for t in splinesky])

            padded = pad_arrays([t.gridvals[0] for t in splinesky])
            T.gridvals = np.concatenate([[p] for p in padded])
            padded = pad_arrays([t.xgrid[0] for t in splinesky])
            T.xgrid = np.concatenate([[p] for p in padded])
            padded = pad_arrays([t.xgrid[0] for t in splinesky])
            T.ygrid = np.concatenate([[p] for p in padded])
    
            cols = splinesky[0].columns()
            print('Columns:', cols)
            for c in ['gridvals', 'xgrid', 'ygrid']:
                cols.remove(c)

            T.add_columns_from(merge_tables(splinesky, columns=cols))
            T.skyclass = np.array([h[0] for h in skyhdrvals])
            T.legpipev = np.array([h[1] for h in skyhdrvals])
            T.plver    = np.array([h[2] for h in skyhdrvals])
            T.expnum   = np.array([h[3] for h in skyhdrvals])
            T.ccdname  = np.array([h[4] for h in skyhdrvals])
            fn = skyoutfn
            trymakedirs(fn, dir=True)
            T.writeto(fn)
            print('Wrote', fn)
예제 #25
0
def main(survey=None, opt=None, args=None):
    '''Driver function for forced photometry of individual Legacy
    Survey images.
    '''
    if args is None:
        args = sys.argv[1:]
    print('forced_photom.py', ' '.join(args))

    if opt is None:
        parser = get_parser()
        opt = parser.parse_args(args)

    import logging
    if opt.verbose == 0:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG
    logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
    # tractor logging is *soooo* chatty
    logging.getLogger('tractor.engine').setLevel(lvl + 10)

    t0 = Time()
    if survey is None:
        survey = LegacySurveyData(survey_dir=opt.survey_dir,
                                  cache_dir=opt.cache_dir,
                                  output_dir=opt.out_dir)
    if opt.skip:
        if opt.out is not None:
            outfn = opt.out
        else:
            outfn = survey.find_file('forced',
                                     output=True,
                                     camera=opt.camera,
                                     expnum=opt.expnum)
        if os.path.exists(outfn):
            print('Ouput file exists:', outfn)
            return 0

    if opt.derivs and opt.agn:
        print('Sorry, can\'t do --derivs AND --agn')
        return -1

    if opt.out is None and opt.out_dir is None:
        print('Must supply either --out or --out-dir')
        return -1

    if opt.expnum is None and opt.out is None:
        print('If no --expnum is given, must supply --out filename')
        return -1

    if not opt.forced:
        opt.apphot = True

    zoomslice = None
    if opt.zoom is not None:
        (x0, x1, y0, y1) = opt.zoom
        zoomslice = (slice(y0, y1), slice(x0, x1))

    ps = None
    if opt.plots is not None:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence(opt.plots)

    # Cache CCDs files before the find_ccds call...
    # Copy required files into the cache?
    if opt.pre_cache:

        def copy_files_to_cache(fns):
            for fn in fns:
                cachefn = fn.replace(survey.survey_dir, survey.cache_dir)
                if not cachefn.startswith(survey.cache_dir):
                    print('Skipping', fn)
                    continue
                outdir = os.path.dirname(cachefn)
                trymakedirs(outdir)
                print('Copy', fn)
                print('  to', cachefn)
                shutil.copyfile(fn, cachefn)

        assert (survey.cache_dir is not None)
        fnset = set()
        fn = survey.find_file('bricks')
        fnset.add(fn)
        fns = survey.find_file('ccd-kds')
        fnset.update(fns)
        copy_files_to_cache(fnset)

    # Read metadata from survey-ccds.fits table
    ccds = survey.find_ccds(camera=opt.camera,
                            expnum=opt.expnum,
                            ccdname=opt.ccdname)
    print(len(ccds), 'with camera', opt.camera, 'and expnum', opt.expnum,
          'and ccdname', opt.ccdname)
    # sort CCDs
    ccds.cut(np.lexsort((ccds.ccdname, ccds.expnum, ccds.camera)))

    # If there is only one catalog survey_dir, we pass it to get_catalog_in_wcs
    # as the northern survey.
    catsurvey_north = survey
    catsurvey_south = None

    if opt.catalog_dir_north is not None:
        assert (opt.catalog_dir_south is not None)
        assert (opt.catalog_resolve_dec_ngc is not None)
        catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
        catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)
    elif opt.catalog_dir is not None:
        catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)

    # Copy required CCD & calib files into the cache?
    if opt.pre_cache:
        assert (survey.cache_dir is not None)
        fnset = set()
        for ccd in ccds:
            im = survey.get_image_object(ccd)
            for key in im.get_cacheable_filename_variables():
                fn = getattr(im, key)
                if fn is None or not (os.path.exists(fn)):
                    continue
                fnset.add(fn)
        copy_files_to_cache(fnset)

    args = []
    for ccd in ccds:
        args.append((survey, catsurvey_north, catsurvey_south,
                     opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))

    if opt.threads:
        from astrometry.util.multiproc import multiproc
        from astrometry.util.timingpool import TimingPool, TimingPoolMeas
        pool = TimingPool(opt.threads)
        poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
        Time.add_measurement(poolmeas)
        mp = multiproc(None, pool=pool)
        tm = Time()
        FF = mp.map(bounce_one_ccd, args)
        print('Multi-processing forced-phot:', Time() - tm)
        del mp
        Time.measurements.remove(poolmeas)
        del poolmeas
        pool.close()
        pool.join()
        del pool
    else:
        FF = map(bounce_one_ccd, args)

    FF = [F for F in FF if F is not None]
    if len(FF) == 0:
        print('No photometry results to write.')
        return 0
    # Keep only the first header
    _, version_hdr, _, _ = FF[0]
    # unpack results
    outlier_masks = [m for _, _, m, _ in FF]
    outlier_hdrs = [h for _, _, _, h in FF]
    FF = [F for F, _, _, _ in FF]
    F = merge_tables(FF)

    if len(ccds):
        version_hdr.delete('CPHDU')
        version_hdr.delete('CCDNAME')

    from legacypipe.utils import add_bits
    from legacypipe.bits import DQ_BITS
    add_bits(version_hdr, DQ_BITS, 'DQMASK', 'DQ', 'D')
    from legacyzpts.psfzpt_cuts import CCD_CUT_BITS
    add_bits(version_hdr, CCD_CUT_BITS, 'CCD_CUTS', 'CC', 'C')
    for i, ap in enumerate(apertures_arcsec):
        version_hdr.add_record(
            dict(name='APRAD%i' % i,
                 value=ap,
                 comment='(optical) Aperture radius, in arcsec'))

    unitmap = {
        'exptime': 'sec',
        'flux': 'nanomaggy',
        'flux_ivar': '1/nanomaggy^2',
        'apflux': 'nanomaggy',
        'apflux_ivar': '1/nanomaggy^2',
        'psfdepth': '1/nanomaggy^2',
        'galdepth': '1/nanomaggy^2',
        'sky': 'nanomaggy/arcsec^2',
        'psfsize': 'arcsec',
        'fwhm': 'pixels',
        'ccdrarms': 'arcsec',
        'ccddecrms': 'arcsec',
        'ra': 'deg',
        'dec': 'deg',
        'skyrms': 'counts/sec',
        'dra': 'arcsec',
        'ddec': 'arcsec',
        'dra_ivar': '1/arcsec^2',
        'ddec_ivar': '1/arcsec^2'
    }

    columns = F.get_columns()
    order = [
        'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
        'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'fwhm', 'ccd_cuts',
        'airmass', 'sky', 'skyrms', 'psfdepth', 'galdepth', 'ccdzpt',
        'ccdrarms', 'ccddecrms', 'ccdphrms', 'ra', 'dec', 'flux', 'flux_ivar',
        'fracflux', 'rchisq', 'fracmasked', 'fracin', 'apflux', 'apflux_ivar',
        'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar', 'ddec_ivar'
    ]
    columns = [c for c in order if c in columns]
    units = [unitmap.get(c, '') for c in columns]

    if opt.out is not None:
        outdir = os.path.dirname(opt.out)
        if len(outdir):
            trymakedirs(outdir)
        tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.out))
        fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
        F.writeto(tmpfn, units=units, append=True, columns=columns)
        os.rename(tmpfn, opt.out)
        print('Wrote', opt.out)
    else:
        with survey.write_output('forced',
                                 camera=opt.camera,
                                 expnum=opt.expnum) as out:
            F.writeto(None,
                      fits_object=out.fits,
                      primheader=version_hdr,
                      units=units,
                      columns=columns)
            print('Wrote', out.real_fn)

    if opt.outlier_mask is not None:
        # Add outlier bit meanings to the primary header
        version_hdr.add_record(
            dict(name='COMMENT', value='Outlier mask bit meanings'))
        version_hdr.add_record(
            dict(name='OUTL_POS',
                 value=1,
                 comment='Outlier mask bit for Positive outlier'))
        version_hdr.add_record(
            dict(name='OUTL_NEG',
                 value=2,
                 comment='Outlier mask bit for Negative outlier'))

    if opt.outlier_mask == 'default':
        outdir = os.path.join(opt.out_dir, 'outlier-masks')
        camexp = set(zip(ccds.camera, ccds.expnum))
        for c, e in camexp:
            I = np.flatnonzero((ccds.camera == c) * (ccds.expnum == e))
            ccd = ccds[I[0]]
            imfn = ccd.image_filename.strip()
            outfn = os.path.join(outdir, imfn.replace('.fits',
                                                      '-outlier.fits'))
            trymakedirs(outfn, dir=True)
            tempfn = outfn.replace('.fits', '-tmp.fits')
            with fitsio.FITS(tempfn, 'rw', clobber=True) as fits:
                fits.write(None, header=version_hdr)
                for i in I:
                    mask = outlier_masks[i]
                    _, _, _, meth, tile = survey.get_compression_args(
                        'outliers_mask', shape=mask.shape)
                    fits.write(mask,
                               header=outlier_hdrs[i],
                               extname=ccds.ccdname[i],
                               compress=meth,
                               tile_dims=tile)
            os.rename(tempfn, outfn)
            print('Wrote', outfn)
    elif opt.outlier_mask is not None:
        with fitsio.FITS(opt.outlier_mask, 'rw', clobber=True) as F:
            F.write(None, header=version_hdr)
            for i, (hdr, mask) in enumerate(zip(outlier_hdrs, outlier_masks)):
                _, _, _, meth, tile = survey.get_compression_args(
                    'outliers_mask', shape=mask.shape)
                F.write(mask,
                        header=hdr,
                        extname=ccds.ccdname[i],
                        compress=meth,
                        tile_dims=tile)
        print('Wrote', opt.outlier_mask)

    tnow = Time()
    print('Total:', tnow - t0)
    return 0
예제 #26
0
def main():
    """Main program.
    """
    import argparse
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--force', action='store_true',
                      help='Run calib processes even if files already exist?')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')

    parser.add_argument('--expnum', type=int, help='Cut to a single exposure')
    parser.add_argument('--extname', '--ccdname', help='Cut to a single extension/CCD name')

    parser.add_argument('--no-psf', dest='psfex', action='store_false',
                      help='Do not compute PsfEx calibs')
    parser.add_argument('--no-sky', dest='sky', action='store_false',
                      help='Do not compute sky models')
    parser.add_argument('--run-se', action='store_true', help='Run SourceExtractor')

    parser.add_argument('--splinesky', action='store_true', help='Spline sky, not constant')
    parser.add_argument('--threads', type=int, help='Run multi-threaded', default=None)
    parser.add_argument('args',nargs=argparse.REMAINDER)
    opt = parser.parse_args()

    survey = LegacySurveyData()
    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        print('Read', len(T), 'from', opt.ccds)
    else:
        T = survey.get_ccds()
        #print len(T), 'CCDs'

    if len(opt.args) == 0:
        if opt.expnum is not None:
            T.cut(T.expnum == opt.expnum)
            print('Cut to', len(T), 'with expnum =', opt.expnum)
        if opt.extname is not None:
            T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname]))
            print('Cut to', len(T), 'with extname =', opt.extname)

        opt.args = range(len(T))

    args = []
    for a in opt.args:
        # Check for "expnum-ccdname" format.
        if '-' in str(a):
            words = a.split('-')
            assert(len(words) == 2)
            expnum = int(words[0])
            ccdname = words[1]
            I = np.flatnonzero((T.expnum == expnum) * (T.ccdname == ccdname))
            if len(I) != 1:
                print('Found', len(I), 'CCDs for expnum', expnum, 'CCDname', ccdname, ':', I)
            assert(len(I) == 1)
            t = T[I[0]]
        else:
            i = int(a)
            print('Index', i)
            t = T[i]

        print('CCDnmatch', t.ccdnmatch)
        if t.ccdnmatch < 20 and not opt.force:
            print('Skipping ccdnmatch = %i' % t.ccdnmatch)
            continue
            
        im = survey.get_image_object(t)
        print('Running', im.calname)
        
        kwargs = dict(psfex=opt.psfex, sky=opt.sky)
        if opt.force:
            kwargs.update(force=True)
        if opt.run_se:
            kwargs.update(se=True)
        if opt.splinesky:
            kwargs.update(splinesky=True)
            
        if opt.threads:
            args.append((im, kwargs))
        else:
            run_calibs((im, kwargs))

    if opt.threads:
        from astrometry.util.multiproc import multiproc
        mp = multiproc(opt.threads)
        mp.map(run_calibs, args)
        
    return 0
예제 #27
0
def main():
    """Main program.
    """
    import argparse

    parser = argparse.ArgumentParser(
        description=
        "This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)."
    )
    parser.add_argument('--calibs',
                        action='store_true',
                        help='Output CCDs that need to be calibrated.')

    parser.add_argument('--nper',
                        type=int,
                        default=None,
                        help='Batch N calibs per line')
    parser.add_argument(
        '--byexp',
        action='store_true',
        default=False,
        help='Run one whole exposure per job (not one CCD per job)')

    parser.add_argument('--forced',
                        action='store_true',
                        help='Output forced-photometry commands')

    parser.add_argument('--lsb',
                        action='store_true',
                        help='Output Low-Surface-Brightness commands')

    parser.add_argument('--stage', help='Stage image files to given directory')

    parser.add_argument('--touching',
                        action='store_true',
                        help='Cut to only CCDs touching selected bricks')
    parser.add_argument('--near',
                        action='store_true',
                        help='Quick cut to only CCDs near selected bricks')

    parser.add_argument('--check-coadd',
                        action='store_true',
                        help='Check which coadds actually need to run.')
    parser.add_argument('--out',
                        help='Output filename for calibs, default %(default)s',
                        default='jobs')
    parser.add_argument('--command',
                        action='store_true',
                        help='Write out full command-line to run calib')
    parser.add_argument('--opt', help='With --command, extra options to add')

    parser.add_argument('--maxra', type=float, help='Maximum RA to run')
    parser.add_argument('--minra', type=float, help='Minimum RA to run')
    parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
    parser.add_argument('--mindec', type=float, help='Minimum Dec to run')

    parser.add_argument('--region', help='Region to select')

    parser.add_argument('--bricks', help='Set bricks.fits file to load')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')
    parser.add_argument('--ignore_cuts',
                        action='store_true',
                        default=False,
                        help='no photometric cuts')
    parser.add_argument('--save_to_fits',
                        action='store_true',
                        default=False,
                        help='save cut brick,ccd to fits table')
    parser.add_argument(
        '--name',
        action='store',
        default='dr3',
        help='save with this suffix, e.g. refers to ccds table')

    parser.add_argument('--delete-sky',
                        action='store_true',
                        help='Delete any existing sky calibration files')

    parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')

    parser.add_argument('--nccds',
                        action='store_true',
                        default=False,
                        help='Prints number of CCDs per brick')

    parser.add_argument('--bands',
                        default='g,r,z',
                        help='Set bands to keep: comma-separated list.')

    opt = parser.parse_args()

    want_ccds = (opt.calibs or opt.forced or opt.lsb)
    want_bricks = not want_ccds

    survey = LegacySurveyData()
    if opt.bricks is not None:
        B = fits_table(opt.bricks)
        log('Read', len(B), 'from', opt.bricks)
    else:
        B = survey.get_bricks()

    log('Bricks Dec range:', B.dec.min(), B.dec.max())

    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        log('Read', len(T), 'from', opt.ccds)
    else:
        T = survey.get_ccds()
        log(len(T), 'CCDs')
    T.index = np.arange(len(T))

    if opt.ignore_cuts == False:
        log('Applying CCD cuts...')
        if 'ccd_cuts' in T.columns():
            T.cut(T.ccd_cuts == 0)
            log(len(T), 'CCDs survive cuts')

    bands = opt.bands.split(',')
    log('Filters:', np.unique(T.filter))
    T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
    log('Cut to', len(T), 'CCDs in filters', bands)

    log('CCDs Dec range:', T.dec.min(), T.dec.max())

    # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
    # plt.clf()
    # plt.hist(counts, counts.max()+1)
    # plt.savefig('bricks.png')
    # B.cut(I[counts >= 9])
    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # #plt.scatter(B.ra[I], B.dec[I], c=counts)
    # plt.savefig('bricks2.png')

    # DES Stripe82
    #rlo,rhi = 350.,360.
    # rlo,rhi = 300., 10.
    # dlo,dhi = -6., 4.
    # TINY bit
    #rlo,rhi = 350.,351.1
    #dlo,dhi = 0., 1.1

    # EDR+
    # 860 bricks
    # ~10,000 CCDs
    #rlo,rhi = 239,246
    #dlo,dhi =   5, 13

    # DR1
    #rlo,rhi = 0, 360
    # part 1
    #dlo,dhi = 25, 40
    # part 2
    #dlo,dhi = 20,25
    # part 3
    #dlo,dhi = 15,20
    # part 4
    #dlo,dhi = 10,15
    # part 5
    #dlo,dhi = 5,10
    # the rest
    #dlo,dhi = -11, 5
    #dlo,dhi = 15,25.5

    dlo, dhi = -25, 40
    rlo, rhi = 0, 360

    # Arjun says 3x3 coverage area is roughly
    # RA=240-252 DEC=6-12 (but not completely rectangular)

    # COSMOS
    #rlo,rhi = 148.9, 151.2
    #dlo,dhi = 0.9, 3.5

    # A nice well-behaved region (EDR2/3)
    # rlo,rhi = 243.6, 244.6
    # dlo,dhi = 8.1, 8.6

    # 56 bricks, ~725 CCDs
    #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
    # 240 bricks, ~3000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
    # 535 bricks, ~7000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))

    if opt.region in ['test1', 'test2', 'test3', 'test4']:
        nm = dict(
            test1='2446p115',  # weird stuff around bright star
            test2='1183p292',  # faint sources around bright galaxy
            test3='3503p005',  # DES
            test4='1163p277',  # Pollux
        )[opt.region]

        B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
        log('Cut to', len(B), 'bricks')
        log(B.ra, B.dec)
        dlo, dhi = -90, 90
        rlo, rhi = 0, 360

    elif opt.region == 'edr':
        # EDR:
        # 535 bricks, ~7000 CCDs
        rlo, rhi = 240, 245
        dlo, dhi = 5, 12

    elif opt.region == 'dr8-decam':
        rlo, rhi = 0, 360
        dlo, dhi = -70, 40
        log('DR8-DECam region')

    elif opt.region == 'edrplus':
        rlo, rhi = 235, 248
        dlo, dhi = 5, 15

    elif opt.region == 'edr-south':
        rlo, rhi = 240, 245
        dlo, dhi = 5, 10

    elif opt.region == 'cosmos1':
        # 16 bricks in the core of the COSMOS field.
        rlo, rhi = 149.75, 150.75
        dlo, dhi = 1.6, 2.6

    elif opt.region == 'pristine':
        # Stream?
        rlo, rhi = 240, 250
        dlo, dhi = 10, 15

    elif opt.region == 'des':
        dlo, dhi = -6., 4.
        rlo, rhi = 317., 7.

        T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
        log('Cut to', len(T), 'CCDs with "CPDES82" in filename')

    elif opt.region == 'subdes':
        rlo, rhi = 320., 360.
        dlo, dhi = -1.25, 1.25

    elif opt.region == 'northwest':
        rlo, rhi = 240, 360
        dlo, dhi = 20, 40
    elif opt.region == 'north':
        rlo, rhi = 120, 240
        dlo, dhi = 20, 40
    elif opt.region == 'northeast':
        rlo, rhi = 0, 120
        dlo, dhi = 20, 40
    elif opt.region == 'southwest':
        rlo, rhi = 240, 360
        dlo, dhi = -20, 0
    elif opt.region == 'south':
        rlo, rhi = 120, 240
        dlo, dhi = -20, 0
    elif opt.region == 'southeast':
        rlo, rhi = 0, 120
        dlo, dhi = -20, 0
    elif opt.region == 'southsoutheast':
        rlo, rhi = 0, 120
        dlo, dhi = -20, -10
    elif opt.region == 'midwest':
        rlo, rhi = 240, 360
        dlo, dhi = 0, 20
    elif opt.region == 'middle':
        rlo, rhi = 120, 240
        dlo, dhi = 0, 20
    elif opt.region == 'mideast':
        rlo, rhi = 0, 120
        dlo, dhi = 0, 20

    elif opt.region == 'grz':
        # Bricks with grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
        log('Cut to', len(B), 'bricks with grz coverage')

    elif opt.region == 'nogrz':
        # Bricks without grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
        log('Cut to', len(B), 'bricks withOUT grz coverage')

    elif opt.region == 'deep2':
        rlo, rhi = 250, 260
        dlo, dhi = 30, 35

    elif opt.region == 'deep2f2':
        rlo, rhi = 251.4, 254.4
        dlo, dhi = 34.6, 35.3

    elif opt.region == 'deep2f3':
        rlo, rhi = 351.25, 353.75
        dlo, dhi = 0, 0.5

    elif opt.region == 'deep3':
        rlo, rhi = 214, 216
        dlo, dhi = 52.25, 53.25

    elif opt.region == 'virgo':
        rlo, rhi = 185, 190
        dlo, dhi = 10, 15

    elif opt.region == 'virgo2':
        rlo, rhi = 182, 192
        dlo, dhi = 8, 18

    elif opt.region == 'coma':
        # van Dokkum et al Coma cluster ultra-diffuse galaxies: 3x3 field centered on Coma cluster
        rc, dc = 195., 28.
        dd = 1.5
        cosdec = np.cos(np.deg2rad(dc))
        rlo, rhi = rc - dd / cosdec, rc + dd / cosdec
        dlo, dhi = dc - dd, dc + dd

    elif opt.region == 'lsb':
        rlo, rhi = 147.2, 147.8
        dlo, dhi = -0.4, 0.4

    elif opt.region == 'eboss-sgc':
        # generous boundaries to make sure get all relevant images
        # RA -45 to +45
        # Dec -5 to +7
        rlo, rhi = 310., 50.
        dlo, dhi = -6., 6.

    elif opt.region == 'eboss-ngc':
        # generous boundaries to make sure get all relevant images
        # NGC ELGs
        # RA 115 to 175
        # Dec 15 to  30
        # rlo,rhi = 122., 177.
        # dlo,dhi =  12.,  32.
        rlo, rhi = 126., 168.
        dlo, dhi = 18., 33.

    elif opt.region == 'mzls':
        dlo, dhi = -10., 90.  # -10: pull in Stripe 82 data too

    elif opt.region == 'dr4-bootes':
        # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
        #dlo,dhi = 34., 35.
        #rlo,rhi = 209.5, 210.5
        dlo, dhi = 33., 36.
        rlo, rhi = 216.5, 219.5

    elif opt.region == 'des-sn-x3':
        #rlo,rhi = 36., 37.
        #dlo,dhi = -5., -4.
        rlo, rhi = 36., 36.5
        dlo, dhi = -4.5, -4.

    elif opt.region == 'ngc2632':
        # open cluster
        rlo, rhi = 129.0, 131.0
        dlo, dhi = 19.0, 20.5

    elif opt.region == 'dr8sky':
        rlo, rhi = 35.0, 37.0
        dlo, dhi = -3.0, -1.0

    # ADM DR8 test regions, see, e.g.:
    # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR8#Testregions
    elif opt.region == 'dr8-test-s82':
        rlo, rhi = 0, 45
        dlo, dhi = -1.25, 1.25
    elif opt.region == 'dr8-test-hsc-sgc':
        rlo, rhi = 30, 40
        dlo, dhi = -6.5, -1.25
    elif opt.region == 'dr8-test-hsc-ngc':
        rlo, rhi = 177.5, 182.5
        dlo, dhi = -1, 1
    elif opt.region == 'dr8-test-edr':
        rlo, rhi = 240, 245
        dlo, dhi = 5, 12
    elif opt.region == 'dr8-test-hsc-north':
        rlo, rhi = 240, 250
        dlo, dhi = 42, 45
    elif opt.region == 'dr8-test-deep2-egs':
        rlo, rhi = 213, 216.5
        dlo, dhi = 52, 54
    elif opt.region == 'dr8-test-overlap':
        rlo, rhi = 132, 140.5
        dlo, dhi = 31.5, 35

    if opt.mindec is not None:
        dlo = opt.mindec
    if opt.maxdec is not None:
        dhi = opt.maxdec
    if opt.minra is not None:
        rlo = opt.minra
    if opt.maxra is not None:
        rhi = opt.maxra

    if rlo < rhi:
        B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi))
    else:  # RA wrap
        B.cut(
            np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) *
            (B.dec <= dhi))
    log(len(B), 'bricks in range; cut Dec range', B.dec.min(), B.dec.max())
    #for name in B.get('brickname'):
    #    print(name)
    #B.writeto('bricks-cut.fits')

    bricksize = 0.25
    # A bit more than 0.25-degree brick radius + Bok image radius ~ 0.57
    search_radius = 1.05 * np.sqrt(2.) * (bricksize +
                                          (0.455 * 4096 / 3600.)) / 2.

    log(len(T), 'CCDs')
    log(len(B), 'Bricks')
    I, J, d = match_radec(B.ra,
                          B.dec,
                          T.ra,
                          T.dec,
                          search_radius,
                          nearest=True)
    B.cut(I)
    log('Cut to', len(B), 'bricks near CCDs')
    log('Bricks Dec range:', B.dec.min(), B.dec.max())

    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # plt.title('DR3 bricks')
    # plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
    # plt.savefig('bricks.png')

    if opt.touching:
        I, J, d = match_radec(T.ra,
                              T.dec,
                              B.ra,
                              B.dec,
                              search_radius,
                              nearest=True)
        # list the ones that will be cut
        # drop = np.ones(len(T))
        # drop[I] = False
        # for i in np.flatnonzero(drop):
        #     from astrometry.util.starutil_numpy import degrees_between
        #     dists = degrees_between(B.ra, B.dec, T.ra[i], T.dec[i])
        #     mindist = min(dists)
        #     print('Dropping:', T.ra[i], T.dec[i], 'min dist', mindist, 'search_radius', search_radius)

        T.cut(I)
        log('Cut to', len(T), 'CCDs near bricks')

    # sort by RA increasing
    B.cut(np.argsort(B.ra))

    if opt.save_to_fits:
        assert (opt.touching)
        # Write cut tables to file
        for tab, typ in zip([B, T], ['bricks', 'ccds']):
            fn = '%s-%s-cut.fits' % (typ, opt.region)
            if os.path.exists(fn):
                os.remove(fn)
            tab.writeto(fn)
            log('Wrote %s' % fn)
        # Write text files listing ccd and filename names
        # nm1,nm2= 'ccds-%s.txt'% opt.region,'filenames-%s.txt' % opt.region
        # if os.path.exists(nm1):
        #     os.remove(nm1)
        # if os.path.exists(nm2):
        #     os.remove(nm2)
        # f1,f2=open(nm1,'w'),open(nm2,'w')
        # fns= list(set(T.get('image_filename')))
        # for fn in fns:
        #     f2.write('%s\n' % fn.strip())
        # for ti in T:
        #     f1.write('%s\n' % ti.get('image_filename').strip())
        # f1.close()
        # f2.close()
        # log('Wrote *-names.txt')

    if opt.touching:

        if want_bricks:
            # Shortcut the list of bricks that are definitely touching CCDs --
            # a brick-ccd pair within this radius must be touching.
            closest_radius = 0.95 * (bricksize + 0.262 * 2048 / 3600.) / 2.

            J1, nil, nil = match_radec(B.ra,
                                       B.dec,
                                       T.ra,
                                       T.dec,
                                       closest_radius,
                                       nearest=True)
            log(len(J1), 'of', len(B), 'bricks definitely touch CCDs')
            tocheck = np.ones(len(B), bool)
            tocheck[J1] = False
            J2 = []
            for j in np.flatnonzero(tocheck):
                b = B[j]
                wcs = wcs_for_brick(b)
                I = ccds_touching_wcs(wcs, T)
                log(len(I), 'CCDs for brick', b.brickname)
                if len(I) == 0:
                    continue
                J2.append(j)
            J = np.hstack((J1, J2))
            J = np.sort(J).astype(int)
            B.cut(J)
            log('Cut to', len(B), 'bricks touching CCDs')

        else:
            J = []
            allI = set()
            for j, b in enumerate(B):
                wcs = wcs_for_brick(b)
                I = ccds_touching_wcs(wcs, T)
                log(len(I), 'CCDs for brick', b.brickname)
                if len(I) == 0:
                    continue
                allI.update(I)
                J.append(j)
            allI = list(allI)
            allI.sort()
            B.cut(np.array(J))
            log('Cut to', len(B), 'bricks touching CCDs')

    elif opt.near:
        # Find CCDs near bricks
        allI, nil, nil = match_radec(T.ra,
                                     T.dec,
                                     B.ra,
                                     B.dec,
                                     search_radius,
                                     nearest=True)
        # Find bricks near CCDs
        J, nil, nil = match_radec(B.ra,
                                  B.dec,
                                  T.ra,
                                  T.dec,
                                  search_radius,
                                  nearest=True)
        B.cut(J)
        log('Cut to', len(B), 'bricks near CCDs')

    else:
        allI = np.arange(len(T))

    if opt.byexp:
        nil, eI = np.unique(T.expnum[allI], return_index=True)
        allI = allI[eI]
        print('Cut to', len(allI), 'expnums')

    if opt.nccds:
        from queue import Queue
        from threading import Thread

        log('Checking number of CCDs per brick')

        def worker():
            while True:
                i = q.get()
                if i is None:
                    break
                b = B[i]
                wcs = wcs_for_brick(b)
                I = ccds_touching_wcs(wcs, T)
                log(b.brickname, len(I))
                q.task_done()

        q = Queue()
        num_threads = 24
        threads = []

        for i in range(num_threads):
            t = Thread(target=worker)
            t.start()
            threads.append(t)

        for i in range(len(B)):
            q.put(i)

        q.join()
        for i in range(num_threads):
            q.put(None)
        for t in threads:
            t.join()

    if opt.write_ccds:
        T[allI].writeto(opt.write_ccds)
        log('Wrote', opt.write_ccds)

    if want_bricks:
        # Print the list of bricks and exit.
        for b in B:
            print(b.brickname)
        if opt.save_to_fits:
            B.writeto('bricks-%s-touching.fits' % opt.region)
        if not want_ccds:
            sys.exit(0)

    ## Be careful here -- T has been cut; we want to write out T.index.
    ## 'allI' contains indices into T.

    if opt.stage is not None:
        cmd_pat = 'rsync -LRarv %s %s'
        fns = set()
        for iccd in allI:
            im = survey.get_image_object(T[iccd])
            fns.update([
                im.imgfn, im.wtfn, im.dqfn, im.psffn, im.merged_psffn,
                im.merged_splineskyfn, im.splineskyfn
            ])
        for i, fn in enumerate(fns):
            print('File', i + 1, 'of', len(fns), ':', fn)
            if not os.path.exists(fn):
                print('No such file:', fn)
                continue
            base = survey.get_survey_dir()
            if base.endswith('/'):
                base = base[:-1]

            rel = os.path.relpath(fn, base)

            dest = os.path.join(opt.stage, rel)
            print('Dest:', dest)
            if os.path.exists(dest):
                print('Exists:', dest)
                continue

            cmd = cmd_pat % ('%s/./%s' % (base, rel), opt.stage)
            print(cmd)
            rtn = os.system(cmd)
            assert (rtn == 0)
        sys.exit(0)

    if opt.forced:
        log('Writing forced-photometry commands to', opt.out)
        f = open(opt.out, 'w')
        log('Total of', len(allI), 'CCDs')
        for j, i in enumerate(allI):
            expstr = '%08i' % T.expnum[i]
            imgfn = os.path.join(survey.survey_dir, 'images',
                                 T.image_filename[i].strip())
            if (not os.path.exists(imgfn) and imgfn.endswith('.fz')
                    and os.path.exists(imgfn[:-3])):
                imgfn = imgfn[:-3]

            outfn = os.path.join(
                expstr[:5], expstr, 'forced-%s-%s-%s.fits' %
                (T.camera[i].strip(), expstr, T.ccdname[i]))

            f.write(
                'python legacypipe/forced_photom.py --apphot --derivs --catalog-dir /project/projectdirs/cosmo/data/legacysurvey/dr7/ %i %s forced/%s\n'
                % (T.expnum[i], T.ccdname[i], outfn))

        f.close()
        log('Wrote', opt.out)

        fn = 'forced-ccds.fits'
        T[allI].writeto(fn)
        print('Wrote', fn)

        sys.exit(0)

    if opt.lsb:
        log('Writing LSB commands to', opt.out)
        f = open(opt.out, 'w')
        log('Total of', len(allI), 'CCDs')
        for j, i in enumerate(allI):
            exp = T.expnum[i]
            ext = T.ccdname[i].strip()
            outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
            f.write(
                'python legacyanalysis/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n'
                % (exp, ext, outfn, exp, ext))
        f.close()
        log('Wrote', opt.out)
        sys.exit(0)

    log('Writing calibs to', opt.out)
    f = open(opt.out, 'w')
    log('Total of', len(allI), 'CCDs')

    batch = []

    def write_batch(f, batch, cmd):
        if cmd is None:
            cmd = ''
        f.write(cmd + ' '.join(batch) + '\n')

    cmd = None
    if opt.command:
        cmd = 'python legacypipe/run-calib.py '
        if opt.opt is not None:
            cmd += opt.opt + ' '

    for j, i in enumerate(allI):

        if opt.delete_sky:
            log(j + 1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if opt.delete_sky and os.path.exists(im.skyfn):
                log('  deleting:', im.skyfn)
                os.unlink(im.skyfn)

        if opt.command:
            if opt.byexp:
                s = '--expnum %i' % (T.expnum[i])
            else:
                s = '%i-%s' % (T.expnum[i], T.ccdname[i])
            prefix = 'python legacypipe/run-calib.py '
            if opt.opt is not None:
                prefix = prefix + opt.opt
            #('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
            #     (T.expnum[i], T.ccdname[i]))
        else:
            s = '%i' % T.index[i]
            prefix = ''

        if j < 10:
            print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname',
                  T.ccdname[i], 'filename', T.image_filename[i])

        if not opt.nper:
            f.write(prefix + s + '\n')
        else:
            batch.append(s)
            if len(batch) >= opt.nper:
                write_batch(f, batch, cmd)
                batch = []

    if len(batch):
        write_batch(f, batch, cmd)

    f.close()
    log('Wrote', opt.out)
    return 0
예제 #28
0
def psf_residuals(expnum,ccdname,stampsize=35,nstar=30,
                  magrange=(13,17),verbose=0, splinesky=False):

    # Set the debugging level.
    if verbose==0:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG
    logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout)

    pngprefix = 'qapsf-{}-{}'.format(expnum,ccdname)

    # Gather all the info we need about this CCD.
    survey = LegacySurveyData()
    ccd = survey.find_ccds(expnum=expnum,ccdname=ccdname)[0]
    band = ccd.filter
    ps1band = dict(g=0,r=1,i=2,z=3,Y=4)
    print('Band {}'.format(band))

    #scales = dict(g=0.0066, r=0.01, z=0.025)
    #vmin, vmax = np.arcsinh(-1), np.arcsinh(100)
    #print(scales[band])

    im = survey.get_image_object(ccd)
    iminfo = im.get_image_info()
    H,W = iminfo['dims']

    wcs = im.get_wcs()

    # Choose a uniformly selected subset of PS1 stars on this CCD.
    ps1 = ps1cat(ccdwcs=wcs)
    cat = ps1.get_stars(band=band,magrange=magrange)

    rand = np.random.RandomState(seed=expnum*ccd.ccdnum)
    these = rand.choice(len(cat)-1,nstar,replace=False)
    #these = rand.random_integers(0,len(cat)-1,nstar)
    cat = cat[these]
    cat = cat[np.argsort(cat.median[:,ps1band[band]])] # sort by magnitude
    #print(cat.nmag_ok)

    get_tim_kwargs = dict(pixPsf=True, splinesky=splinesky)

    # Make a QAplot of the positions of all the stars.
    tim = im.get_tractor_image(**get_tim_kwargs)
    img = tim.getImage()
    #img = tim.getImage()/scales[band]

    fig = plt.figure(figsize=(5,10))
    ax = fig.gca()
    ax.get_xaxis().get_major_formatter().set_useOffset(False)
    #ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest',
    #          origin='lower',vmin=vmax,vmax=vmax)
    
    ax.imshow(img, **tim.ima)
    ax.axis('off')
    ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.
                 format(band,expnum,ccdname,ccd.airmass,ccd.seeing))

    for istar, ps1star in enumerate(cat):
        ra, dec = (ps1star.ra, ps1star.dec)
        ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
        ax.text(xpos,ypos,'{:2d}'.format(istar+1),color='red',
                horizontalalignment='left')
        circ = plt.Circle((xpos,ypos),radius=30,color='g',fill=False,lw=1)
        ax.add_patch(circ)

    #radec = wcs.radec_bounds()
    #ax.scatter(cat.ra,cat.dec)
    #ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998])
    #ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015])
    #ax.set_xlabel('$RA\ (deg)$',fontsize=18)
    #ax.set_ylabel('$Dec\ (deg)$',fontsize=18)
    fig.savefig(pngprefix+'-ccd.png',bbox_inches='tight')

    # Initialize the many-stamp QAplot
    ncols = 3
    nrows = np.ceil(nstar/ncols).astype('int')

    inchperstamp = 2.0
    fig = plt.figure(figsize=(inchperstamp*3*ncols,inchperstamp*nrows))
    irow = 0
    icol = 0
    
    for istar, ps1star in enumerate(cat):
        ra, dec = (ps1star.ra, ps1star.dec)
        mag = ps1star.median[ps1band[band]] # r-band

        ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
        ix,iy = int(xpos), int(ypos)

        # create a little tractor Image object around the star
        slc = (slice(max(iy-stampsize, 0), min(iy+stampsize+1, H)),
               slice(max(ix-stampsize, 0), min(ix+stampsize+1, W)))

        # The PSF model 'const2Psf' is the one used in DR1: a 2-component
        # Gaussian fit to PsfEx instantiated in the image center.
        tim = im.get_tractor_image(slc=slc, **get_tim_kwargs)
        stamp = tim.getImage()
        ivarstamp = tim.getInvvar()

        # Initialize a tractor PointSource from PS1 measurements
        flux = NanoMaggies.magToNanomaggies(mag)
        star = PointSource(RaDecPos(ra,dec), NanoMaggies(**{band: flux}))

        # Fit just the source RA,Dec,flux.
        tractor = Tractor([tim], [star])
        tractor.freezeParam('images')

        print('2-component MOG:', tim.psf)
        tractor.printThawedParams()

        for step in range(50):
            dlnp,X,alpha = tractor.optimize()
            if dlnp < 0.1:
                break
        print('Fit:', star)
        model_mog = tractor.getModelImage(0)
        chi2_mog = -2.0*tractor.getLogLikelihood()
        mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0]

        # Now change the PSF model to a pixelized PSF model from PsfEx instantiated
        # at this place in the image.
        psf = PixelizedPsfEx(im.psffn)
        tim.psf = psf.constantPsfAt(xpos, ypos)

        #print('PSF model:', tim.psf)
        #tractor.printThawedParams()
        for step in range(50):
            dlnp,X,alpha = tractor.optimize()
            if dlnp < 0.1:
                break

        print('Fit:', star)
        model_psfex = tractor.getModelImage(0)
        chi2_psfex = -2.0*tractor.getLogLikelihood()
        mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0]

        #mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95])
        sig = np.std((stamp-model_psfex)[ivarstamp>0])
        mn, mx = [-2.0*sig,5*sig]

        # Generate a QAplot.
        if (istar>0) and (istar%(ncols)==0):
            irow = irow+1
        icol = 3*istar - 3*ncols*irow
        #print(istar, irow, icol, icol+1, icol+2)

        ax1 = plt.subplot2grid((nrows,3*ncols), (irow,icol), aspect='equal')
        ax1.axis('off')
        #ax1.imshow(stamp, **tim.ima)
        ax1.imshow(stamp,cmap='gray',interpolation='nearest',
                   origin='lower',vmin=mn,vmax=mx)
        ax1.text(0.1,0.9,'{:2d}'.format(istar+1),color='white',
                horizontalalignment='left',verticalalignment='top',
                transform=ax1.transAxes)

        ax2 = plt.subplot2grid((nrows,3*ncols), (irow,icol+1), aspect='equal')
        ax2.axis('off')
        #ax2.imshow(stamp-model_mog, **tim.ima)
        ax2.imshow(stamp-model_mog,cmap='gray',interpolation='nearest',
                   origin='lower',vmin=mn,vmax=mx)
        ax2.text(0.1,0.9,'MoG',color='white',
                horizontalalignment='left',verticalalignment='top',
                transform=ax2.transAxes)
        ax2.text(0.08,0.08,'{:.3f}'.format(mag_mog),color='white',
                 horizontalalignment='left',verticalalignment='bottom',
                 transform=ax2.transAxes)

        #ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14)
        #ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex))

        ax3 = plt.subplot2grid((nrows,3*ncols), (irow,icol+2), aspect='equal')
        ax3.axis('off')
        #ax3.imshow(stamp-model_psfex, **tim.ima)
        ax3.imshow(stamp-model_psfex,cmap='gray',interpolation='nearest',
                   origin='lower',vmin=mn,vmax=mx)
        ax3.text(0.1,0.9,'PSFEx',color='white',
                horizontalalignment='left',verticalalignment='top',
                transform=ax3.transAxes)
        ax3.text(0.08,0.08,'{:.3f}'.format(mag_psfex),color='white',
                 horizontalalignment='left',verticalalignment='bottom',
                 transform=ax3.transAxes)

        if istar==(nstar-1):
            break
    fig.savefig(pngprefix+'-stargrid.png',bbox_inches='tight')
예제 #29
0
def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--expnum',
        type=str,
        help='Run specified exposure numbers (can be comma-separated list')
    parser.add_argument(
        '--all-found',
        action='store_true',
        default=False,
        help='Only write output if all required input files are found')
    parser.add_argument('--ccds',
                        help='Set ccds.fits file to load, default is all')
    parser.add_argument('--continue',
                        dest='con',
                        help='Continue even if one exposure is bad',
                        action='store_true',
                        default=False)
    parser.add_argument('--outdir',
                        help='Output directory, default %(default)s',
                        default='calib')

    opt = parser.parse_args()

    survey = LegacySurveyData()
    if opt.ccds:
        ccds = fits_table(opt.ccds)
        ccds = survey.cleanup_ccds_table(ccds)
        survey.ccds = ccds

    if opt.expnum is not None:
        expnums = [(None, int(x, 10)) for x in opt.expnum.split(',')]
    else:
        ccds = survey.get_ccds()
        expnums = set(zip(ccds.camera, ccds.expnum))
        print(len(expnums), 'unique camera+expnums')

    for i, (camera, expnum) in enumerate(expnums):
        print()
        print('Exposure', i + 1, 'of', len(expnums), ':', camera, 'expnum',
              expnum)
        if camera is None:
            C = survey.find_ccds(expnum=expnum)
            print(len(C), 'CCDs with expnum', expnum)
            camera = C.camera[0]
            print('Set camera to', camera)

        C = survey.find_ccds(expnum=expnum, camera=camera)
        print(len(C), 'CCDs with expnum', expnum, 'and camera', camera)

        im0 = survey.get_image_object(C[0])

        skyoutfn = im0.merged_skyfn
        psfoutfn = im0.merged_psffn

        print('Checking for', skyoutfn)
        print('Checking for', psfoutfn)
        if os.path.exists(skyoutfn) and os.path.exists(psfoutfn):
            print('Exposure', expnum, 'is done already')
            continue

        if not os.path.exists(skyoutfn):
            try:
                merge_splinesky(survey, expnum, C, skyoutfn, opt)
            except:
                if not opt.con:
                    raise
                import traceback
                traceback.print_exc()
                print('Exposure failed:', expnum, '.  Continuing...')

        if not os.path.exists(psfoutfn):
            try:
                merge_psfex(survey, expnum, C, psfoutfn, opt)
            except:
                if not opt.con:
                    raise
                import traceback
                traceback.print_exc()
                print('Exposure failed:', expnum, '.  Continuing...')
예제 #30
0
def main():
    """Main program.
    """
    import argparse

    parser = argparse.ArgumentParser(
        description=
        "This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)."
    )
    parser.add_argument('--calibs',
                        action='store_true',
                        help='Output CCDs that need to be calibrated.')

    parser.add_argument('--nper',
                        type=int,
                        default=None,
                        help='Batch N calibs per line')

    parser.add_argument('--forced',
                        action='store_true',
                        help='Output forced-photometry commands')

    parser.add_argument('--lsb',
                        action='store_true',
                        help='Output Low-Surface-Brightness commands')

    parser.add_argument('--touching',
                        action='store_true',
                        help='Cut to only CCDs touching selected bricks')
    parser.add_argument('--near',
                        action='store_true',
                        help='Quick cut to only CCDs near selected bricks')

    parser.add_argument('--check',
                        action='store_true',
                        help='Check which calibrations actually need to run.')
    parser.add_argument('--check-coadd',
                        action='store_true',
                        help='Check which caoadds actually need to run.')
    parser.add_argument('--out',
                        help='Output filename for calibs, default %(default)s',
                        default='jobs')
    parser.add_argument('--command',
                        action='store_true',
                        help='Write out full command-line to run calib')
    parser.add_argument('--opt', help='With --command, extra options to add')

    parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
    parser.add_argument('--mindec', type=float, help='Minimum Dec to run')

    parser.add_argument('--region', help='Region to select')

    parser.add_argument('--bricks', help='Set bricks.fits file to load')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')
    parser.add_argument('--ignore_cuts',
                        action='store_true',
                        default=False,
                        help='no photometric or blacklist cuts')
    parser.add_argument('--save_to_fits',
                        action='store_true',
                        default=False,
                        help='save cut brick,ccd to fits table')
    parser.add_argument(
        '--name',
        action='store',
        default='dr3',
        help='save with this suffix, e.g. refers to ccds table')

    parser.add_argument('--delete-sky',
                        action='store_true',
                        help='Delete any existing sky calibration files')
    parser.add_argument('--delete-pvastrom',
                        action='store_true',
                        help='Delete any existing PV WCS calibration files')

    parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')

    parser.add_argument(
        '--brickq',
        type=int,
        default=None,
        help='Queue only bricks with the given "brickq" value [0 to 3]')

    parser.add_argument(
        '--brickq-deps',
        action='store_true',
        default=False,
        help='Queue bricks directly using qdo API, setting brickq dependencies'
    )
    parser.add_argument('--queue',
                        default='bricks',
                        help='With --brickq-deps, the QDO queue name to use')

    opt = parser.parse_args()

    survey = LegacySurveyData()
    if opt.bricks is not None:
        B = fits_table(opt.bricks)
        log('Read', len(B), 'from', opt.bricks)
    else:
        B = survey.get_bricks()

    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        log('Read', len(T), 'from', opt.ccds)
    else:
        T = survey.get_ccds()
        log(len(T), 'CCDs')
    T.index = np.arange(len(T))

    if opt.ignore_cuts == False:
        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
    print(len(T), 'CCDs remain')

    # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
    # plt.clf()
    # plt.hist(counts, counts.max()+1)
    # plt.savefig('bricks.png')
    # B.cut(I[counts >= 9])
    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # #plt.scatter(B.ra[I], B.dec[I], c=counts)
    # plt.savefig('bricks2.png')

    # DES Stripe82
    #rlo,rhi = 350.,360.
    # rlo,rhi = 300., 10.
    # dlo,dhi = -6., 4.
    # TINY bit
    #rlo,rhi = 350.,351.1
    #dlo,dhi = 0., 1.1

    # EDR+
    # 860 bricks
    # ~10,000 CCDs
    #rlo,rhi = 239,246
    #dlo,dhi =   5, 13

    # DR1
    #rlo,rhi = 0, 360
    # part 1
    #dlo,dhi = 25, 40
    # part 2
    #dlo,dhi = 20,25
    # part 3
    #dlo,dhi = 15,20
    # part 4
    #dlo,dhi = 10,15
    # part 5
    #dlo,dhi = 5,10
    # the rest
    #dlo,dhi = -11, 5
    #dlo,dhi = 15,25.5

    dlo, dhi = -25, 40
    rlo, rhi = 0, 360

    # Arjun says 3x3 coverage area is roughly
    # RA=240-252 DEC=6-12 (but not completely rectangular)

    # COSMOS
    #rlo,rhi = 148.9, 151.2
    #dlo,dhi = 0.9, 3.5

    # A nice well-behaved region (EDR2/3)
    # rlo,rhi = 243.6, 244.6
    # dlo,dhi = 8.1, 8.6

    # 56 bricks, ~725 CCDs
    #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
    # 240 bricks, ~3000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
    # 535 bricks, ~7000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))

    if opt.region in ['test1', 'test2', 'test3', 'test4']:
        nm = dict(
            test1='2446p115',  # weird stuff around bright star
            test2='1183p292',  # faint sources around bright galaxy
            test3='3503p005',  # DES
            test4='1163p277',  # Pollux
        )[opt.region]

        B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
        log('Cut to', len(B), 'bricks')
        log(B.ra, B.dec)
        dlo, dhi = -90, 90
        rlo, rhi = 0, 360

    elif opt.region == 'edr':
        # EDR:
        # 535 bricks, ~7000 CCDs
        rlo, rhi = 240, 245
        dlo, dhi = 5, 12

    elif opt.region == 'edrplus':
        rlo, rhi = 235, 248
        dlo, dhi = 5, 15

    elif opt.region == 'edr-south':
        rlo, rhi = 240, 245
        dlo, dhi = 5, 10

    elif opt.region == 'cosmos1':
        # 16 bricks in the core of the COSMOS field.
        rlo, rhi = 149.75, 150.75
        dlo, dhi = 1.6, 2.6

    elif opt.region == 'pristine':
        # Stream?
        rlo, rhi = 240, 250
        dlo, dhi = 10, 15

    elif opt.region == 'des':
        dlo, dhi = -6., 4.
        rlo, rhi = 317., 7.

        T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
        log('Cut to', len(T), 'CCDs with "CPDES82" in filename')

    elif opt.region == 'subdes':
        rlo, rhi = 320., 360.
        dlo, dhi = -1.25, 1.25

    elif opt.region == 'northwest':
        rlo, rhi = 240, 360
        dlo, dhi = 20, 40
    elif opt.region == 'north':
        rlo, rhi = 120, 240
        dlo, dhi = 20, 40
    elif opt.region == 'northeast':
        rlo, rhi = 0, 120
        dlo, dhi = 20, 40
    elif opt.region == 'southwest':
        rlo, rhi = 240, 360
        dlo, dhi = -20, 0
    elif opt.region == 'south':
        rlo, rhi = 120, 240
        dlo, dhi = -20, 0
    elif opt.region == 'southeast':
        rlo, rhi = 0, 120
        dlo, dhi = -20, 0
    elif opt.region == 'southsoutheast':
        rlo, rhi = 0, 120
        dlo, dhi = -20, -10
    elif opt.region == 'midwest':
        rlo, rhi = 240, 360
        dlo, dhi = 0, 20
    elif opt.region == 'middle':
        rlo, rhi = 120, 240
        dlo, dhi = 0, 20
    elif opt.region == 'mideast':
        rlo, rhi = 0, 120
        dlo, dhi = 0, 20

    elif opt.region == 'grz':
        # Bricks with grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
        log('Cut to', len(B), 'bricks with grz coverage')

    elif opt.region == 'nogrz':
        # Bricks without grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
        log('Cut to', len(B), 'bricks withOUT grz coverage')
    elif opt.region == 'deep2':
        rlo, rhi = 250, 260
        dlo, dhi = 30, 35

    elif opt.region == 'deep2f3':
        rlo, rhi = 351.25, 353.75
        dlo, dhi = 0, 0.5

    elif opt.region == 'virgo':
        rlo, rhi = 185, 190
        dlo, dhi = 10, 15

    elif opt.region == 'virgo2':
        rlo, rhi = 182, 192
        dlo, dhi = 8, 18

    elif opt.region == 'lsb':
        rlo, rhi = 147.2, 147.8
        dlo, dhi = -0.4, 0.4

    elif opt.region == 'eboss-sgc':
        # generous boundaries to make sure get all relevant images
        # RA -45 to +45
        # Dec -5 to +7
        rlo, rhi = 310., 50.
        dlo, dhi = -6., 6.

    elif opt.region == 'eboss-ngc':
        # generous boundaries to make sure get all relevant images
        # NGC ELGs
        # RA 115 to 175
        # Dec 15 to  30
        rlo, rhi = 122., 177.
        dlo, dhi = 12., 32.

    elif opt.region == 'mzls':
        dlo, dhi = 30., 90.
    elif opt.region == 'dr4-bootes':
        # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
        #dlo,dhi = 34., 35.
        #rlo,rhi = 209.5, 210.5
        dlo, dhi = 33., 36.
        rlo, rhi = 216.5, 219.5

    if opt.mindec is not None:
        dlo = opt.mindec
    if opt.maxdec is not None:
        dhi = opt.maxdec

    if rlo < rhi:
        B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi))
    else:  # RA wrap
        B.cut(
            np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) *
            (B.dec <= dhi))
    log(len(B), 'bricks in range')
    #for name in B.get('brickname'):
    #print(name)
    B.writeto('bricks-cut.fits')

    I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize)
    keep = np.zeros(len(B), bool)
    for i in I:
        keep[i] = True
    B.cut(keep)
    log('Cut to', len(B), 'bricks near CCDs')

    plt.clf()
    plt.plot(B.ra, B.dec, 'b.')
    plt.title('DR3 bricks')
    plt.axis([360, 0, np.min(B.dec) - 1, np.max(B.dec) + 1])
    plt.savefig('bricks.png')

    if opt.brickq is not None:
        B.cut(B.brickq == opt.brickq)
        log('Cut to', len(B), 'with brickq =', opt.brickq)

    if opt.touching:
        keep = np.zeros(len(T), bool)
        for j in J:
            keep[j] = True
        T.cut(keep)
        log('Cut to', len(T), 'CCDs near bricks')

    # Aside -- how many near DR1=1 CCDs?
    if False:
        T2 = D.get_ccds()
        log(len(T2), 'CCDs')
        T2.cut(T2.dr1 == 1)
        log(len(T2), 'CCDs marked DR1=1')
        log(len(B), 'bricks in range')
        I, J, d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize)
        keep = np.zeros(len(B), bool)
        for i in I:
            keep[i] = True
        B2 = B[keep]
        log('Total of', len(B2), 'bricks near CCDs with DR1=1')
        for band in 'grz':
            Tb = T2[T2.filter == band]
            log(len(Tb), 'in filter', band)
            I, J, d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec,
                                  survey.bricksize)
            good = np.zeros(len(B2), np.uint8)
            for i in I:
                good[i] = 1
            B2.set('has_' + band, good)

        B2.writeto('survey-bricks-in-dr1.fits')
        sys.exit(0)

    # sort by dec decreasing
    #B.cut(np.argsort(-B.dec))
    # RA increasing
    B.cut(np.argsort(B.ra))

    for b in B:
        if opt.check:
            fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3],
                                                      b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue
        if opt.check_coadd:
            fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (
                b.brickname[:3], b.brickname, b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue

        #print(b.brickname)

    if opt.save_to_fits:
        assert (opt.touching)
        # Write cut tables to file
        for tab, typ in zip([B, T], ['bricks', 'ccds']):
            fn = '%s-%s-cut.fits' % (typ, opt.region)
            if os.path.exists(fn):
                os.remove(fn)
            tab.writeto(fn)
            print('Wrote %s' % fn)
        # Write text files listing ccd and filename names
        nm1, nm2 = 'ccds-%s.txt' % opt.region, 'filenames-%s.txt' % opt.region
        if os.path.exists(nm1):
            os.remove(nm1)
        if os.path.exists(nm2):
            os.remove(nm2)
        f1, f2 = open(nm1, 'w'), open(nm2, 'w')
        fns = list(set(T.get('image_filename')))
        for fn in fns:
            f2.write('%s\n' % fn.strip())
        for ti in T:
            f1.write('%s\n' % ti.get('image_filename').strip())
        f1.close()
        f2.close()
        print('Wrote *-names.txt')

    if opt.brickq_deps:
        import qdo
        from legacypipe.survey import on_bricks_dependencies

        #... find Queue...
        q = qdo.connect(opt.queue, create_ok=True)
        print('Connected to QDO queue', opt.queue, q)
        brick_to_task = dict()

        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
        print(len(T), 'CCDs remaining')

        T.wra = T.ra + (T.ra > 180) * -360
        wra = rlo - 360
        plt.clf()
        plt.plot(T.wra, T.dec, 'b.')
        ax = [wra, rhi, dlo, dhi]
        plt.axis(ax)
        plt.title('CCDs')
        plt.savefig('q-ccds.png')

        B.wra = B.ra + (B.ra > 180) * -360

        # this slight overestimate (for DECam images) is fine
        radius = 0.3
        Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius, indexlist=True)
        ikeep = []
        for ib, (b, Iccd) in enumerate(zip(B, Iccds)):
            if Iccd is None or len(Iccd) == 0:
                print('No matched CCDs to brick', b.brickname)
                continue
            wcs = wcs_for_brick(b)
            cI = ccds_touching_wcs(wcs, T[np.array(Iccd)])
            print(len(cI), 'CCDs touching brick', b.brickname)
            if len(cI) == 0:
                continue
            ikeep.append(ib)
        B.cut(np.array(ikeep))
        print('Cut to', len(B), 'bricks touched by CCDs')

        for brickq in range(4):
            I = np.flatnonzero(B.brickq == brickq)
            print(len(I), 'bricks with brickq =', brickq)

            J = np.flatnonzero(B.brickq < brickq)
            preB = B[J]
            reqs = []
            if brickq > 0:
                for b in B[I]:
                    # find brick dependencies
                    brickdeps = on_bricks_dependencies(b, survey, bricks=preB)
                    # convert to task ids
                    taskdeps = [
                        brick_to_task.get(b.brickname, None) for b in brickdeps
                    ]
                    # If we dropped a dependency brick from a previous brickq because
                    # of no overlapping CCDs, it won't appear in the brick_to_task map.
                    taskdeps = [t for t in taskdeps if t is not None]
                    reqs.append(taskdeps)

            plt.clf()
            plt.plot(B.wra, B.dec, '.', color='0.5')
            plt.plot(B.wra[I], B.dec[I], 'b.')
            plt.axis(ax)
            plt.title('Bricks: brickq=%i' % brickq)
            plt.savefig('q-bricks-%i.png' % brickq)

            # submit to qdo queue
            print('Queuing', len(B[I]), 'bricks')
            if brickq == 0:
                reqs = None
            else:
                assert (len(I) == len(reqs))
            taskids = q.add_multiple(B.brickname[I], requires=reqs)
            assert (len(taskids) == len(I))
            print('Queued', len(taskids), 'bricks')
            brick_to_task.update(dict(zip(B.brickname[I], taskids)))

    if not (opt.calibs or opt.forced or opt.lsb):
        sys.exit(0)

    bands = 'grz'
    log('Filters:', np.unique(T.filter))
    T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
    log('Cut to', len(T), 'CCDs in filters', bands)

    if opt.touching:
        allI = set()
        for b in B:
            wcs = wcs_for_brick(b)
            I = ccds_touching_wcs(wcs, T)
            log(len(I), 'CCDs for brick', b.brickid,
                'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
            if len(I) == 0:
                continue
            allI.update(I)
        allI = list(allI)
        allI.sort()
    elif opt.near:
        # Roughly brick radius + DECam image radius
        radius = 0.35
        allI, nil, nil = match_radec(T.ra,
                                     T.dec,
                                     B.ra,
                                     B.dec,
                                     radius,
                                     nearest=True)
    else:
        allI = np.arange(len(T))

    if opt.write_ccds:
        T[allI].writeto(opt.write_ccds)
        log('Wrote', opt.write_ccds)

    ## Be careful here -- T has been cut; we want to write out T.index.
    ## 'allI' contains indices into T.

    if opt.forced:
        log('Writing forced-photometry commands to', opt.out)
        f = open(opt.out, 'w')
        log('Total of', len(allI), 'CCDs')
        for j, i in enumerate(allI):
            expstr = '%08i' % T.expnum[i]
            outfn = os.path.join(
                'forced', expstr[:5], expstr,
                'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i]))
            imgfn = os.path.join(survey.survey_dir, 'images',
                                 T.image_filename[i].strip())
            if (not os.path.exists(imgfn) and imgfn.endswith('.fz')
                    and os.path.exists(imgfn[:-3])):
                imgfn = imgfn[:-3]

            #f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' %
            #        (imgfn, T.image_hdu[i], outfn))

            f.write(
                'python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n'
                % (T.expnum[i], T.ccdname[i], outfn))

        f.close()
        log('Wrote', opt.out)
        sys.exit(0)

    if opt.lsb:
        log('Writing LSB commands to', opt.out)
        f = open(opt.out, 'w')
        log('Total of', len(allI), 'CCDs')
        for j, i in enumerate(allI):
            exp = T.expnum[i]
            ext = T.ccdname[i].strip()
            outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
            f.write(
                'python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n'
                % (exp, ext, outfn, exp, ext))
        f.close()
        log('Wrote', opt.out)
        sys.exit(0)

    log('Writing calibs to', opt.out)
    f = open(opt.out, 'w')
    log('Total of', len(allI), 'CCDs')

    batch = []

    def write_batch(f, batch, cmd):
        if cmd is None:
            cmd = ''
        f.write(cmd + ' '.join(batch) + '\n')

    cmd = None
    if opt.command:
        cmd = 'python legacypipe/run-calib.py '
        if opt.opt is not None:
            cmd += opt.opt + ' '

    for j, i in enumerate(allI):

        if opt.delete_sky or opt.delete_pvastrom:
            log(j + 1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if opt.delete_sky and os.path.exists(im.skyfn):
                log('  deleting:', im.skyfn)
                os.unlink(im.skyfn)
            if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
                log('  deleting:', im.pvwcsfn)
                os.unlink(im.pvwcsfn)

        if opt.check:
            log(j + 1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if not im.run_calibs(im, just_check=True):
                log('Calibs for', im.expnum, im.ccdname, im.calname,
                    'already done')
                continue

        if opt.command:
            s = '%i-%s' % (T.expnum[i], T.ccdname[i])
            prefix = 'python legacypipe/run-calib.py ' + opt.opt
            #('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
            #     (T.expnum[i], T.ccdname[i]))
        else:
            s = '%i' % T.index[i]
            prefix = ''

        if j < 10:
            print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname',
                  T.ccdname[i], 'filename', T.image_filename[i])

        if not opt.nper:
            f.write(prefix + s + '\n')
        else:
            batch.append(s)
            if len(batch) >= opt.nper:
                write_batch(f, batch, cmd)
                batch = []

        if opt.check:
            f.flush()

    if len(batch):
        write_batch(f, batch, cmd)

    f.close()
    log('Wrote', opt.out)
    return 0