Ejemplo n.º 1
0
def getbrickfiles(brickname=None):

    survey = LegacySurveyData()
    brickinfo = survey.get_brick_by_name(brickname)
    brickwcs = wcs_for_brick(brickinfo)
    ccdinfo = survey.ccds_touching_wcs(brickwcs)
    nccd = len(ccdinfo)

    calibdir = survey.get_calib_dir()
    imagedir = survey.survey_dir

    # Construct image file names and the calibration file names.
    expnum = ccdinfo.expnum
    ccdname = ccdinfo.ccdname

    psffiles = list()
    skyfiles = list()
    imagefiles = list()
    for ccd in ccdinfo:
        info = survey.get_image_object(ccd)
        for attr in ['imgfn', 'dqfn', 'wtfn']:
            fn = getattr(info, attr).replace(imagedir+'/', '')
            #if '160108_073601' in fn:
            #    pdb.set_trace()
            imagefiles.append(fn)
        psffiles.append(info.psffn.replace(calibdir, 'calib'))
        skyfiles.append(info.splineskyfn.replace(calibdir, 'calib'))
        
    #for ii in range(nccd):
        #exp = '{0:08d}'.format(expnum[ii])
        #rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits')
        #psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile))
        #skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile))
        #imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii]))))

    #print(np.array(imagefiles))
    #print(np.array(psffiles))
    #print(np.array(skyfiles))
    return imagefiles, psffiles, skyfiles
Ejemplo n.º 2
0
def main():
    """Main program.
    """
    import argparse

    parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).")
    parser.add_argument('--calibs', action='store_true',
                      help='Output CCDs that need to be calibrated.')

    parser.add_argument('--nper', type=int, default=None,
                      help='Batch N calibs per line')

    parser.add_argument('--forced', action='store_true',
                      help='Output forced-photometry commands')

    parser.add_argument('--lsb', action='store_true',
                      help='Output Low-Surface-Brightness commands')

    parser.add_argument('--touching', action='store_true',
                      help='Cut to only CCDs touching selected bricks')
    parser.add_argument('--near', action='store_true',
                      help='Quick cut to only CCDs near selected bricks')

    parser.add_argument('--check', action='store_true',
                      help='Check which calibrations actually need to run.')
    parser.add_argument('--check-coadd', action='store_true',
                      help='Check which caoadds actually need to run.')
    parser.add_argument('--out', help='Output filename for calibs, default %(default)s',
                      default='jobs')
    parser.add_argument('--command', action='store_true',
                      help='Write out full command-line to run calib')
    parser.add_argument('--opt', help='With --command, extra options to add')
    
    parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
    parser.add_argument('--mindec', type=float, help='Minimum Dec to run')

    parser.add_argument('--region', help='Region to select')

    parser.add_argument('--bricks', help='Set bricks.fits file to load')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')
    parser.add_argument('--ignore_cuts', action='store_true',default=False,help='no photometric or blacklist cuts')
    parser.add_argument('--save_to_fits', action='store_true',default=False,help='save cut brick,ccd to fits table')
    parser.add_argument('--name', action='store',default='dr3',help='save with this suffix, e.g. refers to ccds table')

    parser.add_argument('--delete-sky', action='store_true',
                      help='Delete any existing sky calibration files')
    parser.add_argument('--delete-pvastrom', action='store_true',
                      help='Delete any existing PV WCS calibration files')

    parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')

    parser.add_argument('--brickq', type=int, default=None,
                        help='Queue only bricks with the given "brickq" value [0 to 3]')

    parser.add_argument('--brickq-deps', action='store_true', default=False,
                        help='Queue bricks directly using qdo API, setting brickq dependencies')
    parser.add_argument('--queue', default='bricks',
                        help='With --brickq-deps, the QDO queue name to use')
    
    opt = parser.parse_args()

    survey = LegacySurveyData()
    if opt.bricks is not None:
        B = fits_table(opt.bricks)
        log('Read', len(B), 'from', opt.bricks)
    else:
        B = survey.get_bricks()

    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        log('Read', len(T), 'from', opt.ccds)
    else:
        T = survey.get_ccds()
        log(len(T), 'CCDs')
    T.index = np.arange(len(T))

    if opt.ignore_cuts == False:
        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
    print(len(T), 'CCDs remain')

    # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
    # plt.clf()
    # plt.hist(counts, counts.max()+1)
    # plt.savefig('bricks.png')
    # B.cut(I[counts >= 9])
    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # #plt.scatter(B.ra[I], B.dec[I], c=counts)
    # plt.savefig('bricks2.png')


    # DES Stripe82
    #rlo,rhi = 350.,360.
    # rlo,rhi = 300., 10.
    # dlo,dhi = -6., 4.
    # TINY bit
    #rlo,rhi = 350.,351.1
    #dlo,dhi = 0., 1.1

    # EDR+
    # 860 bricks
    # ~10,000 CCDs
    #rlo,rhi = 239,246
    #dlo,dhi =   5, 13

    # DR1
    #rlo,rhi = 0, 360
    # part 1
    #dlo,dhi = 25, 40
    # part 2
    #dlo,dhi = 20,25
    # part 3
    #dlo,dhi = 15,20
    # part 4
    #dlo,dhi = 10,15
    # part 5
    #dlo,dhi = 5,10
    # the rest
    #dlo,dhi = -11, 5
    #dlo,dhi = 15,25.5

    dlo,dhi = -25, 40
    rlo,rhi = 0, 360

    # Arjun says 3x3 coverage area is roughly
    # RA=240-252 DEC=6-12 (but not completely rectangular)

    # COSMOS
    #rlo,rhi = 148.9, 151.2
    #dlo,dhi = 0.9, 3.5

    # A nice well-behaved region (EDR2/3)
    # rlo,rhi = 243.6, 244.6
    # dlo,dhi = 8.1, 8.6

    # 56 bricks, ~725 CCDs
    #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
    # 240 bricks, ~3000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
    # 535 bricks, ~7000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))


    if opt.region in ['test1', 'test2', 'test3', 'test4']:
        nm = dict(test1='2446p115', # weird stuff around bright star
                  test2='1183p292', # faint sources around bright galaxy
                  test3='3503p005', # DES
                  test4='1163p277', # Pollux
                  )[opt.region]

        B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
        log('Cut to', len(B), 'bricks')
        log(B.ra, B.dec)
        dlo,dhi = -90,90
        rlo,rhi = 0, 360

    elif opt.region == 'edr':
        # EDR:
        # 535 bricks, ~7000 CCDs
        rlo,rhi = 240,245
        dlo,dhi =   5, 12

    elif opt.region == 'edrplus':
        rlo,rhi = 235,248
        dlo,dhi =   5, 15

    elif opt.region == 'edr-south':
        rlo,rhi = 240,245
        dlo,dhi =   5, 10

    elif opt.region == 'cosmos1':
        # 16 bricks in the core of the COSMOS field.
        rlo,rhi = 149.75, 150.75
        dlo,dhi = 1.6, 2.6

    elif opt.region == 'pristine':
        # Stream?
        rlo,rhi = 240,250
        dlo,dhi = 10,15

    elif opt.region == 'des':
        dlo, dhi = -6., 4.
        rlo, rhi = 317., 7.

        T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
        log('Cut to', len(T), 'CCDs with "CPDES82" in filename')

    elif opt.region == 'subdes':
        rlo,rhi = 320., 360.
        dlo,dhi = -1.25, 1.25

    elif opt.region == 'northwest':
        rlo,rhi = 240,360
        dlo,dhi = 20,40
    elif opt.region == 'north':
        rlo,rhi = 120,240
        dlo,dhi = 20,40
    elif opt.region == 'northeast':
        rlo,rhi = 0,120
        dlo,dhi = 20,40
    elif opt.region == 'southwest':
        rlo,rhi = 240,360
        dlo,dhi = -20,0
    elif opt.region == 'south':
        rlo,rhi = 120,240
        dlo,dhi = -20,0
    elif opt.region == 'southeast':
        rlo,rhi = 0,120
        dlo,dhi = -20,0
    elif opt.region == 'southsoutheast':
        rlo,rhi = 0,120
        dlo,dhi = -20,-10
    elif opt.region == 'midwest':
        rlo,rhi = 240,360
        dlo,dhi = 0,20
    elif opt.region == 'middle':
        rlo,rhi = 120,240
        dlo,dhi = 0,20
    elif opt.region == 'mideast':
        rlo,rhi = 0,120
        dlo,dhi = 0,20

    elif opt.region == 'grz':
        # Bricks with grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
        log('Cut to', len(B), 'bricks with grz coverage')

    elif opt.region == 'nogrz':
        # Bricks without grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
        log('Cut to', len(B), 'bricks withOUT grz coverage')
    elif opt.region == 'deep2':
        rlo,rhi = 250,260
        dlo,dhi = 30,35

    elif opt.region == 'deep2f3':
        rlo,rhi = 351.25, 353.75
        dlo,dhi = 0, 0.5

    elif opt.region == 'virgo':
        rlo,rhi = 185,190
        dlo,dhi =  10, 15

    elif opt.region == 'virgo2':
        rlo,rhi = 182,192
        dlo,dhi =   8, 18

    elif opt.region == 'lsb':
        rlo,rhi = 147.2, 147.8
        dlo,dhi = -0.4, 0.4

    elif opt.region == 'eboss-elg':
        # RA -45 to +45
        # Dec -5 to +7
        rlo,rhi = 315., 45.
        dlo,dhi = -5., 7.

    elif opt.region == 'eboss-ngc':
        # NGC ELGs
        # RA 115 to 175
        # Dec 15 to  30
        rlo,rhi = 115., 175.
        dlo,dhi =  15.,  30.

    elif opt.region == 'mzls':
        dlo,dhi = 30., 90.
    elif opt.region == 'dr4-bootes':
        # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched 
        #dlo,dhi = 34., 35.
        #rlo,rhi = 209.5, 210.5
        dlo,dhi = 33., 36.
        rlo,rhi = 216.5, 219.5

        
    if opt.mindec is not None:
        dlo = opt.mindec
    if opt.maxdec is not None:
        dhi = opt.maxdec

    if rlo < rhi:
        B.cut((B.ra >= rlo) * (B.ra <= rhi) *
              (B.dec >= dlo) * (B.dec <= dhi))
    else: # RA wrap
        B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) *
              (B.dec >= dlo) * (B.dec <= dhi))
    log(len(B), 'bricks in range')
    for name in B.get('brickname'):
        print(name)
    B.writeto('bricks-cut.fits')

    I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize)
    keep = np.zeros(len(B), bool)
    for i in I:
        keep[i] = True
    B.cut(keep)
    log('Cut to', len(B), 'bricks near CCDs')

    plt.clf()
    plt.plot(B.ra, B.dec, 'b.')
    plt.title('DR3 bricks')
    plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
    plt.savefig('bricks.png')

    if opt.brickq is not None:
        B.cut(B.brickq == opt.brickq)
        log('Cut to', len(B), 'with brickq =', opt.brickq)
    
    if opt.touching:
        keep = np.zeros(len(T), bool)
        for j in J:
            keep[j] = True
        T.cut(keep)
        log('Cut to', len(T), 'CCDs near bricks')

    # Aside -- how many near DR1=1 CCDs?
    if False:
        T2 = D.get_ccds()
        log(len(T2), 'CCDs')
        T2.cut(T2.dr1 == 1)
        log(len(T2), 'CCDs marked DR1=1')
        log(len(B), 'bricks in range')
        I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize)
        keep = np.zeros(len(B), bool)
        for i in I:
            keep[i] = True
        B2 = B[keep]
        log('Total of', len(B2), 'bricks near CCDs with DR1=1')
        for band in 'grz':
            Tb = T2[T2.filter == band]
            log(len(Tb), 'in filter', band)
            I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, survey.bricksize)
            good = np.zeros(len(B2), np.uint8)
            for i in I:
                good[i] = 1
            B2.set('has_' + band, good)

        B2.writeto('survey-bricks-in-dr1.fits')
        sys.exit(0)

    # sort by dec decreasing
    #B.cut(np.argsort(-B.dec))
    # RA increasing
    B.cut(np.argsort(B.ra))

    for b in B:
        if opt.check:
            fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue
        if opt.check_coadd:
            fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue

        print(b.brickname)

    if opt.save_to_fits:
        assert(opt.touching)
        # Write cut tables to file
        for tab,typ in zip([B,T],['bricks','ccds']):
            fn='%s-%s-cut.fits' % (typ,opt.name)
            if os.path.exists(fn):
                os.remove(fn)
            tab.writeto(fn)
            print('Wrote %s' % fn)
        # Write text files listing ccd and filename names
        nm1,nm2= 'ccds-%s.txt'% opt.name,'filenames-%s.txt' % opt.name
        if os.path.exists(nm1):
            os.remove(nm1)
        if os.path.exists(nm2):
            os.remove(nm2)
        f1,f2=open(nm1,'w'),open(nm2,'w')
        fns= list(set(T.get('image_filename')))
        for fn in fns:
            f2.write('%s\n' % fn.strip())
        for ti in T:
            f1.write('%s\n' % ti.get('image_filename').strip())
        f1.close()
        f2.close()
        print('Wrote *-names.txt')
    

    if opt.brickq_deps:
        import qdo
        from legacypipe.survey import on_bricks_dependencies

        #... find Queue...
        q = qdo.connect(opt.queue, create_ok=True)
        print('Connected to QDO queue', opt.queue, q)
        brick_to_task = dict()

        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
        print(len(T), 'CCDs remaining')

        T.wra = T.ra + (T.ra > 180) * -360
        wra = rlo - 360
        plt.clf()
        plt.plot(T.wra, T.dec, 'b.')
        ax = [wra, rhi, dlo, dhi]
        plt.axis(ax)
        plt.title('CCDs')
        plt.savefig('q-ccds.png')

        B.wra = B.ra + (B.ra > 180) * -360

        # this slight overestimate (for DECam images) is fine
        radius = 0.3
        Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius,
                            indexlist=True)
        ikeep = []
        for ib,(b,Iccd) in enumerate(zip(B, Iccds)):
            if Iccd is None or len(Iccd) == 0:
                print('No matched CCDs to brick', b.brickname)
                continue
            wcs = wcs_for_brick(b)
            cI = ccds_touching_wcs(wcs, T[np.array(Iccd)])
            print(len(cI), 'CCDs touching brick', b.brickname)
            if len(cI) == 0:
                continue
            ikeep.append(ib)
        B.cut(np.array(ikeep))
        print('Cut to', len(B), 'bricks touched by CCDs')
        
        for brickq in range(4):
            I = np.flatnonzero(B.brickq == brickq)
            print(len(I), 'bricks with brickq =', brickq)

            J = np.flatnonzero(B.brickq < brickq)
            preB = B[J]
            reqs = []
            if brickq > 0:
                for b in B[I]:
                    # find brick dependencies
                    brickdeps = on_bricks_dependencies(b, survey, bricks=preB)
                    # convert to task ids
                    taskdeps = [brick_to_task.get(b.brickname,None) for b in brickdeps]
                    # If we dropped a dependency brick from a previous brickq because
                    # of no overlapping CCDs, it won't appear in the brick_to_task map.
                    taskdeps = [t for t in taskdeps if t is not None]
                    reqs.append(taskdeps)

            plt.clf()
            plt.plot(B.wra, B.dec, '.', color='0.5')
            plt.plot(B.wra[I], B.dec[I], 'b.')
            plt.axis(ax)
            plt.title('Bricks: brickq=%i' % brickq)
            plt.savefig('q-bricks-%i.png' % brickq)
            
            # submit to qdo queue
            print('Queuing', len(B[I]), 'bricks')
            if brickq == 0:
                reqs = None
            else:
                assert(len(I) == len(reqs))
            taskids = q.add_multiple(B.brickname[I], requires=reqs)
            assert(len(taskids) == len(I))
            print('Queued', len(taskids), 'bricks')
            brick_to_task.update(dict(zip(B.brickname[I], taskids)))
        
    if not (opt.calibs or opt.forced or opt.lsb):
        sys.exit(0)

    bands = 'grz'
    log('Filters:', np.unique(T.filter))
    T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
    log('Cut to', len(T), 'CCDs in filters', bands)

    if opt.touching:
        allI = set()
        for b in B:
            wcs = wcs_for_brick(b)
            I = ccds_touching_wcs(wcs, T)
            log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
            if len(I) == 0:
                continue
            allI.update(I)
        allI = list(allI)
        allI.sort()
    elif opt.near:
        # Roughly brick radius + DECam image radius
        radius = 0.35
        allI,nil,nil = match_radec(T.ra, T.dec, B.ra, B.dec, radius, nearest=True)
    else:
        allI = np.arange(len(T))

    if opt.write_ccds:
        T[allI].writeto(opt.write_ccds)
        log('Wrote', opt.write_ccds)

    ## Be careful here -- T has been cut; we want to write out T.index.
    ## 'allI' contains indices into T.

    if opt.forced:
        log('Writing forced-photometry commands to', opt.out)
        f = open(opt.out,'w')
        log('Total of', len(allI), 'CCDs')
        for j,i in enumerate(allI):
            expstr = '%08i' % T.expnum[i]
            outfn = os.path.join('forced', expstr[:5], expstr,
                                 'decam-%s-%s-forced.fits' %
                                 (expstr, T.ccdname[i]))
            imgfn = os.path.join(survey.survey_dir, 'images',
                                 T.image_filename[i].strip())
            if (not os.path.exists(imgfn) and
                imgfn.endswith('.fz') and
                os.path.exists(imgfn[:-3])):
                imgfn = imgfn[:-3]

            #f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' %
            #        (imgfn, T.image_hdu[i], outfn))

            f.write('python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n' %
                    (T.expnum[i], T.ccdname[i], outfn))
            
        f.close()
        log('Wrote', opt.out)
        sys.exit(0)

    if opt.lsb:
        log('Writing LSB commands to', opt.out)
        f = open(opt.out,'w')
        log('Total of', len(allI), 'CCDs')
        for j,i in enumerate(allI):
            exp = T.expnum[i]
            ext = T.ccdname[i].strip()
            outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
            f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext))
        f.close()
        log('Wrote', opt.out)
        sys.exit(0)


    log('Writing calibs to', opt.out)
    f = open(opt.out,'w')
    log('Total of', len(allI), 'CCDs')

    batch = []

    def write_batch(f, batch, cmd):
        if cmd is None:
            cmd = ''
        f.write(cmd + ' '.join(batch) + '\n')

    cmd = None
    if opt.command:
        cmd = 'python legacypipe/run-calib.py '
        if opt.opt is not None:
            cmd += opt.opt + ' '
        
    for j,i in enumerate(allI):

        if opt.delete_sky or opt.delete_pvastrom:
            log(j+1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if opt.delete_sky and os.path.exists(im.skyfn):
                log('  deleting:', im.skyfn)
                os.unlink(im.skyfn)
            if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
                log('  deleting:', im.pvwcsfn)
                os.unlink(im.pvwcsfn)

        if opt.check:
            log(j+1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if not im.run_calibs(im, just_check=True):
                log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done')
                continue

        if opt.command:
            s = '%i-%s' % (T.expnum[i], T.ccdname[i])
            prefix = 'python legacypipe/run-calib.py ' + opt.opt
            #('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
            #     (T.expnum[i], T.ccdname[i]))
        else:
            s = '%i' % T.index[i]
            prefix = ''
            
        if j < 10:
            print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i],
                  'filename', T.image_filename[i])
            
        if not opt.nper:
            f.write(prefix + s + '\n')
        else:
            batch.append(s)
            if len(batch) >= opt.nper:
                write_batch(f, batch, cmd)
                batch = []

        if opt.check:
            f.flush()

    if len(batch):
        write_batch(f, batch, cmd)

    f.close()
    log('Wrote', opt.out)
    return 0
Ejemplo n.º 3
0
def main(args=None):
    """Main routine which parses the optional inputs."""
    # Command line options
    parser= get_parser()    
    args = parser.parse_args(args=args)
    # Setup loggers
    if args.verbose:
        lvl = logging.DEBUG
    else:
        lvl = logging.INFO
    logging.basicConfig(level=lvl, stream=sys.stdout) #,format='%(message)s')
    log = logging.getLogger('decals_sim')
    # Sort through args 
    log.info('decals_sim.py args={}'.format(args))
    max_nobj=500
    max_nchunk=1000
    if args.ith_chunk is not None: assert(args.ith_chunk <= max_nchunk-1)
    assert(args.nchunk <= max_nchunk)
    assert(args.nobj <= max_nobj)
    if args.ith_chunk is not None: 
        assert(args.nchunk == 1) #if choose a chunk, only doing 1 chunk
    if args.nobj is None:
        parser.print_help()
        sys.exit(1)
 
    brickname = args.brick
    objtype = args.objtype.upper()
    lobjtype = objtype.lower()

    for obj in ('LRG', 'LSB', 'QSO'):
        if objtype == obj:
            log.warning('{} objtype not yet supported!'.format(objtype))
            return 0

    # Deal with the paths.
    if 'DECALS_SIM_DIR' in os.environ:
        decals_sim_dir = os.getenv('DECALS_SIM_DIR')
    else:
        decals_sim_dir = '.'
        
    nobj = args.nobj
    nchunk = args.nchunk
    rand = np.random.RandomState(args.seed) # determines seed for all chunks
    seeds = rand.random_integers(0,2**18, max_nchunk)

    log.info('Object type = {}'.format(objtype))
    log.info('Number of objects = {}'.format(nobj))
    log.info('Number of chunks = {}'.format(nchunk))

    # Optionally zoom into a portion of the brick
    survey = LegacySurveyData()
    brickinfo = survey.get_brick_by_name(brickname)
    brickwcs = wcs_for_brick(brickinfo)
    W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale()

    log.info('Brick = {}'.format(brickname))
    if args.zoom is not None: # See also runbrick.stage_tims()
        (x0, x1, y0, y1) = args.zoom
        W = x1 - x0
        H = y1 - y0
        brickwcs = brickwcs.get_subimage(x0, y0, W, H)
        log.info('Zoom (pixel boundaries) = {}'.format(args.zoom))
    targetrd = np.array([brickwcs.pixelxy2radec(x, y) for x, y in
                         [(1,1), (W,1), (W,H), (1,H), (1,1)]])
 
    radec_center = brickwcs.radec_center()
    log.info('RA, Dec center = {}'.format(radec_center))
    log.info('Brick = {}'.format(brickname))
    
    if args.ith_chunk is not None: 
        chunk_list= [args.ith_chunk]
    else: 
        chunk_list= range(nchunk)

    # Store args in dict for easy func passing
    kwargs=dict(seeds=seeds,\
                brickname=brickname, \
                decals_sim_dir= decals_sim_dir,\
                brickwcs= brickwcs, \
                objtype=objtype,\
                lobjtype=lobjtype,\
                nobj=nobj,\
                nchunk=nchunk,\
                args=args) 
    
    # Create simulated catalogues and run Tractor
    create_metadata(kwargs=kwargs)
    # do chunks
    for ith_chunk in chunk_list:
        log.info('Working on chunk {:02d}/{:02d}'.format(ith_chunk,kwargs['nchunk']-1))
        # Random ra,dec and source properties
        create_ith_simcat(ith_chunk, d=kwargs)
        # Run tractor
        do_one_chunk(d=kwargs)
        # Clean up output
        do_ith_cleanup(ith_chunk=ith_chunk, d=kwargs)
    log.info('All done!')
Ejemplo n.º 4
0
def main():
    import argparse

    parser = argparse.ArgumentParser(
        description='This script creates small self-contained data sets that '
        'are useful for test cases of the pipeline codes.')

    parser.add_argument('ccds', help='CCDs table describing region to grab')
    parser.add_argument('outdir', help='Output directory name')
    parser.add_argument('brick', help='Brick containing these images')

    parser.add_argument('--cache-dir',
                        type=str,
                        default=None,
                        help='Directory to search for cached files')
    parser.add_argument(
        '--wise',
        help=
        'For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image'
    )
    parser.add_argument(
        '--wise-wcs-hdu',
        help=
        'For WISE outputs, the HDU to read the WCS from in the file given by --wise.',
        type=int,
        default=0)
    parser.add_argument('--fpack', action='store_true', default=False)
    parser.add_argument('--gzip', action='store_true', default=False)
    parser.add_argument(
        '--pad',
        action='store_true',
        default=False,
        help='Keep original image size, but zero out pixels outside ROI')

    args = parser.parse_args()

    C = fits_table(args.ccds)
    print(len(C), 'CCDs in', args.ccds)
    C.camera = np.array([c.strip() for c in C.camera])

    survey = LegacySurveyData(cache_dir=args.cache_dir)

    if ',' in args.brick:
        ra, dec = args.brick.split(',')
        ra = float(ra)
        dec = float(dec)
        fakebricks = fits_table()
        fakebricks.brickname = np.array([(
            'custom-%06i%s%05i' %
            (int(1000 * ra), 'm' if dec < 0 else 'p', int(1000 * np.abs(dec))))
                                         ])
        fakebricks.ra = np.array([ra])
        fakebricks.dec = np.array([dec])
        bricks = fakebricks
        outbricks = bricks
    else:
        bricks = survey.get_bricks_readonly()
        outbricks = bricks[np.array(
            [n == args.brick for n in bricks.brickname])]
        assert (len(outbricks) == 1)

    outsurvey = LegacySurveyData(survey_dir=args.outdir)
    trymakedirs(args.outdir)
    outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz'))

    targetwcs = wcs_for_brick(outbricks[0])
    H, W = targetwcs.shape

    tycho2fn = survey.find_file('tycho2')
    kd = tree_open(tycho2fn, 'stars')
    radius = 1.
    rc, dc = targetwcs.radec_center()
    I = tree_search_radec(kd, rc, dc, radius)
    print(len(I), 'Tycho-2 stars within', radius,
          'deg of RA,Dec (%.3f, %.3f)' % (rc, dc))
    # Read only the rows within range.
    tycho = fits_table(tycho2fn, rows=I)
    del kd
    print('Read', len(tycho), 'Tycho-2 stars')
    ok, tx, ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec)
    #margin = 100
    #tycho.cut(ok * (tx > -margin) * (tx < W+margin) *
    #          (ty > -margin) * (ty < H+margin))
    print('Cut to', len(tycho), 'Tycho-2 stars within brick')
    del ok, tx, ty
    #tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz'))
    f, tfn = tempfile.mkstemp(suffix='.fits')
    os.close(f)
    tycho.writeto(tfn)
    outfn = os.path.join(args.outdir, 'tycho2.kd.fits')
    cmd = 'startree -i %s -o %s -P -k -n stars -T' % (tfn, outfn)
    print(cmd)
    rtn = os.system(cmd)
    assert (rtn == 0)
    os.unlink(tfn)

    outccds = C.copy()
    cols = outccds.get_columns()
    for c in [
            'ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1', 'brick_x0', 'brick_x1',
            'brick_y0', 'brick_y1', 'skyver', 'wcsver', 'psfver', 'skyplver',
            'wcsplver', 'psfplver'
    ]:
        if c in cols:
            outccds.delete_column(c)
    outccds.image_hdu[:] = 1

    # Convert to list to avoid truncating filenames
    outccds.image_filename = [fn for fn in outccds.image_filename]

    for iccd, ccd in enumerate(C):

        decam = (ccd.camera.strip() == 'decam')
        bok = (ccd.camera.strip() == '90prime')

        im = survey.get_image_object(ccd)
        print('Got', im)
        if survey.cache_dir is not None:
            im.check_for_cached_files(survey)
        slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1))

        psfkwargs = dict(pixPsf=True,
                         gaussPsf=False,
                         hybridPsf=False,
                         normalizePsf=False)

        tim = im.get_tractor_image(slc,
                                   pixPsf=True,
                                   splinesky=True,
                                   subsky=False,
                                   nanomaggies=False,
                                   no_remap_invvar=True,
                                   old_calibs_ok=True)
        print('Tim:', tim.shape)

        psfrow = psfhdr = None

        if args.pad:
            psf = im.read_psf_model(0, 0, w=im.width, h=im.height, **psfkwargs)
            psfex = psf.psfex
        else:
            psf = tim.getPsf()
            psfex = psf.psfex

            # Did the PSF model come from a merged file?
            mpsf = im.read_merged_psfex_model(old_calibs_ok=True)
            if mpsf is not None:
                T = fits_table(im.merged_psffn)
                I, = np.nonzero(
                    (T.expnum == im.expnum) *
                    np.array([c.strip() == im.ccdname for c in T.ccdname]))
                psfrow = T[I]
                x0 = ccd.ccd_x0
                y0 = ccd.ccd_y0
                psfrow.polzero1[0] += x0
                psfrow.polzero2[0] += y0
                psfhdr = fitsio.read_header(im.merged_psffn)

        psfex.fwhm = tim.psf_fwhm

        if psfrow is not None:
            print('PSF row:', psfrow)
        else:
            print('PSF:', psf)
            print('PsfEx:', psfex)

        skyrow = skyhdr = None

        if args.pad:
            primhdr = fitsio.read_header(im.imgfn)
            imghdr = fitsio.read_header(im.imgfn, hdu=im.hdu)
            sky = im.read_sky_model(splinesky=True,
                                    primhdr=primhdr,
                                    imghdr=imghdr)
        else:
            sky = tim.getSky()

            # Did the sky model come from a merged file?
            msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True)
            if msky is not None:
                T = fits_table(im.merged_splineskyfn)
                I, = np.nonzero(
                    (T.expnum == im.expnum) *
                    np.array([c.strip() == im.ccdname for c in T.ccdname]))
                skyrow = T[I]
                skyrow.x0[0] = ccd.ccd_x0
                skyrow.y0[0] = ccd.ccd_y0
                skyhdr = fitsio.read_header(im.merged_splineskyfn)

        if skyrow is not None:
            print('Sky row:', skyrow)
        else:
            print('Sky:', sky)

        outim = outsurvey.get_image_object(ccd)
        print('Output image:', outim)

        print('Image filename:', outim.imgfn)
        trymakedirs(outim.imgfn, dir=True)

        imgdata = tim.getImage()
        ivdata = tim.getInvvar()

        # Since we remap DQ codes (always with Mosaic and Bok, sometimes with DECam),
        # re-read from the FITS file rather than using tim.dq.
        print('Reading data quality from', im.dqfn, 'hdu', im.hdu)
        dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice)

        print('Tim shape:', tim.shape, 'Slice', tim.slice)
        print('image shape:', imgdata.shape, 'iv', ivdata.shape, 'DQ',
              dqdata.shape)

        from collections import Counter
        dqvals = Counter(dqdata.ravel())
        print('DQ pixel counts:')
        for k, n in dqvals.most_common():
            print('  0x%x' % k, ':', n)

        if args.pad:
            # Create zero image of full size, copy in data.
            fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype)
            fullsize[slc] = imgdata
            imgdata = fullsize

            fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype)
            fullsize[slc] = dqdata
            dqdata = fullsize

            fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype)
            fullsize[slc] = ivdata
            ivdata = fullsize

        else:
            # Adjust the header WCS by x0,y0
            crpix1 = tim.hdr['CRPIX1']
            crpix2 = tim.hdr['CRPIX2']
            tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0
            tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0

        # Add image extension to filename
        # fitsio doesn't compress .fz by default, so drop .fz suffix

        outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname)
        if not args.fpack:
            outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits')
        if args.gzip:
            outim.imgfn = outim.imgfn.replace('.fits', '.fits.gz')

        outim.wtfn = outim.wtfn.replace('.fits', '-%s.fits' % im.ccdname)
        if not args.fpack:
            outim.wtfn = outim.wtfn.replace('.fits.fz', '.fits')
        if args.gzip:
            outim.wtfn = outim.wtfn.replace('.fits', '.fits.gz')

        if outim.dqfn is not None:
            outim.dqfn = outim.dqfn.replace('.fits', '-%s.fits' % im.ccdname)
            if not args.fpack:
                outim.dqfn = outim.dqfn.replace('.fits.fz', '.fits')
            if args.gzip:
                outim.dqfn = outim.dqfn.replace('.fits', '.fits.gz')

        if bok:
            outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname)

        ccdfn = outim.imgfn
        ccdfn = ccdfn.replace(outsurvey.get_image_dir(), '')
        if ccdfn.startswith('/'):
            ccdfn = ccdfn[1:]
        outccds.image_filename[iccd] = ccdfn

        print('Changed output filenames to:')
        print(outim.imgfn)
        print(outim.dqfn)

        ofn = outim.imgfn
        if args.fpack:
            f, ofn = tempfile.mkstemp(suffix='.fits')
            os.close(f)
        fits = fitsio.FITS(ofn, 'rw', clobber=True)
        fits.write(None, header=tim.primhdr)
        fits.write(imgdata, header=tim.hdr, extname=ccd.ccdname)
        fits.close()

        if args.fpack:
            cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn)
            print('Running:', cmd)
            rtn = os.system(cmd)
            assert (rtn == 0)

        h, w = tim.shape
        if not args.pad:
            outccds.width[iccd] = w
            outccds.height[iccd] = h
            outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0
            outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0

        wcs = Tan(*[
            float(x) for x in [
                ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2, ccd.cd1_1,
                ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height
            ]
        ])

        if args.pad:
            subwcs = wcs
        else:
            subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h)
            outccds.ra[iccd], outccds.dec[iccd] = subwcs.radec_center()

        print('Weight filename:', outim.wtfn)
        wfn = outim.wtfn
        trymakedirs(wfn, dir=True)

        ofn = wfn
        if args.fpack:
            f, ofn = tempfile.mkstemp(suffix='.fits')
            os.close(f)

        fits = fitsio.FITS(ofn, 'rw', clobber=True)
        fits.write(None, header=tim.primhdr)
        fits.write(ivdata, header=tim.hdr, extname=ccd.ccdname)
        fits.close()

        if args.fpack:
            cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn)
            print('Running:', cmd)
            rtn = os.system(cmd)
            assert (rtn == 0)

        if outim.dqfn is not None:
            print('DQ filename', outim.dqfn)
            trymakedirs(outim.dqfn, dir=True)

            ofn = outim.dqfn
            if args.fpack:
                f, ofn = tempfile.mkstemp(suffix='.fits')
                os.close(f)

            fits = fitsio.FITS(ofn, 'rw', clobber=True)
            fits.write(None, header=tim.primhdr)
            fits.write(dqdata, header=tim.hdr, extname=ccd.ccdname)
            fits.close()

            if args.fpack:
                cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn,
                                                             ofn)
                print('Running:', cmd)
                rtn = os.system(cmd)
                assert (rtn == 0)

        psfout = outim.psffn
        if psfrow:
            psfout = outim.merged_psffn
        print('PSF output filename:', psfout)
        trymakedirs(psfout, dir=True)
        if psfrow:
            psfrow.writeto(psfout, primhdr=psfhdr)
        else:
            print('Writing PsfEx:', psfout)
            psfex.writeto(psfout)
            # update header
            F = fitsio.FITS(psfout, 'rw')
            F[0].write_keys([
                dict(name='EXPNUM', value=ccd.expnum),
                dict(name='PLVER', value=psf.plver),
                dict(name='PROCDATE', value=psf.procdate),
            ])
            F.close()

        skyout = outim.splineskyfn
        if skyrow:
            skyout = outim.merged_splineskyfn

        print('Sky output filename:', skyout)
        trymakedirs(skyout, dir=True)
        if skyrow is not None:
            skyrow.writeto(skyout, primhdr=skyhdr)
        else:
            primhdr = fitsio.FITSHDR()
            primhdr['PLVER'] = sky.plver
            primhdr['PROCDATE'] = sky.procdate
            primhdr['EXPNUM'] = ccd.expnum
            primhdr['IMGDSUM'] = sky.datasum
            sky.write_fits(skyout, primhdr=primhdr)

        # HACK -- check result immediately.
        outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))
        outsurvey.ccds = None
        outC = outsurvey.get_ccds_readonly()
        occd = outC[iccd]
        outim = outsurvey.get_image_object(occd)
        print('Got output image:', outim)
        otim = outim.get_tractor_image(pixPsf=True,
                                       splinesky=True,
                                       hybridPsf=True,
                                       old_calibs_ok=True)
        print('Got output tim:', otim)

    outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))

    # WISE
    if args.wise is not None:
        from wise.forcedphot import unwise_tiles_touching_wcs
        from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs,
                                 get_unwise_tractor_image, get_unwise_tile_dir)
        # Read WCS...
        print('Reading TAN wcs header from', args.wise, 'HDU',
              args.wise_wcs_hdu)
        targetwcs = Tan(args.wise, args.wise_wcs_hdu)
        tiles = unwise_tiles_touching_wcs(targetwcs)
        print('Cut to', len(tiles), 'unWISE tiles')
        H, W = targetwcs.shape
        r, d = targetwcs.pixelxy2radec(np.array([1, W, W / 2, W / 2]),
                                       np.array([H / 2, H / 2, 1, H]))
        roiradec = [r[0], r[1], d[2], d[3]]

        unwise_dir = os.environ['UNWISE_COADDS_DIR']
        wise_out = os.path.join(args.outdir, 'images', 'unwise')
        print('Will write WISE outputs to', wise_out)

        unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR']
        wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr')
        print('Will write WISE time-resolved outputs to', wise_tr_out)
        trymakedirs(wise_tr_out)

        W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_atlas.fits'))
        print('Read', len(W), 'time-resolved WISE coadd tiles')
        W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id]))
        print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth')

        # Write the time-resolved index subset.
        W.writeto(os.path.join(wise_tr_out, 'time_resolved_atlas.fits'))

        # this ought to be enough for anyone =)
        _, Nepochs = W.epoch_bitmask.shape
        print('N epochs in time-resolved atlas:', Nepochs)

        wisedata = []

        # full depth
        for band in [1, 2, 3, 4]:
            wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band))

        # time-resolved
        for band in [1, 2]:
            # W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2)
            bitmask = (1 << (band - 1))
            for e in range(Nepochs):
                # Which tiles have images for this epoch?
                I = np.flatnonzero(W.epoch_bitmask[:, e] & bitmask)
                if len(I) == 0:
                    continue
                print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I])
                edir = os.path.join(unwise_tr_dir, 'e%03i' % e)
                eoutdir = os.path.join(wise_tr_out, 'e%03i' % e)
                wisedata.append((edir, eoutdir, tiles.coadd_id[I], band))

        wrote_masks = set()

        for indir, outdir, tiles, band in wisedata:
            for tile in tiles:
                wanyband = 'w'
                tim = get_unwise_tractor_image(indir,
                                               tile,
                                               band,
                                               bandname=wanyband,
                                               roiradecbox=roiradec)
                print('Got unWISE tim', tim)
                print(tim.shape)

                thisdir = get_unwise_tile_dir(outdir, tile)
                print('Directory for this WISE tile:', thisdir)
                base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band))
                print('Base filename:', base)

                masked = True
                mu = 'm' if masked else 'u'

                imfn = base + 'img-%s.fits' % mu
                ivfn = base + 'invvar-%s.fits.gz' % mu
                nifn = base + 'n-%s.fits.gz' % mu
                nufn = base + 'n-u.fits.gz'

                #print('WISE image header:', tim.hdr)

                # Adjust the header WCS by x0,y0
                wcs = tim.wcs.wcs
                tim.hdr['CRPIX1'] = wcs.crpix[0]
                tim.hdr['CRPIX2'] = wcs.crpix[1]

                H, W = tim.shape
                tim.hdr['IMAGEW'] = W
                tim.hdr['IMAGEH'] = H

                print('WCS:', wcs)
                print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2'])

                trymakedirs(imfn, dir=True)
                fitsio.write(imfn,
                             tim.getImage(),
                             header=tim.hdr,
                             clobber=True)
                print('Wrote', imfn)
                fitsio.write(ivfn,
                             tim.getInvvar(),
                             header=tim.hdr,
                             clobber=True)
                print('Wrote', ivfn)
                fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True)
                print('Wrote', nifn)
                fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True)
                print('Wrote', nufn)

                if not (indir, tile) in wrote_masks:
                    print('Looking for mask file for', indir, tile)
                    # record that we tried this dir/tile combo
                    wrote_masks.add((indir, tile))
                    for idir in indir.split(':'):
                        tdir = get_unwise_tile_dir(idir, tile)
                        maskfn = 'unwise-%s-msk.fits.gz' % tile
                        fn = os.path.join(tdir, maskfn)
                        print('Mask file:', fn)
                        if os.path.exists(fn):
                            print('Reading', fn)
                            (x0, x1, y0, y1) = tim.roi
                            roislice = (slice(y0, y1), slice(x0, x1))
                            F = fitsio.FITS(fn)[0]
                            hdr = F.read_header()
                            M = F[roislice]
                            outfn = os.path.join(thisdir, maskfn)
                            fitsio.write(outfn,
                                         M,
                                         header=tim.hdr,
                                         clobber=True)
                            print('Wrote', outfn)
                            break

    outC = outsurvey.get_ccds_readonly()
    for iccd, ccd in enumerate(outC):
        outim = outsurvey.get_image_object(ccd)
        print('Got output image:', outim)
        otim = outim.get_tractor_image(pixPsf=True,
                                       splinesky=True,
                                       hybridPsf=True,
                                       old_calibs_ok=True)
        print('Got output tim:', otim)
Ejemplo n.º 5
0
def main():
    import optparse

    parser = optparse.OptionParser()
    parser.add_option('--zoom', '-z', type=int, action='append', default=[],
                      help='Add zoom level; default 13')
    parser.add_option('--threads', type=int, default=1, help='Number of threads')
    parser.add_option('--y0', type=int, default=0, help='Start row')
    parser.add_option('--y1', type=int, default=None, help='End row (non-inclusive)')

    parser.add_option('--x0', type=int, default=None)
    parser.add_option('--x1', type=int, default=None)

    parser.add_option('-x', type=int)
    parser.add_option('-y', type=int)

    parser.add_option('--mindec', type=float, default=None, help='Minimum Dec to run')
    parser.add_option('--maxdec', type=float, default=None, help='Maximum Dec to run')

    parser.add_option('--minra', type=float, default=None,   help='Minimum RA to run')
    parser.add_option('--maxra', type=float, default=None, help='Maximum RA to run')

    parser.add_option('--near', action='store_true', help='Only run tiles near bricks')

    parser.add_option('--near-ccds', action='store_true', help='Only run tiles near CCDs')

    parser.add_option('--queue', action='store_true', default=False,
                      help='Print qdo commands')

    parser.add_option('--all', action='store_true', help='Render all tiles')

    parser.add_option('--ignore', action='store_true', help='Ignore cached tile files',
                      default=False)

    parser.add_option('--top', action='store_true', help='Top levels of the pyramid')

    parser.add_option('--split', action='store_true', help='For split layers (DR6+DR7), only compute one y strip per zoom level')

    parser.add_option('--bricks-exist', action='store_true', help='Create table of bricks that exist')

    parser.add_option('--kind', default='image')
    parser.add_option('--scale', action='store_true', help='Scale images?')
    parser.add_option('--bricks', action='store_true', help='Compute scaled brick tables?')
    parser.add_option('--coadd', action='store_true', help='Create SDSS coadd images?')
    parser.add_option('--grass', action='store_true', help='progress plot')

    parser.add_option('--bands', default=None)

    parser.add_option('-v', '--verbose', dest='verbose', action='count',
                      default=0, help='Make more verbose')


    opt,args = parser.parse_args()

    if opt.verbose == 0:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG
    logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)


    mp = multiproc(opt.threads)

    if opt.kind in ['sdss', 'sdss2']:
        if opt.maxdec is None:
            opt.maxdec = 90
        if opt.mindec is None:
            opt.mindec = -25
    elif opt.kind in ['halpha', 'unwise-neo1', 'unwise-neo2', 'unwise-neo3', 'unwise-neo4', 'unwise-cat-model',
                      'galex', 'wssa', 'vlass', 'hsc']:
        if opt.maxdec is None:
            opt.maxdec = 90.
        if opt.mindec is None:
            opt.mindec = -90.
        if opt.maxra is None:
            opt.maxra = 360.
        if opt.minra is None:
            opt.minra = 0.

        if opt.kind == 'galex' and opt.bands is None:
            opt.bands = 'nf'
        if 'unwise' in opt.kind and opt.bands is None:
            opt.bands = '12'
        if 'vlass' in opt.kind:
            opt.bands = [1]

    elif opt.kind == 'm33':
        if opt.mindec is None:
            opt.mindec = 30.40
        if opt.maxdec is None:
            opt.maxdec = 30.90
        if opt.minra is None:
            opt.minra = 23.29
        if opt.maxra is None:
            opt.maxra = 23.73

    elif opt.kind in ['des-dr1']:
        if opt.maxdec is None:
            opt.maxdec = 6
        if opt.mindec is None:
            opt.mindec = -68
        if opt.maxra is None:
            opt.maxra = 360
        if opt.minra is None:
            opt.minra = 0

    elif opt.kind in ['mzls+bass-dr4', 'mzls+bass-dr4-model', 'mzls+bass-dr4-resid']:
        if opt.maxdec is None:
            opt.maxdec = 90
        if opt.mindec is None:
            opt.mindec = 30
        if opt.maxra is None:
            opt.maxra = 54
        if opt.minra is None:
            opt.minra = 301
    elif opt.kind in ['mzls+bass-dr6', 'mzls+bass-dr6-model', 'mzls+bass-dr6-resid']:
        if opt.maxdec is None:
            opt.maxdec = 90
        if opt.mindec is None:
            opt.mindec = -20
        if opt.maxra is None:
            opt.maxra = 360
        if opt.minra is None:
            opt.minra = 0
    elif opt.kind in ['decaps2', 'decaps2-model', 'decaps2-resid']:
        if opt.maxdec is None:
            opt.maxdec = -20
        if opt.mindec is None:
            opt.mindec = -70
        if opt.maxra is None:
            opt.maxra = 280
        if opt.minra is None:
            opt.minra = 90
    else:
        if opt.maxdec is None:
            opt.maxdec = 40
        if opt.mindec is None:
            opt.mindec = -25
        if opt.maxra is None:
            opt.maxra = 360
        if opt.minra is None:
            opt.minra = 0

    if opt.bands is None:
        opt.bands = 'grz'

    if opt.top:
        top_levels(mp, opt)
        sys.exit(0)

    if opt.bricks:
        from map.views import get_layer
        layer = get_layer(opt.kind)
        for scale in range(1,8):
            B = layer.get_bricks_for_scale(scale)
        sys.exit(0)

    if opt.scale:

        if opt.kind == 'ps1':
            from map.views import get_layer

            fns = glob('data/ps1/skycells/*/ps1-*.fits')
            fns.sort()
            #ps1-1561-021-r.fits
            if len(opt.zoom) == 0:
                opt.zoom = [1,2,3,4,5,6,7]
            print(len(fns), 'PS1 image files')
            layer = get_layer(opt.kind)
            B = layer.get_bricks()
            for i,brick in enumerate(B):
                for band in opt.bands:
                    fn0 = layer.get_filename(brick, band, 0)
                    if not os.path.exists(fn0):
                        continue
                    for scale in opt.zoom:
                        fn = layer.get_filename(brick, band, scale)
                        layer.create_scaled_image(brick, band, scale, fn)
            sys.exit(0)


        # Rebricked
        if opt.kind in ['decals-dr5', 'decals-dr5-model', 'decals-dr7', 'decals-dr7-model',
                        'eboss',
                        'mzls+bass-dr6', 'mzls+bass-dr6-model',
                        'unwise-neo3', 'unwise-neo4', 'unwise-cat-model',
                        'galex', 'wssa', 'des-dr1', 'hsc',
                    ] or opt.kind.startswith('dr8-test'): # or True:
            from map.views import get_layer

            layer = get_layer(opt.kind)

            if opt.queue:
                if len(opt.zoom) == 0:
                    opt.zoom = [1,2,3,4,5,6,7]
                #step = 0.1
                #ras = np.arange(opt.minra, opt.maxra+step, step)
                step = 3.
                #ras = np.arange(opt.minra, opt.maxra+step, step)
                decs = np.arange(opt.mindec, opt.maxdec+step, step)
                for zoom in opt.zoom:
                    for declo,dechi in zip(decs, np.clip(decs[1:], opt.mindec, opt.maxdec)):
                        rstep = step / np.maximum(0.05, np.cos(np.deg2rad((declo+dechi)/2.)))
                        ras = np.arange(opt.minra, opt.maxra+rstep, rstep)
                        for ralo,rahi in zip(ras, np.clip(ras[1:], opt.minra, opt.maxra)):
                            cmd = ('python render-tiles.py --kind %s --scale --minra %f --maxra %f --mindec %f --maxdec %f -z %i' %
                                   (opt.kind, ralo, rahi, declo, dechi, zoom))
                            print(cmd)
                sys.exit(0)


            if len(opt.zoom) == 0:
                opt.zoom = [1]

            for scale in opt.zoom:
                B = layer.get_bricks_for_scale(scale)
                print(len(B), 'bricks for scale', scale)
                B.cut((B.dec >= opt.mindec) * (B.dec <= opt.maxdec))
                print(len(B), 'in Dec range')
                B.cut((B.ra  >= opt.minra)  * (B.ra  <= opt.maxra))
                print(len(B), 'in RA range')

                bands = opt.bands

                has = {}
                for band in bands:
                    if 'has_%s' % band in B.get_columns():
                        has[band] = B.get('has_%s' % band)
                    else:
                        # assume yes
                        has[band] = np.ones(len(B), bool)

                # Run one scale at a time
                args = []
                for ibrick,brick in enumerate(B):
                    for band in bands:
                        if has[band][ibrick]:
                            args.append((layer, brick, band, scale, opt.ignore))
                print(len(args), 'bricks for scale', scale)
                mp.map(_layer_get_filename, args)

            sys.exit(0)
                
        
        if (opt.kind in ['decals-dr3', 'decals-dr3-model',
                         'mzls+bass-dr4', 'mzls+bass-dr4-model',
                         'decaps2', 'decaps2-model', 'eboss', 'ps1']
            or 'dr8b' in opt.kind
            or 'dr8c' in opt.kind):

            from map.views import get_survey, get_layer

            surveyname = opt.kind
            # *-model -> *
            # for prefix in ['decals-dr3', 'mzls+bass-dr4', 'decaps2', 'decals-dr5']:
            #     if prefix in surveyname:
            #         surveyname = prefix

            for suffix in ['-model', '-resid']:
                if surveyname.endswith(suffix):
                    surveyname = surveyname[:-len(suffix)]

            survey = get_survey(surveyname)

            print('Survey:', survey)
            print('  cache_dir:', survey.cache_dir)

            B = survey.get_bricks()
            print(len(B), 'bricks')
            B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec))
            print(len(B), 'in Dec range')
            B.cut((B.ra  >= opt.minra)  * (B.ra  < opt.maxra))
            print(len(B), 'in RA range')

            # find all image files
            filetype = 'image'
            model = False
            if '-model' in opt.kind:
                model = True
                filetype = 'model'

            bands = opt.bands

            layer = get_layer(opt.kind)

            has = {}
            for band in bands:
                if 'has_%s' % band in B.get_columns():
                    has[band] = B.get('has_%s' % band)
                else:
                    # assume yes
                    has[band] = np.ones(len(B), bool)

            for scale in opt.zoom:
                args = []
                for ibrick,brick in enumerate(B):
                    for band in bands:
                        if not has[band][ibrick]:
                            print('Brick', brick.brickname, 'does not have', band)
                            continue
                        args.append((layer, brick, band, scale, opt.ignore))
                mp.map(_layer_get_filename, args)
                
            sys.exit(0)

        elif opt.kind == 'sdss2':
            layer = get_layer(opt.kind)
            bands = 'gri'
            scales = opt.zoom
            if len(scales) == 0:
                scales = list(range(1,8))
            for scale in scales:
                #maxscale = 7
                bricks = layer.get_bricks_for_scale(scale)
                print('Got', len(bricks), 'bricks for scale', scale)
                bricks.cut((bricks.dec > opt.mindec) * (bricks.dec <= opt.maxdec) *
                           (bricks.ra  > opt.minra ) * (bricks.ra  <= opt.maxra))
                print('Cut to', len(bricks), 'bricks within RA,Dec box')

                for ibrick,brick in enumerate(bricks):
                    for band in bands:
                        print('Scaling brick', ibrick, 'of', len(bricks), 'scale', scale, 'brick', brick.brickname, 'band', band)
                        fn = layer.get_filename(brick, band, scale)
            sys.exit(0)

        if opt.kind in ['unwise-w1w2', 'unwise-neo2']:
            # scaledir = opt.kind
            # basedir = settings.DATA_DIR
            # dirnm = os.path.join(basedir, 'scaled', scaledir)
            # B = fits_table(os.path.join(basedir, 'unwise-bricks.fits'))
            layer = get_layer(opt.kind)
            B = layer.get_bricks()
            print(len(B), 'unWISE tiles')
            for b in B:
                for band in ['1','2']:
                    for scale in [1,2,3,4,5,6,7]:
                        print('Get brick', b.brickname, 'band', band, 'scale', scale)
                        layer.get_filename(b, band, scale)

        else:
            assert(False)


    if opt.bricks_exist:
        from map.views import get_survey

        surveyname = opt.kind
        filetype = 'image'

        survey = get_survey(surveyname)

        B = survey.get_bricks()
        print(len(B), 'bricks')
        B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec))
        print(len(B), 'in Dec range')
        B.cut((B.ra  >= opt.minra)  * (B.ra  < opt.maxra))
        print(len(B), 'in RA range')

        # find all image files
        bands = opt.bands

        has_band = {}
        for b in bands:
            B.set('has_%s' % b, np.zeros(len(B), bool))
            has_band[b] = B.get('has_%s' % b)
        exists = np.zeros(len(B), bool)

        for i,brick in enumerate(B.brickname):
            found = False
            for band in bands:
                fn = survey.find_file(filetype, brick=brick, band=band)
                ex = os.path.exists(fn)
                print('Brick', brick, 'band', band, 'exists?', ex)
                has_band[band][i] = ex
                if ex:
                    found = True
            exists[i] = found

        B.cut(exists)
        B.writeto('bricks-exist-%s.fits' % opt.kind)
        sys.exit(0)

    from map.views import get_survey
    surveyname = opt.kind
    if surveyname.endswith('-model'):
        surveyname = surveyname.replace('-model','')
    if surveyname.endswith('-resid'):
        surveyname = surveyname.replace('-resid','')
    survey = get_survey(surveyname)

    if len(opt.zoom) == 0:
        opt.zoom = [13]
    
    if opt.near:
        if opt.kind == 'sdss':
            B = fits_table(os.path.join(settings.DATA_DIR, 'bricks-sdssco.fits'))
        else:
            B = survey.get_bricks()
        print(len(B), 'bricks')

    #if opt.scale:
    #    opt.near_ccds = True

    if opt.near_ccds:
        if opt.kind == 'sdss':
            C = fits_table(os.path.join(settings.DATA_DIR, 'sdss', 'window_flist.fits'),
                           columns=['rerun','ra','dec', 'run', 'camcol', 'field', 'score'])
            C.cut(C.rerun == '301')
            C.cut(C.score >= 0.6)
            #C.delete_column('rerun')
            # SDSS field size
            radius = 1.01 * np.hypot(10., 14.)/2. / 60.
            ccdsize = radius
            print(len(C), 'SDSS fields')

        else:
            C = survey.get_ccds()
            print(len(C), 'CCDs')
            ccdsize = 0.2

    if opt.x is not None:
        opt.x0 = opt.x
        opt.x1 = opt.x + 1
    if opt.y is not None:
        opt.y0 = opt.y
        opt.y1 = opt.y + 1

    if opt.coadd and opt.kind == 'galex':
        layer = GalexLayer('galex')
        
        # base-level (coadd) bricks
        B = layer.get_bricks()
        print(len(B), 'bricks')
        B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec))
        print(len(B), 'in Dec range')
        B.cut((B.ra  >= opt.minra)  * (B.ra  < opt.maxra))
        print(len(B), 'in RA range')

        pat = layer.get_scaled_pattern()
        tempfiles = []
        for b in B:
            for band in ['n','f']:
                fn = pat % dict(scale=0, band=band, brickname=b.brickname)
                layer.create_coadd_image(b, band, 0, fn, tempfiles=tempfiles)
            for fn in tempfiles:
                os.unlink(fn)
        sys.exit(0)

    if opt.coadd and opt.kind == 'sdss':
        from legacypipe.survey import wcs_for_brick
        from map.views import trymakedirs

        #B = survey.get_bricks()
        B = fits_table(os.path.join(settings.DATA_DIR, 'sdss2', 'bricks-sdssco.fits'))
        print(len(B), 'bricks')
        B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec))
        print(len(B), 'in Dec range')
        B.cut((B.ra  >= opt.minra)  * (B.ra  < opt.maxra))
        print(len(B), 'in RA range')

        if opt.queue:
            # ~ square-degree tiles
            # RA slices
            rr = np.arange(opt.minra , opt.maxra +1)
            dd = np.arange(opt.mindec, opt.maxdec+1)
            for rlo,rhi in zip(rr, rr[1:]):
                for dlo,dhi in zip(dd, dd[1:]):
                    print('time python render-tiles.py --kind sdss --coadd --minra %f --maxra %f --mindec %f --maxdec %f' % (rlo, rhi, dlo, dhi))
            sys.exit(0)

        if opt.grass:
            basedir = settings.DATA_DIR
            codir = os.path.join(basedir, 'coadd', 'sdssco')
            rr,dd = [],[]
            exist = []
            for i,b in enumerate(B):
                print('Brick', b.brickname,)
                fn = os.path.join(codir, b.brickname[:3], 'sdssco-%s-%s.fits' % (b.brickname, 'r'))
                print('-->', fn,)
                if not os.path.exists(fn):
                    print()
                    continue
                print('found')
                rr.append(b.ra)
                dd.append(b.dec)
                exist.append(i)

            exist = np.array(exist)
            B.cut(exist)
            B.writeto('bricks-sdssco-exist.fits')

            import pylab as plt
            plt.clf()
            plt.plot(rr, dd, 'k.')
            plt.title('SDSS coadd tiles')
            plt.savefig('sdss.png')
            sys.exit(0)

        basedir = settings.DATA_DIR
        codir = os.path.join(basedir, 'coadd', 'sdssco')
        for b in B:
            print('Brick', b.brickname)
            wcs = wcs_for_brick(b, W=2400, H=2400, pixscale=0.396)
            #bands = 'gri'
            bands = 'z'

            dirnm = os.path.join(codir, b.brickname[:3])
            fns = [os.path.join(dirnm, 'sdssco-%s-%s.fits' % (b.brickname, band))
                   for band in bands]

            hdr = fitsio.FITSHDR()
            hdr['SURVEY'] = 'SDSS'
            wcs.add_to_header(hdr)

            if all([os.path.exists(fn) for fn in fns]):
                print('Already exist')
                continue

            from map.oldviews import map_sdss

            ims = map_sdss(req, 1, 0, 0, 0, get_images=True, wcs=wcs, ignoreCached=True,
                           forcescale=0, bands=bands)
            if ims is None:
                print('No overlap')
                continue
            trymakedirs(os.path.join(dirnm, 'xxx'))
            for fn,band,im in zip(fns,bands, ims):
                fitsio.write(fn, im, header=hdr, clobber=True)
                print('Wrote', fn)

            # Also write scaled versions
            # dirnm = os.path.join(basedir, 'scaled', 'sdssco')
            # scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(brickname).3s', 'sdssco-%(brickname)s-%(band)s.fits')
            # for im,band in zip(ims,bands):
            #     scalekwargs = dict(band=band, brick=b.brickid, brickname=b.brickname)
            #     imwcs = wcs
            #     for scale in range(1, 7):
            #         print('Writing scale level', scale)
            #         im,imwcs,sfn = get_scaled(scalepat, scalekwargs, scale, None,
            #                                   wcs=imwcs, img=im, return_data=True)
        sys.exit(0)


    for zoom in opt.zoom:
        N = 2**zoom
        if opt.y1 is None:
            y1 = N
        else:
            y1 = opt.y1

        if opt.x0 is None:
            opt.x0 = 0
        x1 = opt.x1
        if x1 is None:
            x1 = N

        if opt.split:
            decsplit = 32.
            y = 2.**zoom/(2.*np.pi) * (np.pi - np.log(np.tan(np.pi/4. + np.deg2rad(decsplit)/2.)))
            y = int(y)
            opt.y0 = y
            y1 = y+1

        # Find grid of Ra,Dec tile centers and select the ones near DECaLS bricks.
        rr,dd = [],[]
        yy = np.arange(opt.y0, y1)
        xx = np.arange(opt.x0, x1)

        if opt.grass:
            import pylab as plt
            tileexists = np.zeros((len(yy),len(xx)), bool)

            

            basedir = settings.DATA_DIR
            ver = tileversions[opt.kind][-1]
            tiledir = os.path.join(basedir, 'tiles', opt.kind, '%i'%ver, '%i'%zoom)
            for dirpath,dirnames,filenames in os.walk(tiledir):
                # change walk order
                dirnames.sort()
                if len(filenames) == 0:
                    continue
                print('Dirpath', dirpath)
                #print 'Dirnames', dirnames
                #print 'Filenames', filenames

                # check for symlinks
                if False:
                    fns = []
                    for fn in filenames:
                        fullfn = os.path.join(tiledir, dirpath, fn)
                        if os.path.isfile(fullfn) and not os.path.islink(fullfn):
                            fns.append(fn)
                    print(len(fns), 'of', len(filenames), 'are files (not symlinks)')
                    filenames = fns

                x = os.path.basename(dirpath)
                x = int(x)
                #print 'x', x

                yy = [int(fn.replace('.jpg','')) for fn in filenames]
                #print 'yy', yy
                print(len(yy), 'tiles')
                for y in yy:
                    tileexists[y - opt.y0, x - opt.x0] = True
            plt.clf()
            plt.imshow(tileexists, interpolation='nearest', origin='upper',
                       vmin=0, vmax=1, cmap='gray')
            fn = 'exist-%s-z%02i' % (opt.kind, zoom)
            plt.savefig(fn+'.png')
            fitsio.write(fn+'.fits', tileexists, clobber=True)
            print('Wrote', fn+'.png and', fn+'.fits')

            continue

        if not opt.all:
            for y in yy:
                wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, y)
                r,d = wcs.get_center()
                dd.append(d)
            for x in xx:
                wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, x, 0)
                r,d = wcs.get_center()
                rr.append(r)
            dd = np.array(dd)
            rr = np.array(rr)
            if len(dd) > 1:
                tilesize = max(np.abs(np.diff(dd)))
                print('Tile size:', tilesize)
            else:
                if opt.near_ccds or opt.near:
                    try:
                        wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0+1)
                        r2,d2 = wcs.get_center()
                    except:
                        wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0-1)
                        r2,d2 = wcs.get_center()
                    tilesize = np.abs(dd[0] - d2)
                    print('Tile size:', tilesize)
                else:
                    tilesize = 180.
            I = np.flatnonzero((dd >= opt.mindec) * (dd <= opt.maxdec))
            print('Keeping', len(I), 'Dec points between', opt.mindec, 'and', opt.maxdec)
            dd = dd[I]
            yy = yy[I]

            if opt.near_ccds:
                margin = tilesize + ccdsize
                I = np.flatnonzero((dd > C.dec.min()-margin) * (dd < C.dec.max()+margin))
                if len(I) == 0:
                    print('No Dec points within range of CCDs')
                    continue
                dd = dd[I]
                yy = yy[I]
                print('Keeping', len(I), 'Dec points within range of CCDs: Dec',
                      dd.min(), dd.max())

            I = np.flatnonzero((rr >= opt.minra) * (rr <= opt.maxra))
            print('Keeping', len(I), 'RA points between', opt.minra, 'and', opt.maxra)
            rr = rr[I]
            xx = xx[I]
            
            print(len(rr), 'RA points x', len(dd), 'Dec points')
            print('x tile range:', xx.min(), xx.max(), 'y tile range:', yy.min(), yy.max())

        for iy,y in enumerate(yy):
            print()
            print('Y row', y)

            if opt.queue:

                if 'decaps2' in opt.kind:
                    layer = get_layer(opt.kind)

                    if zoom >= layer.nativescale:
                        oldscale = 0
                    else:
                        oldscale = (layer.nativescale - zoom)
                        oldscale = np.clip(oldscale, layer.minscale, layer.maxscale)

                    x = 0
                    wcs, W, H, zoomscale, z,xi,yi = get_tile_wcs(zoom, x, y)
                    newscale = layer.get_scale(zoom, 0, y, wcs)

                    if oldscale == newscale:
                        print('Oldscale = newscale = ', oldscale)
                        continue
                    
                
                cmd = 'python -u render-tiles.py --zoom %i --y0 %i --y1 %i --kind %s --mindec %f --maxdec %f' % (zoom, y, y+1, opt.kind, opt.mindec, opt.maxdec)
                cmd += ' --threads 32'
                if opt.near_ccds:
                    cmd += ' --near-ccds'
                if opt.all:
                    cmd += ' --all'
                if opt.ignore:
                    cmd += ' --ignore'
                print(cmd)
                continue

            if opt.near:
                d = dd[iy]
                I,J,dist = match_radec(rr, d+np.zeros_like(rr), B.ra, B.dec, 0.25 + tilesize, nearest=True)
                if len(I) == 0:
                    print('No matches to bricks')
                    continue
                keep = np.zeros(len(rr), bool)
                keep[I] = True
                print('Keeping', sum(keep), 'tiles in row', y, 'Dec', d)
                x = xx[keep]
            elif opt.near_ccds:
                d = dd[iy]
                print('RA range of tiles:', rr.min(), rr.max())
                print('Dec of tile row:', d)
                I,J,dist = match_radec(rr, d+np.zeros_like(rr), C.ra, C.dec, ccdsize + tilesize, nearest=True)
                if len(I) == 0:
                    print('No matches to CCDs')
                    continue
                keep = np.zeros(len(rr), bool)
                keep[I] = True
                print('Keeping', sum(keep), 'tiles in row', y, 'Dec', d)
                x = xx[keep]
            else:
                x = xx

            # if opt.grass:
            #     for xi in x:
            #         basedir = settings.DATA_DIR
            #         ver = tileversions[opt.kind][-1]
            #         tilefn = os.path.join(basedir, 'tiles', opt.kind,
            #                               '%i/%i/%i/%i.jpg' % (ver, zoom, xi, y))
            #         print 'Checking for', tilefn
            #         if os.path.exists(tilefn):
            #             print 'EXISTS'
            #             tileexists[yi-opt.y0, xi-opt.x0]
            #     continue

            args = []
            for xi in x:
                args.append((opt.kind,zoom,xi,y, opt.ignore, False))
                #args.append((opt.kind,zoom,xi,y, opt.ignore, True))
            print('Rendering', len(args), 'tiles in row y =', y)
            mp.map(_bounce_one_tile, args, chunksize=min(100, max(1, int(len(args)/opt.threads))))
            #mp.map(_one_tile, args, chunksize=min(100, max(1, int(len(args)/opt.threads))))
            print('Rendered', len(args), 'tiles')
Ejemplo n.º 6
0
def main():
    """Main program.
    """
    import argparse

    parser = argparse.ArgumentParser(
        description=
        "This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)."
    )
    parser.add_argument('--calibs',
                        action='store_true',
                        help='Output CCDs that need to be calibrated.')

    parser.add_argument('--nper',
                        type=int,
                        default=None,
                        help='Batch N calibs per line')

    parser.add_argument('--forced',
                        action='store_true',
                        help='Output forced-photometry commands')

    parser.add_argument('--lsb',
                        action='store_true',
                        help='Output Low-Surface-Brightness commands')

    parser.add_argument('--touching',
                        action='store_true',
                        help='Cut to only CCDs touching selected bricks')
    parser.add_argument('--near',
                        action='store_true',
                        help='Quick cut to only CCDs near selected bricks')

    parser.add_argument('--check',
                        action='store_true',
                        help='Check which calibrations actually need to run.')
    parser.add_argument('--check-coadd',
                        action='store_true',
                        help='Check which caoadds actually need to run.')
    parser.add_argument('--out',
                        help='Output filename for calibs, default %(default)s',
                        default='jobs')
    parser.add_argument('--command',
                        action='store_true',
                        help='Write out full command-line to run calib')
    parser.add_argument('--opt', help='With --command, extra options to add')

    parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
    parser.add_argument('--mindec', type=float, help='Minimum Dec to run')

    parser.add_argument('--region', help='Region to select')

    parser.add_argument('--bricks', help='Set bricks.fits file to load')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')
    parser.add_argument('--ignore_cuts',
                        action='store_true',
                        default=False,
                        help='no photometric or blacklist cuts')
    parser.add_argument('--save_to_fits',
                        action='store_true',
                        default=False,
                        help='save cut brick,ccd to fits table')
    parser.add_argument(
        '--name',
        action='store',
        default='dr3',
        help='save with this suffix, e.g. refers to ccds table')

    parser.add_argument('--delete-sky',
                        action='store_true',
                        help='Delete any existing sky calibration files')
    parser.add_argument('--delete-pvastrom',
                        action='store_true',
                        help='Delete any existing PV WCS calibration files')

    parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')

    parser.add_argument(
        '--brickq',
        type=int,
        default=None,
        help='Queue only bricks with the given "brickq" value [0 to 3]')

    parser.add_argument(
        '--brickq-deps',
        action='store_true',
        default=False,
        help='Queue bricks directly using qdo API, setting brickq dependencies'
    )
    parser.add_argument('--queue',
                        default='bricks',
                        help='With --brickq-deps, the QDO queue name to use')

    opt = parser.parse_args()

    survey = LegacySurveyData()
    if opt.bricks is not None:
        B = fits_table(opt.bricks)
        log('Read', len(B), 'from', opt.bricks)
    else:
        B = survey.get_bricks()

    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        log('Read', len(T), 'from', opt.ccds)
    else:
        T = survey.get_ccds()
        log(len(T), 'CCDs')
    T.index = np.arange(len(T))

    if opt.ignore_cuts == False:
        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
    print(len(T), 'CCDs remain')

    # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
    # plt.clf()
    # plt.hist(counts, counts.max()+1)
    # plt.savefig('bricks.png')
    # B.cut(I[counts >= 9])
    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # #plt.scatter(B.ra[I], B.dec[I], c=counts)
    # plt.savefig('bricks2.png')

    # DES Stripe82
    #rlo,rhi = 350.,360.
    # rlo,rhi = 300., 10.
    # dlo,dhi = -6., 4.
    # TINY bit
    #rlo,rhi = 350.,351.1
    #dlo,dhi = 0., 1.1

    # EDR+
    # 860 bricks
    # ~10,000 CCDs
    #rlo,rhi = 239,246
    #dlo,dhi =   5, 13

    # DR1
    #rlo,rhi = 0, 360
    # part 1
    #dlo,dhi = 25, 40
    # part 2
    #dlo,dhi = 20,25
    # part 3
    #dlo,dhi = 15,20
    # part 4
    #dlo,dhi = 10,15
    # part 5
    #dlo,dhi = 5,10
    # the rest
    #dlo,dhi = -11, 5
    #dlo,dhi = 15,25.5

    dlo, dhi = -25, 40
    rlo, rhi = 0, 360

    # Arjun says 3x3 coverage area is roughly
    # RA=240-252 DEC=6-12 (but not completely rectangular)

    # COSMOS
    #rlo,rhi = 148.9, 151.2
    #dlo,dhi = 0.9, 3.5

    # A nice well-behaved region (EDR2/3)
    # rlo,rhi = 243.6, 244.6
    # dlo,dhi = 8.1, 8.6

    # 56 bricks, ~725 CCDs
    #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
    # 240 bricks, ~3000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
    # 535 bricks, ~7000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))

    if opt.region in ['test1', 'test2', 'test3', 'test4']:
        nm = dict(
            test1='2446p115',  # weird stuff around bright star
            test2='1183p292',  # faint sources around bright galaxy
            test3='3503p005',  # DES
            test4='1163p277',  # Pollux
        )[opt.region]

        B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
        log('Cut to', len(B), 'bricks')
        log(B.ra, B.dec)
        dlo, dhi = -90, 90
        rlo, rhi = 0, 360

    elif opt.region == 'edr':
        # EDR:
        # 535 bricks, ~7000 CCDs
        rlo, rhi = 240, 245
        dlo, dhi = 5, 12

    elif opt.region == 'edrplus':
        rlo, rhi = 235, 248
        dlo, dhi = 5, 15

    elif opt.region == 'edr-south':
        rlo, rhi = 240, 245
        dlo, dhi = 5, 10

    elif opt.region == 'cosmos1':
        # 16 bricks in the core of the COSMOS field.
        rlo, rhi = 149.75, 150.75
        dlo, dhi = 1.6, 2.6

    elif opt.region == 'pristine':
        # Stream?
        rlo, rhi = 240, 250
        dlo, dhi = 10, 15

    elif opt.region == 'des':
        dlo, dhi = -6., 4.
        rlo, rhi = 317., 7.

        T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
        log('Cut to', len(T), 'CCDs with "CPDES82" in filename')

    elif opt.region == 'subdes':
        rlo, rhi = 320., 360.
        dlo, dhi = -1.25, 1.25

    elif opt.region == 'northwest':
        rlo, rhi = 240, 360
        dlo, dhi = 20, 40
    elif opt.region == 'north':
        rlo, rhi = 120, 240
        dlo, dhi = 20, 40
    elif opt.region == 'northeast':
        rlo, rhi = 0, 120
        dlo, dhi = 20, 40
    elif opt.region == 'southwest':
        rlo, rhi = 240, 360
        dlo, dhi = -20, 0
    elif opt.region == 'south':
        rlo, rhi = 120, 240
        dlo, dhi = -20, 0
    elif opt.region == 'southeast':
        rlo, rhi = 0, 120
        dlo, dhi = -20, 0
    elif opt.region == 'southsoutheast':
        rlo, rhi = 0, 120
        dlo, dhi = -20, -10
    elif opt.region == 'midwest':
        rlo, rhi = 240, 360
        dlo, dhi = 0, 20
    elif opt.region == 'middle':
        rlo, rhi = 120, 240
        dlo, dhi = 0, 20
    elif opt.region == 'mideast':
        rlo, rhi = 0, 120
        dlo, dhi = 0, 20

    elif opt.region == 'grz':
        # Bricks with grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
        log('Cut to', len(B), 'bricks with grz coverage')

    elif opt.region == 'nogrz':
        # Bricks without grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
        log('Cut to', len(B), 'bricks withOUT grz coverage')
    elif opt.region == 'deep2':
        rlo, rhi = 250, 260
        dlo, dhi = 30, 35

    elif opt.region == 'deep2f3':
        rlo, rhi = 351.25, 353.75
        dlo, dhi = 0, 0.5

    elif opt.region == 'virgo':
        rlo, rhi = 185, 190
        dlo, dhi = 10, 15

    elif opt.region == 'virgo2':
        rlo, rhi = 182, 192
        dlo, dhi = 8, 18

    elif opt.region == 'lsb':
        rlo, rhi = 147.2, 147.8
        dlo, dhi = -0.4, 0.4

    elif opt.region == 'eboss-sgc':
        # generous boundaries to make sure get all relevant images
        # RA -45 to +45
        # Dec -5 to +7
        rlo, rhi = 310., 50.
        dlo, dhi = -6., 6.

    elif opt.region == 'eboss-ngc':
        # generous boundaries to make sure get all relevant images
        # NGC ELGs
        # RA 115 to 175
        # Dec 15 to  30
        rlo, rhi = 122., 177.
        dlo, dhi = 12., 32.

    elif opt.region == 'mzls':
        dlo, dhi = 30., 90.
    elif opt.region == 'dr4-bootes':
        # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
        #dlo,dhi = 34., 35.
        #rlo,rhi = 209.5, 210.5
        dlo, dhi = 33., 36.
        rlo, rhi = 216.5, 219.5

    if opt.mindec is not None:
        dlo = opt.mindec
    if opt.maxdec is not None:
        dhi = opt.maxdec

    if rlo < rhi:
        B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi))
    else:  # RA wrap
        B.cut(
            np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) *
            (B.dec <= dhi))
    log(len(B), 'bricks in range')
    #for name in B.get('brickname'):
    #print(name)
    B.writeto('bricks-cut.fits')

    I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize)
    keep = np.zeros(len(B), bool)
    for i in I:
        keep[i] = True
    B.cut(keep)
    log('Cut to', len(B), 'bricks near CCDs')

    plt.clf()
    plt.plot(B.ra, B.dec, 'b.')
    plt.title('DR3 bricks')
    plt.axis([360, 0, np.min(B.dec) - 1, np.max(B.dec) + 1])
    plt.savefig('bricks.png')

    if opt.brickq is not None:
        B.cut(B.brickq == opt.brickq)
        log('Cut to', len(B), 'with brickq =', opt.brickq)

    if opt.touching:
        keep = np.zeros(len(T), bool)
        for j in J:
            keep[j] = True
        T.cut(keep)
        log('Cut to', len(T), 'CCDs near bricks')

    # Aside -- how many near DR1=1 CCDs?
    if False:
        T2 = D.get_ccds()
        log(len(T2), 'CCDs')
        T2.cut(T2.dr1 == 1)
        log(len(T2), 'CCDs marked DR1=1')
        log(len(B), 'bricks in range')
        I, J, d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize)
        keep = np.zeros(len(B), bool)
        for i in I:
            keep[i] = True
        B2 = B[keep]
        log('Total of', len(B2), 'bricks near CCDs with DR1=1')
        for band in 'grz':
            Tb = T2[T2.filter == band]
            log(len(Tb), 'in filter', band)
            I, J, d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec,
                                  survey.bricksize)
            good = np.zeros(len(B2), np.uint8)
            for i in I:
                good[i] = 1
            B2.set('has_' + band, good)

        B2.writeto('survey-bricks-in-dr1.fits')
        sys.exit(0)

    # sort by dec decreasing
    #B.cut(np.argsort(-B.dec))
    # RA increasing
    B.cut(np.argsort(B.ra))

    for b in B:
        if opt.check:
            fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3],
                                                      b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue
        if opt.check_coadd:
            fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (
                b.brickname[:3], b.brickname, b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue

        #print(b.brickname)

    if opt.save_to_fits:
        assert (opt.touching)
        # Write cut tables to file
        for tab, typ in zip([B, T], ['bricks', 'ccds']):
            fn = '%s-%s-cut.fits' % (typ, opt.region)
            if os.path.exists(fn):
                os.remove(fn)
            tab.writeto(fn)
            print('Wrote %s' % fn)
        # Write text files listing ccd and filename names
        nm1, nm2 = 'ccds-%s.txt' % opt.region, 'filenames-%s.txt' % opt.region
        if os.path.exists(nm1):
            os.remove(nm1)
        if os.path.exists(nm2):
            os.remove(nm2)
        f1, f2 = open(nm1, 'w'), open(nm2, 'w')
        fns = list(set(T.get('image_filename')))
        for fn in fns:
            f2.write('%s\n' % fn.strip())
        for ti in T:
            f1.write('%s\n' % ti.get('image_filename').strip())
        f1.close()
        f2.close()
        print('Wrote *-names.txt')

    if opt.brickq_deps:
        import qdo
        from legacypipe.survey import on_bricks_dependencies

        #... find Queue...
        q = qdo.connect(opt.queue, create_ok=True)
        print('Connected to QDO queue', opt.queue, q)
        brick_to_task = dict()

        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
        print(len(T), 'CCDs remaining')

        T.wra = T.ra + (T.ra > 180) * -360
        wra = rlo - 360
        plt.clf()
        plt.plot(T.wra, T.dec, 'b.')
        ax = [wra, rhi, dlo, dhi]
        plt.axis(ax)
        plt.title('CCDs')
        plt.savefig('q-ccds.png')

        B.wra = B.ra + (B.ra > 180) * -360

        # this slight overestimate (for DECam images) is fine
        radius = 0.3
        Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius, indexlist=True)
        ikeep = []
        for ib, (b, Iccd) in enumerate(zip(B, Iccds)):
            if Iccd is None or len(Iccd) == 0:
                print('No matched CCDs to brick', b.brickname)
                continue
            wcs = wcs_for_brick(b)
            cI = ccds_touching_wcs(wcs, T[np.array(Iccd)])
            print(len(cI), 'CCDs touching brick', b.brickname)
            if len(cI) == 0:
                continue
            ikeep.append(ib)
        B.cut(np.array(ikeep))
        print('Cut to', len(B), 'bricks touched by CCDs')

        for brickq in range(4):
            I = np.flatnonzero(B.brickq == brickq)
            print(len(I), 'bricks with brickq =', brickq)

            J = np.flatnonzero(B.brickq < brickq)
            preB = B[J]
            reqs = []
            if brickq > 0:
                for b in B[I]:
                    # find brick dependencies
                    brickdeps = on_bricks_dependencies(b, survey, bricks=preB)
                    # convert to task ids
                    taskdeps = [
                        brick_to_task.get(b.brickname, None) for b in brickdeps
                    ]
                    # If we dropped a dependency brick from a previous brickq because
                    # of no overlapping CCDs, it won't appear in the brick_to_task map.
                    taskdeps = [t for t in taskdeps if t is not None]
                    reqs.append(taskdeps)

            plt.clf()
            plt.plot(B.wra, B.dec, '.', color='0.5')
            plt.plot(B.wra[I], B.dec[I], 'b.')
            plt.axis(ax)
            plt.title('Bricks: brickq=%i' % brickq)
            plt.savefig('q-bricks-%i.png' % brickq)

            # submit to qdo queue
            print('Queuing', len(B[I]), 'bricks')
            if brickq == 0:
                reqs = None
            else:
                assert (len(I) == len(reqs))
            taskids = q.add_multiple(B.brickname[I], requires=reqs)
            assert (len(taskids) == len(I))
            print('Queued', len(taskids), 'bricks')
            brick_to_task.update(dict(zip(B.brickname[I], taskids)))

    if not (opt.calibs or opt.forced or opt.lsb):
        sys.exit(0)

    bands = 'grz'
    log('Filters:', np.unique(T.filter))
    T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
    log('Cut to', len(T), 'CCDs in filters', bands)

    if opt.touching:
        allI = set()
        for b in B:
            wcs = wcs_for_brick(b)
            I = ccds_touching_wcs(wcs, T)
            log(len(I), 'CCDs for brick', b.brickid,
                'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
            if len(I) == 0:
                continue
            allI.update(I)
        allI = list(allI)
        allI.sort()
    elif opt.near:
        # Roughly brick radius + DECam image radius
        radius = 0.35
        allI, nil, nil = match_radec(T.ra,
                                     T.dec,
                                     B.ra,
                                     B.dec,
                                     radius,
                                     nearest=True)
    else:
        allI = np.arange(len(T))

    if opt.write_ccds:
        T[allI].writeto(opt.write_ccds)
        log('Wrote', opt.write_ccds)

    ## Be careful here -- T has been cut; we want to write out T.index.
    ## 'allI' contains indices into T.

    if opt.forced:
        log('Writing forced-photometry commands to', opt.out)
        f = open(opt.out, 'w')
        log('Total of', len(allI), 'CCDs')
        for j, i in enumerate(allI):
            expstr = '%08i' % T.expnum[i]
            outfn = os.path.join(
                'forced', expstr[:5], expstr,
                'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i]))
            imgfn = os.path.join(survey.survey_dir, 'images',
                                 T.image_filename[i].strip())
            if (not os.path.exists(imgfn) and imgfn.endswith('.fz')
                    and os.path.exists(imgfn[:-3])):
                imgfn = imgfn[:-3]

            #f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' %
            #        (imgfn, T.image_hdu[i], outfn))

            f.write(
                'python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n'
                % (T.expnum[i], T.ccdname[i], outfn))

        f.close()
        log('Wrote', opt.out)
        sys.exit(0)

    if opt.lsb:
        log('Writing LSB commands to', opt.out)
        f = open(opt.out, 'w')
        log('Total of', len(allI), 'CCDs')
        for j, i in enumerate(allI):
            exp = T.expnum[i]
            ext = T.ccdname[i].strip()
            outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
            f.write(
                'python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n'
                % (exp, ext, outfn, exp, ext))
        f.close()
        log('Wrote', opt.out)
        sys.exit(0)

    log('Writing calibs to', opt.out)
    f = open(opt.out, 'w')
    log('Total of', len(allI), 'CCDs')

    batch = []

    def write_batch(f, batch, cmd):
        if cmd is None:
            cmd = ''
        f.write(cmd + ' '.join(batch) + '\n')

    cmd = None
    if opt.command:
        cmd = 'python legacypipe/run-calib.py '
        if opt.opt is not None:
            cmd += opt.opt + ' '

    for j, i in enumerate(allI):

        if opt.delete_sky or opt.delete_pvastrom:
            log(j + 1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if opt.delete_sky and os.path.exists(im.skyfn):
                log('  deleting:', im.skyfn)
                os.unlink(im.skyfn)
            if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
                log('  deleting:', im.pvwcsfn)
                os.unlink(im.pvwcsfn)

        if opt.check:
            log(j + 1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if not im.run_calibs(im, just_check=True):
                log('Calibs for', im.expnum, im.ccdname, im.calname,
                    'already done')
                continue

        if opt.command:
            s = '%i-%s' % (T.expnum[i], T.ccdname[i])
            prefix = 'python legacypipe/run-calib.py ' + opt.opt
            #('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
            #     (T.expnum[i], T.ccdname[i]))
        else:
            s = '%i' % T.index[i]
            prefix = ''

        if j < 10:
            print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname',
                  T.ccdname[i], 'filename', T.image_filename[i])

        if not opt.nper:
            f.write(prefix + s + '\n')
        else:
            batch.append(s)
            if len(batch) >= opt.nper:
                write_batch(f, batch, cmd)
                batch = []

        if opt.check:
            f.flush()

    if len(batch):
        write_batch(f, batch, cmd)

    f.close()
    log('Wrote', opt.out)
    return 0