Пример #1
0
def queue():
    if False:
        survey = LegacySurveyData()
        ccds = survey.get_ccds()
        bricks = survey.get_bricks()
        print(len(bricks), 'bricks')
        print(len(ccds), 'CCDs')

        bricks.cut((bricks.dec >= -30) * (bricks.dec <= 30))
        print(len(bricks), 'in Dec [-30, +30]')

        I = survey.photometric_ccds(ccds)
        ccds.cut(I)
        print(len(ccds), 'pass photometric cut')

        I, J, d = match_radec(bricks.ra,
                              bricks.dec,
                              ccds.ra,
                              ccds.dec,
                              0.5,
                              nearest=True)
        print(len(I), 'bricks with CCDs nearby')
        bricks.cut(I)
        bricknames = bricks.brickname

    else:
        # DR7: use Martin's list of bricks w/ CCD coverage
        f = open('nccds.dat')
        bricknames = []
        for line in f.readlines():
            words = line.strip().split(' ')
            brick = words[0]
            nccd = int(words[1])
            if nccd > 0:
                bricknames.append(brick)

    # qdo
    bb = bricknames
    while len(bb):
        print(' '.join(bb[:100]))
        bb = bb[100:]
    return

    mp = multiproc(16)
    N = len(bricks)
    args = [(brick, i, N, plots, {}) for i, brick in enumerate(bricks)]
    mp.map(run_one_brick, args)
Пример #2
0
def main():
    """Main program.
    """
    import argparse

    parser = argparse.ArgumentParser(
        description=
        "This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)."
    )
    parser.add_argument('--calibs',
                        action='store_true',
                        help='Output CCDs that need to be calibrated.')

    parser.add_argument('--nper',
                        type=int,
                        default=None,
                        help='Batch N calibs per line')
    parser.add_argument(
        '--byexp',
        action='store_true',
        default=False,
        help='Run one whole exposure per job (not one CCD per job)')

    parser.add_argument('--forced',
                        action='store_true',
                        help='Output forced-photometry commands')

    parser.add_argument('--lsb',
                        action='store_true',
                        help='Output Low-Surface-Brightness commands')

    parser.add_argument('--stage', help='Stage image files to given directory')

    parser.add_argument('--touching',
                        action='store_true',
                        help='Cut to only CCDs touching selected bricks')
    parser.add_argument('--near',
                        action='store_true',
                        help='Quick cut to only CCDs near selected bricks')

    parser.add_argument('--check-coadd',
                        action='store_true',
                        help='Check which coadds actually need to run.')
    parser.add_argument('--out',
                        help='Output filename for calibs, default %(default)s',
                        default='jobs')
    parser.add_argument('--command',
                        action='store_true',
                        help='Write out full command-line to run calib')
    parser.add_argument('--opt', help='With --command, extra options to add')

    parser.add_argument('--maxra', type=float, help='Maximum RA to run')
    parser.add_argument('--minra', type=float, help='Minimum RA to run')
    parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
    parser.add_argument('--mindec', type=float, help='Minimum Dec to run')

    parser.add_argument('--region', help='Region to select')

    parser.add_argument('--bricks', help='Set bricks.fits file to load')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')
    parser.add_argument('--ignore_cuts',
                        action='store_true',
                        default=False,
                        help='no photometric cuts')
    parser.add_argument('--save_to_fits',
                        action='store_true',
                        default=False,
                        help='save cut brick,ccd to fits table')
    parser.add_argument(
        '--name',
        action='store',
        default='dr3',
        help='save with this suffix, e.g. refers to ccds table')

    parser.add_argument('--delete-sky',
                        action='store_true',
                        help='Delete any existing sky calibration files')

    parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')

    parser.add_argument('--nccds',
                        action='store_true',
                        default=False,
                        help='Prints number of CCDs per brick')

    parser.add_argument('--bands',
                        default='g,r,z',
                        help='Set bands to keep: comma-separated list.')

    opt = parser.parse_args()

    want_ccds = (opt.calibs or opt.forced or opt.lsb)
    want_bricks = not want_ccds

    survey = LegacySurveyData()
    if opt.bricks is not None:
        B = fits_table(opt.bricks)
        log('Read', len(B), 'from', opt.bricks)
    else:
        B = survey.get_bricks()

    log('Bricks Dec range:', B.dec.min(), B.dec.max())

    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        log('Read', len(T), 'from', opt.ccds)
    else:
        T = survey.get_ccds()
        log(len(T), 'CCDs')
    T.index = np.arange(len(T))

    if opt.ignore_cuts == False:
        log('Applying CCD cuts...')
        if 'ccd_cuts' in T.columns():
            T.cut(T.ccd_cuts == 0)
            log(len(T), 'CCDs survive cuts')

    bands = opt.bands.split(',')
    log('Filters:', np.unique(T.filter))
    T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
    log('Cut to', len(T), 'CCDs in filters', bands)

    log('CCDs Dec range:', T.dec.min(), T.dec.max())

    # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
    # plt.clf()
    # plt.hist(counts, counts.max()+1)
    # plt.savefig('bricks.png')
    # B.cut(I[counts >= 9])
    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # #plt.scatter(B.ra[I], B.dec[I], c=counts)
    # plt.savefig('bricks2.png')

    # DES Stripe82
    #rlo,rhi = 350.,360.
    # rlo,rhi = 300., 10.
    # dlo,dhi = -6., 4.
    # TINY bit
    #rlo,rhi = 350.,351.1
    #dlo,dhi = 0., 1.1

    # EDR+
    # 860 bricks
    # ~10,000 CCDs
    #rlo,rhi = 239,246
    #dlo,dhi =   5, 13

    # DR1
    #rlo,rhi = 0, 360
    # part 1
    #dlo,dhi = 25, 40
    # part 2
    #dlo,dhi = 20,25
    # part 3
    #dlo,dhi = 15,20
    # part 4
    #dlo,dhi = 10,15
    # part 5
    #dlo,dhi = 5,10
    # the rest
    #dlo,dhi = -11, 5
    #dlo,dhi = 15,25.5

    dlo, dhi = -25, 40
    rlo, rhi = 0, 360

    # Arjun says 3x3 coverage area is roughly
    # RA=240-252 DEC=6-12 (but not completely rectangular)

    # COSMOS
    #rlo,rhi = 148.9, 151.2
    #dlo,dhi = 0.9, 3.5

    # A nice well-behaved region (EDR2/3)
    # rlo,rhi = 243.6, 244.6
    # dlo,dhi = 8.1, 8.6

    # 56 bricks, ~725 CCDs
    #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
    # 240 bricks, ~3000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
    # 535 bricks, ~7000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))

    if opt.region in ['test1', 'test2', 'test3', 'test4']:
        nm = dict(
            test1='2446p115',  # weird stuff around bright star
            test2='1183p292',  # faint sources around bright galaxy
            test3='3503p005',  # DES
            test4='1163p277',  # Pollux
        )[opt.region]

        B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
        log('Cut to', len(B), 'bricks')
        log(B.ra, B.dec)
        dlo, dhi = -90, 90
        rlo, rhi = 0, 360

    elif opt.region == 'edr':
        # EDR:
        # 535 bricks, ~7000 CCDs
        rlo, rhi = 240, 245
        dlo, dhi = 5, 12

    elif opt.region == 'dr8-decam':
        rlo, rhi = 0, 360
        dlo, dhi = -70, 40
        log('DR8-DECam region')

    elif opt.region == 'edrplus':
        rlo, rhi = 235, 248
        dlo, dhi = 5, 15

    elif opt.region == 'edr-south':
        rlo, rhi = 240, 245
        dlo, dhi = 5, 10

    elif opt.region == 'cosmos1':
        # 16 bricks in the core of the COSMOS field.
        rlo, rhi = 149.75, 150.75
        dlo, dhi = 1.6, 2.6

    elif opt.region == 'pristine':
        # Stream?
        rlo, rhi = 240, 250
        dlo, dhi = 10, 15

    elif opt.region == 'des':
        dlo, dhi = -6., 4.
        rlo, rhi = 317., 7.

        T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
        log('Cut to', len(T), 'CCDs with "CPDES82" in filename')

    elif opt.region == 'subdes':
        rlo, rhi = 320., 360.
        dlo, dhi = -1.25, 1.25

    elif opt.region == 'northwest':
        rlo, rhi = 240, 360
        dlo, dhi = 20, 40
    elif opt.region == 'north':
        rlo, rhi = 120, 240
        dlo, dhi = 20, 40
    elif opt.region == 'northeast':
        rlo, rhi = 0, 120
        dlo, dhi = 20, 40
    elif opt.region == 'southwest':
        rlo, rhi = 240, 360
        dlo, dhi = -20, 0
    elif opt.region == 'south':
        rlo, rhi = 120, 240
        dlo, dhi = -20, 0
    elif opt.region == 'southeast':
        rlo, rhi = 0, 120
        dlo, dhi = -20, 0
    elif opt.region == 'southsoutheast':
        rlo, rhi = 0, 120
        dlo, dhi = -20, -10
    elif opt.region == 'midwest':
        rlo, rhi = 240, 360
        dlo, dhi = 0, 20
    elif opt.region == 'middle':
        rlo, rhi = 120, 240
        dlo, dhi = 0, 20
    elif opt.region == 'mideast':
        rlo, rhi = 0, 120
        dlo, dhi = 0, 20

    elif opt.region == 'grz':
        # Bricks with grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
        log('Cut to', len(B), 'bricks with grz coverage')

    elif opt.region == 'nogrz':
        # Bricks without grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
        log('Cut to', len(B), 'bricks withOUT grz coverage')

    elif opt.region == 'deep2':
        rlo, rhi = 250, 260
        dlo, dhi = 30, 35

    elif opt.region == 'deep2f2':
        rlo, rhi = 251.4, 254.4
        dlo, dhi = 34.6, 35.3

    elif opt.region == 'deep2f3':
        rlo, rhi = 351.25, 353.75
        dlo, dhi = 0, 0.5

    elif opt.region == 'deep3':
        rlo, rhi = 214, 216
        dlo, dhi = 52.25, 53.25

    elif opt.region == 'virgo':
        rlo, rhi = 185, 190
        dlo, dhi = 10, 15

    elif opt.region == 'virgo2':
        rlo, rhi = 182, 192
        dlo, dhi = 8, 18

    elif opt.region == 'coma':
        # van Dokkum et al Coma cluster ultra-diffuse galaxies: 3x3 field centered on Coma cluster
        rc, dc = 195., 28.
        dd = 1.5
        cosdec = np.cos(np.deg2rad(dc))
        rlo, rhi = rc - dd / cosdec, rc + dd / cosdec
        dlo, dhi = dc - dd, dc + dd

    elif opt.region == 'lsb':
        rlo, rhi = 147.2, 147.8
        dlo, dhi = -0.4, 0.4

    elif opt.region == 'eboss-sgc':
        # generous boundaries to make sure get all relevant images
        # RA -45 to +45
        # Dec -5 to +7
        rlo, rhi = 310., 50.
        dlo, dhi = -6., 6.

    elif opt.region == 'eboss-ngc':
        # generous boundaries to make sure get all relevant images
        # NGC ELGs
        # RA 115 to 175
        # Dec 15 to  30
        # rlo,rhi = 122., 177.
        # dlo,dhi =  12.,  32.
        rlo, rhi = 126., 168.
        dlo, dhi = 18., 33.

    elif opt.region == 'mzls':
        dlo, dhi = -10., 90.  # -10: pull in Stripe 82 data too

    elif opt.region == 'dr4-bootes':
        # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
        #dlo,dhi = 34., 35.
        #rlo,rhi = 209.5, 210.5
        dlo, dhi = 33., 36.
        rlo, rhi = 216.5, 219.5

    elif opt.region == 'des-sn-x3':
        #rlo,rhi = 36., 37.
        #dlo,dhi = -5., -4.
        rlo, rhi = 36., 36.5
        dlo, dhi = -4.5, -4.

    elif opt.region == 'ngc2632':
        # open cluster
        rlo, rhi = 129.0, 131.0
        dlo, dhi = 19.0, 20.5

    elif opt.region == 'dr8sky':
        rlo, rhi = 35.0, 37.0
        dlo, dhi = -3.0, -1.0

    # ADM DR8 test regions, see, e.g.:
    # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR8#Testregions
    elif opt.region == 'dr8-test-s82':
        rlo, rhi = 0, 45
        dlo, dhi = -1.25, 1.25
    elif opt.region == 'dr8-test-hsc-sgc':
        rlo, rhi = 30, 40
        dlo, dhi = -6.5, -1.25
    elif opt.region == 'dr8-test-hsc-ngc':
        rlo, rhi = 177.5, 182.5
        dlo, dhi = -1, 1
    elif opt.region == 'dr8-test-edr':
        rlo, rhi = 240, 245
        dlo, dhi = 5, 12
    elif opt.region == 'dr8-test-hsc-north':
        rlo, rhi = 240, 250
        dlo, dhi = 42, 45
    elif opt.region == 'dr8-test-deep2-egs':
        rlo, rhi = 213, 216.5
        dlo, dhi = 52, 54
    elif opt.region == 'dr8-test-overlap':
        rlo, rhi = 132, 140.5
        dlo, dhi = 31.5, 35

    if opt.mindec is not None:
        dlo = opt.mindec
    if opt.maxdec is not None:
        dhi = opt.maxdec
    if opt.minra is not None:
        rlo = opt.minra
    if opt.maxra is not None:
        rhi = opt.maxra

    if rlo < rhi:
        B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi))
    else:  # RA wrap
        B.cut(
            np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) *
            (B.dec <= dhi))
    log(len(B), 'bricks in range; cut Dec range', B.dec.min(), B.dec.max())
    #for name in B.get('brickname'):
    #    print(name)
    #B.writeto('bricks-cut.fits')

    bricksize = 0.25
    # A bit more than 0.25-degree brick radius + Bok image radius ~ 0.57
    search_radius = 1.05 * np.sqrt(2.) * (bricksize +
                                          (0.455 * 4096 / 3600.)) / 2.

    log(len(T), 'CCDs')
    log(len(B), 'Bricks')
    I, J, d = match_radec(B.ra,
                          B.dec,
                          T.ra,
                          T.dec,
                          search_radius,
                          nearest=True)
    B.cut(I)
    log('Cut to', len(B), 'bricks near CCDs')
    log('Bricks Dec range:', B.dec.min(), B.dec.max())

    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # plt.title('DR3 bricks')
    # plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
    # plt.savefig('bricks.png')

    if opt.touching:
        I, J, d = match_radec(T.ra,
                              T.dec,
                              B.ra,
                              B.dec,
                              search_radius,
                              nearest=True)
        # list the ones that will be cut
        # drop = np.ones(len(T))
        # drop[I] = False
        # for i in np.flatnonzero(drop):
        #     from astrometry.util.starutil_numpy import degrees_between
        #     dists = degrees_between(B.ra, B.dec, T.ra[i], T.dec[i])
        #     mindist = min(dists)
        #     print('Dropping:', T.ra[i], T.dec[i], 'min dist', mindist, 'search_radius', search_radius)

        T.cut(I)
        log('Cut to', len(T), 'CCDs near bricks')

    # sort by RA increasing
    B.cut(np.argsort(B.ra))

    if opt.save_to_fits:
        assert (opt.touching)
        # Write cut tables to file
        for tab, typ in zip([B, T], ['bricks', 'ccds']):
            fn = '%s-%s-cut.fits' % (typ, opt.region)
            if os.path.exists(fn):
                os.remove(fn)
            tab.writeto(fn)
            log('Wrote %s' % fn)
        # Write text files listing ccd and filename names
        # nm1,nm2= 'ccds-%s.txt'% opt.region,'filenames-%s.txt' % opt.region
        # if os.path.exists(nm1):
        #     os.remove(nm1)
        # if os.path.exists(nm2):
        #     os.remove(nm2)
        # f1,f2=open(nm1,'w'),open(nm2,'w')
        # fns= list(set(T.get('image_filename')))
        # for fn in fns:
        #     f2.write('%s\n' % fn.strip())
        # for ti in T:
        #     f1.write('%s\n' % ti.get('image_filename').strip())
        # f1.close()
        # f2.close()
        # log('Wrote *-names.txt')

    if opt.touching:

        if want_bricks:
            # Shortcut the list of bricks that are definitely touching CCDs --
            # a brick-ccd pair within this radius must be touching.
            closest_radius = 0.95 * (bricksize + 0.262 * 2048 / 3600.) / 2.

            J1, nil, nil = match_radec(B.ra,
                                       B.dec,
                                       T.ra,
                                       T.dec,
                                       closest_radius,
                                       nearest=True)
            log(len(J1), 'of', len(B), 'bricks definitely touch CCDs')
            tocheck = np.ones(len(B), bool)
            tocheck[J1] = False
            J2 = []
            for j in np.flatnonzero(tocheck):
                b = B[j]
                wcs = wcs_for_brick(b)
                I = ccds_touching_wcs(wcs, T)
                log(len(I), 'CCDs for brick', b.brickname)
                if len(I) == 0:
                    continue
                J2.append(j)
            J = np.hstack((J1, J2))
            J = np.sort(J).astype(int)
            B.cut(J)
            log('Cut to', len(B), 'bricks touching CCDs')

        else:
            J = []
            allI = set()
            for j, b in enumerate(B):
                wcs = wcs_for_brick(b)
                I = ccds_touching_wcs(wcs, T)
                log(len(I), 'CCDs for brick', b.brickname)
                if len(I) == 0:
                    continue
                allI.update(I)
                J.append(j)
            allI = list(allI)
            allI.sort()
            B.cut(np.array(J))
            log('Cut to', len(B), 'bricks touching CCDs')

    elif opt.near:
        # Find CCDs near bricks
        allI, nil, nil = match_radec(T.ra,
                                     T.dec,
                                     B.ra,
                                     B.dec,
                                     search_radius,
                                     nearest=True)
        # Find bricks near CCDs
        J, nil, nil = match_radec(B.ra,
                                  B.dec,
                                  T.ra,
                                  T.dec,
                                  search_radius,
                                  nearest=True)
        B.cut(J)
        log('Cut to', len(B), 'bricks near CCDs')

    else:
        allI = np.arange(len(T))

    if opt.byexp:
        nil, eI = np.unique(T.expnum[allI], return_index=True)
        allI = allI[eI]
        print('Cut to', len(allI), 'expnums')

    if opt.nccds:
        from queue import Queue
        from threading import Thread

        log('Checking number of CCDs per brick')

        def worker():
            while True:
                i = q.get()
                if i is None:
                    break
                b = B[i]
                wcs = wcs_for_brick(b)
                I = ccds_touching_wcs(wcs, T)
                log(b.brickname, len(I))
                q.task_done()

        q = Queue()
        num_threads = 24
        threads = []

        for i in range(num_threads):
            t = Thread(target=worker)
            t.start()
            threads.append(t)

        for i in range(len(B)):
            q.put(i)

        q.join()
        for i in range(num_threads):
            q.put(None)
        for t in threads:
            t.join()

    if opt.write_ccds:
        T[allI].writeto(opt.write_ccds)
        log('Wrote', opt.write_ccds)

    if want_bricks:
        # Print the list of bricks and exit.
        for b in B:
            print(b.brickname)
        if opt.save_to_fits:
            B.writeto('bricks-%s-touching.fits' % opt.region)
        if not want_ccds:
            sys.exit(0)

    ## Be careful here -- T has been cut; we want to write out T.index.
    ## 'allI' contains indices into T.

    if opt.stage is not None:
        cmd_pat = 'rsync -LRarv %s %s'
        fns = set()
        for iccd in allI:
            im = survey.get_image_object(T[iccd])
            fns.update([
                im.imgfn, im.wtfn, im.dqfn, im.psffn, im.merged_psffn,
                im.merged_splineskyfn, im.splineskyfn
            ])
        for i, fn in enumerate(fns):
            print('File', i + 1, 'of', len(fns), ':', fn)
            if not os.path.exists(fn):
                print('No such file:', fn)
                continue
            base = survey.get_survey_dir()
            if base.endswith('/'):
                base = base[:-1]

            rel = os.path.relpath(fn, base)

            dest = os.path.join(opt.stage, rel)
            print('Dest:', dest)
            if os.path.exists(dest):
                print('Exists:', dest)
                continue

            cmd = cmd_pat % ('%s/./%s' % (base, rel), opt.stage)
            print(cmd)
            rtn = os.system(cmd)
            assert (rtn == 0)
        sys.exit(0)

    if opt.forced:
        log('Writing forced-photometry commands to', opt.out)
        f = open(opt.out, 'w')
        log('Total of', len(allI), 'CCDs')
        for j, i in enumerate(allI):
            expstr = '%08i' % T.expnum[i]
            imgfn = os.path.join(survey.survey_dir, 'images',
                                 T.image_filename[i].strip())
            if (not os.path.exists(imgfn) and imgfn.endswith('.fz')
                    and os.path.exists(imgfn[:-3])):
                imgfn = imgfn[:-3]

            outfn = os.path.join(
                expstr[:5], expstr, 'forced-%s-%s-%s.fits' %
                (T.camera[i].strip(), expstr, T.ccdname[i]))

            f.write(
                'python legacypipe/forced_photom.py --apphot --derivs --catalog-dir /project/projectdirs/cosmo/data/legacysurvey/dr7/ %i %s forced/%s\n'
                % (T.expnum[i], T.ccdname[i], outfn))

        f.close()
        log('Wrote', opt.out)

        fn = 'forced-ccds.fits'
        T[allI].writeto(fn)
        print('Wrote', fn)

        sys.exit(0)

    if opt.lsb:
        log('Writing LSB commands to', opt.out)
        f = open(opt.out, 'w')
        log('Total of', len(allI), 'CCDs')
        for j, i in enumerate(allI):
            exp = T.expnum[i]
            ext = T.ccdname[i].strip()
            outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
            f.write(
                'python legacyanalysis/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n'
                % (exp, ext, outfn, exp, ext))
        f.close()
        log('Wrote', opt.out)
        sys.exit(0)

    log('Writing calibs to', opt.out)
    f = open(opt.out, 'w')
    log('Total of', len(allI), 'CCDs')

    batch = []

    def write_batch(f, batch, cmd):
        if cmd is None:
            cmd = ''
        f.write(cmd + ' '.join(batch) + '\n')

    cmd = None
    if opt.command:
        cmd = 'python legacypipe/run-calib.py '
        if opt.opt is not None:
            cmd += opt.opt + ' '

    for j, i in enumerate(allI):

        if opt.delete_sky:
            log(j + 1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if opt.delete_sky and os.path.exists(im.skyfn):
                log('  deleting:', im.skyfn)
                os.unlink(im.skyfn)

        if opt.command:
            if opt.byexp:
                s = '--expnum %i' % (T.expnum[i])
            else:
                s = '%i-%s' % (T.expnum[i], T.ccdname[i])
            prefix = 'python legacypipe/run-calib.py '
            if opt.opt is not None:
                prefix = prefix + opt.opt
            #('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
            #     (T.expnum[i], T.ccdname[i]))
        else:
            s = '%i' % T.index[i]
            prefix = ''

        if j < 10:
            print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname',
                  T.ccdname[i], 'filename', T.image_filename[i])

        if not opt.nper:
            f.write(prefix + s + '\n')
        else:
            batch.append(s)
            if len(batch) >= opt.nper:
                write_batch(f, batch, cmd)
                batch = []

    if len(batch):
        write_batch(f, batch, cmd)

    f.close()
    log('Wrote', opt.out)
    return 0
Пример #3
0
def main():
    """Main program.
    """
    import argparse

    parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).")
    parser.add_argument('--calibs', action='store_true',
                      help='Output CCDs that need to be calibrated.')

    parser.add_argument('--nper', type=int, default=None,
                      help='Batch N calibs per line')

    parser.add_argument('--forced', action='store_true',
                      help='Output forced-photometry commands')

    parser.add_argument('--lsb', action='store_true',
                      help='Output Low-Surface-Brightness commands')

    parser.add_argument('--touching', action='store_true',
                      help='Cut to only CCDs touching selected bricks')
    parser.add_argument('--near', action='store_true',
                      help='Quick cut to only CCDs near selected bricks')

    parser.add_argument('--check', action='store_true',
                      help='Check which calibrations actually need to run.')
    parser.add_argument('--check-coadd', action='store_true',
                      help='Check which caoadds actually need to run.')
    parser.add_argument('--out', help='Output filename for calibs, default %(default)s',
                      default='jobs')
    parser.add_argument('--command', action='store_true',
                      help='Write out full command-line to run calib')
    parser.add_argument('--opt', help='With --command, extra options to add')
    
    parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
    parser.add_argument('--mindec', type=float, help='Minimum Dec to run')

    parser.add_argument('--region', help='Region to select')

    parser.add_argument('--bricks', help='Set bricks.fits file to load')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')
    parser.add_argument('--ignore_cuts', action='store_true',default=False,help='no photometric or blacklist cuts')
    parser.add_argument('--save_to_fits', action='store_true',default=False,help='save cut brick,ccd to fits table')
    parser.add_argument('--name', action='store',default='dr3',help='save with this suffix, e.g. refers to ccds table')

    parser.add_argument('--delete-sky', action='store_true',
                      help='Delete any existing sky calibration files')
    parser.add_argument('--delete-pvastrom', action='store_true',
                      help='Delete any existing PV WCS calibration files')

    parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')

    parser.add_argument('--brickq', type=int, default=None,
                        help='Queue only bricks with the given "brickq" value [0 to 3]')

    parser.add_argument('--brickq-deps', action='store_true', default=False,
                        help='Queue bricks directly using qdo API, setting brickq dependencies')
    parser.add_argument('--queue', default='bricks',
                        help='With --brickq-deps, the QDO queue name to use')
    
    opt = parser.parse_args()

    survey = LegacySurveyData()
    if opt.bricks is not None:
        B = fits_table(opt.bricks)
        log('Read', len(B), 'from', opt.bricks)
    else:
        B = survey.get_bricks()

    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        log('Read', len(T), 'from', opt.ccds)
    else:
        T = survey.get_ccds()
        log(len(T), 'CCDs')
    T.index = np.arange(len(T))

    if opt.ignore_cuts == False:
        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
    print(len(T), 'CCDs remain')

    # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
    # plt.clf()
    # plt.hist(counts, counts.max()+1)
    # plt.savefig('bricks.png')
    # B.cut(I[counts >= 9])
    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # #plt.scatter(B.ra[I], B.dec[I], c=counts)
    # plt.savefig('bricks2.png')


    # DES Stripe82
    #rlo,rhi = 350.,360.
    # rlo,rhi = 300., 10.
    # dlo,dhi = -6., 4.
    # TINY bit
    #rlo,rhi = 350.,351.1
    #dlo,dhi = 0., 1.1

    # EDR+
    # 860 bricks
    # ~10,000 CCDs
    #rlo,rhi = 239,246
    #dlo,dhi =   5, 13

    # DR1
    #rlo,rhi = 0, 360
    # part 1
    #dlo,dhi = 25, 40
    # part 2
    #dlo,dhi = 20,25
    # part 3
    #dlo,dhi = 15,20
    # part 4
    #dlo,dhi = 10,15
    # part 5
    #dlo,dhi = 5,10
    # the rest
    #dlo,dhi = -11, 5
    #dlo,dhi = 15,25.5

    dlo,dhi = -25, 40
    rlo,rhi = 0, 360

    # Arjun says 3x3 coverage area is roughly
    # RA=240-252 DEC=6-12 (but not completely rectangular)

    # COSMOS
    #rlo,rhi = 148.9, 151.2
    #dlo,dhi = 0.9, 3.5

    # A nice well-behaved region (EDR2/3)
    # rlo,rhi = 243.6, 244.6
    # dlo,dhi = 8.1, 8.6

    # 56 bricks, ~725 CCDs
    #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
    # 240 bricks, ~3000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
    # 535 bricks, ~7000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))


    if opt.region in ['test1', 'test2', 'test3', 'test4']:
        nm = dict(test1='2446p115', # weird stuff around bright star
                  test2='1183p292', # faint sources around bright galaxy
                  test3='3503p005', # DES
                  test4='1163p277', # Pollux
                  )[opt.region]

        B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
        log('Cut to', len(B), 'bricks')
        log(B.ra, B.dec)
        dlo,dhi = -90,90
        rlo,rhi = 0, 360

    elif opt.region == 'edr':
        # EDR:
        # 535 bricks, ~7000 CCDs
        rlo,rhi = 240,245
        dlo,dhi =   5, 12

    elif opt.region == 'edrplus':
        rlo,rhi = 235,248
        dlo,dhi =   5, 15

    elif opt.region == 'edr-south':
        rlo,rhi = 240,245
        dlo,dhi =   5, 10

    elif opt.region == 'cosmos1':
        # 16 bricks in the core of the COSMOS field.
        rlo,rhi = 149.75, 150.75
        dlo,dhi = 1.6, 2.6

    elif opt.region == 'pristine':
        # Stream?
        rlo,rhi = 240,250
        dlo,dhi = 10,15

    elif opt.region == 'des':
        dlo, dhi = -6., 4.
        rlo, rhi = 317., 7.

        T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
        log('Cut to', len(T), 'CCDs with "CPDES82" in filename')

    elif opt.region == 'subdes':
        rlo,rhi = 320., 360.
        dlo,dhi = -1.25, 1.25

    elif opt.region == 'northwest':
        rlo,rhi = 240,360
        dlo,dhi = 20,40
    elif opt.region == 'north':
        rlo,rhi = 120,240
        dlo,dhi = 20,40
    elif opt.region == 'northeast':
        rlo,rhi = 0,120
        dlo,dhi = 20,40
    elif opt.region == 'southwest':
        rlo,rhi = 240,360
        dlo,dhi = -20,0
    elif opt.region == 'south':
        rlo,rhi = 120,240
        dlo,dhi = -20,0
    elif opt.region == 'southeast':
        rlo,rhi = 0,120
        dlo,dhi = -20,0
    elif opt.region == 'southsoutheast':
        rlo,rhi = 0,120
        dlo,dhi = -20,-10
    elif opt.region == 'midwest':
        rlo,rhi = 240,360
        dlo,dhi = 0,20
    elif opt.region == 'middle':
        rlo,rhi = 120,240
        dlo,dhi = 0,20
    elif opt.region == 'mideast':
        rlo,rhi = 0,120
        dlo,dhi = 0,20

    elif opt.region == 'grz':
        # Bricks with grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
        log('Cut to', len(B), 'bricks with grz coverage')

    elif opt.region == 'nogrz':
        # Bricks without grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
        log('Cut to', len(B), 'bricks withOUT grz coverage')
    elif opt.region == 'deep2':
        rlo,rhi = 250,260
        dlo,dhi = 30,35

    elif opt.region == 'deep2f3':
        rlo,rhi = 351.25, 353.75
        dlo,dhi = 0, 0.5

    elif opt.region == 'virgo':
        rlo,rhi = 185,190
        dlo,dhi =  10, 15

    elif opt.region == 'virgo2':
        rlo,rhi = 182,192
        dlo,dhi =   8, 18

    elif opt.region == 'lsb':
        rlo,rhi = 147.2, 147.8
        dlo,dhi = -0.4, 0.4

    elif opt.region == 'eboss-elg':
        # RA -45 to +45
        # Dec -5 to +7
        rlo,rhi = 315., 45.
        dlo,dhi = -5., 7.

    elif opt.region == 'eboss-ngc':
        # NGC ELGs
        # RA 115 to 175
        # Dec 15 to  30
        rlo,rhi = 115., 175.
        dlo,dhi =  15.,  30.

    elif opt.region == 'mzls':
        dlo,dhi = 30., 90.
    elif opt.region == 'dr4-bootes':
        # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched 
        #dlo,dhi = 34., 35.
        #rlo,rhi = 209.5, 210.5
        dlo,dhi = 33., 36.
        rlo,rhi = 216.5, 219.5

        
    if opt.mindec is not None:
        dlo = opt.mindec
    if opt.maxdec is not None:
        dhi = opt.maxdec

    if rlo < rhi:
        B.cut((B.ra >= rlo) * (B.ra <= rhi) *
              (B.dec >= dlo) * (B.dec <= dhi))
    else: # RA wrap
        B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) *
              (B.dec >= dlo) * (B.dec <= dhi))
    log(len(B), 'bricks in range')
    for name in B.get('brickname'):
        print(name)
    B.writeto('bricks-cut.fits')

    I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize)
    keep = np.zeros(len(B), bool)
    for i in I:
        keep[i] = True
    B.cut(keep)
    log('Cut to', len(B), 'bricks near CCDs')

    plt.clf()
    plt.plot(B.ra, B.dec, 'b.')
    plt.title('DR3 bricks')
    plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
    plt.savefig('bricks.png')

    if opt.brickq is not None:
        B.cut(B.brickq == opt.brickq)
        log('Cut to', len(B), 'with brickq =', opt.brickq)
    
    if opt.touching:
        keep = np.zeros(len(T), bool)
        for j in J:
            keep[j] = True
        T.cut(keep)
        log('Cut to', len(T), 'CCDs near bricks')

    # Aside -- how many near DR1=1 CCDs?
    if False:
        T2 = D.get_ccds()
        log(len(T2), 'CCDs')
        T2.cut(T2.dr1 == 1)
        log(len(T2), 'CCDs marked DR1=1')
        log(len(B), 'bricks in range')
        I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize)
        keep = np.zeros(len(B), bool)
        for i in I:
            keep[i] = True
        B2 = B[keep]
        log('Total of', len(B2), 'bricks near CCDs with DR1=1')
        for band in 'grz':
            Tb = T2[T2.filter == band]
            log(len(Tb), 'in filter', band)
            I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, survey.bricksize)
            good = np.zeros(len(B2), np.uint8)
            for i in I:
                good[i] = 1
            B2.set('has_' + band, good)

        B2.writeto('survey-bricks-in-dr1.fits')
        sys.exit(0)

    # sort by dec decreasing
    #B.cut(np.argsort(-B.dec))
    # RA increasing
    B.cut(np.argsort(B.ra))

    for b in B:
        if opt.check:
            fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue
        if opt.check_coadd:
            fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue

        print(b.brickname)

    if opt.save_to_fits:
        assert(opt.touching)
        # Write cut tables to file
        for tab,typ in zip([B,T],['bricks','ccds']):
            fn='%s-%s-cut.fits' % (typ,opt.name)
            if os.path.exists(fn):
                os.remove(fn)
            tab.writeto(fn)
            print('Wrote %s' % fn)
        # Write text files listing ccd and filename names
        nm1,nm2= 'ccds-%s.txt'% opt.name,'filenames-%s.txt' % opt.name
        if os.path.exists(nm1):
            os.remove(nm1)
        if os.path.exists(nm2):
            os.remove(nm2)
        f1,f2=open(nm1,'w'),open(nm2,'w')
        fns= list(set(T.get('image_filename')))
        for fn in fns:
            f2.write('%s\n' % fn.strip())
        for ti in T:
            f1.write('%s\n' % ti.get('image_filename').strip())
        f1.close()
        f2.close()
        print('Wrote *-names.txt')
    

    if opt.brickq_deps:
        import qdo
        from legacypipe.survey import on_bricks_dependencies

        #... find Queue...
        q = qdo.connect(opt.queue, create_ok=True)
        print('Connected to QDO queue', opt.queue, q)
        brick_to_task = dict()

        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
        print(len(T), 'CCDs remaining')

        T.wra = T.ra + (T.ra > 180) * -360
        wra = rlo - 360
        plt.clf()
        plt.plot(T.wra, T.dec, 'b.')
        ax = [wra, rhi, dlo, dhi]
        plt.axis(ax)
        plt.title('CCDs')
        plt.savefig('q-ccds.png')

        B.wra = B.ra + (B.ra > 180) * -360

        # this slight overestimate (for DECam images) is fine
        radius = 0.3
        Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius,
                            indexlist=True)
        ikeep = []
        for ib,(b,Iccd) in enumerate(zip(B, Iccds)):
            if Iccd is None or len(Iccd) == 0:
                print('No matched CCDs to brick', b.brickname)
                continue
            wcs = wcs_for_brick(b)
            cI = ccds_touching_wcs(wcs, T[np.array(Iccd)])
            print(len(cI), 'CCDs touching brick', b.brickname)
            if len(cI) == 0:
                continue
            ikeep.append(ib)
        B.cut(np.array(ikeep))
        print('Cut to', len(B), 'bricks touched by CCDs')
        
        for brickq in range(4):
            I = np.flatnonzero(B.brickq == brickq)
            print(len(I), 'bricks with brickq =', brickq)

            J = np.flatnonzero(B.brickq < brickq)
            preB = B[J]
            reqs = []
            if brickq > 0:
                for b in B[I]:
                    # find brick dependencies
                    brickdeps = on_bricks_dependencies(b, survey, bricks=preB)
                    # convert to task ids
                    taskdeps = [brick_to_task.get(b.brickname,None) for b in brickdeps]
                    # If we dropped a dependency brick from a previous brickq because
                    # of no overlapping CCDs, it won't appear in the brick_to_task map.
                    taskdeps = [t for t in taskdeps if t is not None]
                    reqs.append(taskdeps)

            plt.clf()
            plt.plot(B.wra, B.dec, '.', color='0.5')
            plt.plot(B.wra[I], B.dec[I], 'b.')
            plt.axis(ax)
            plt.title('Bricks: brickq=%i' % brickq)
            plt.savefig('q-bricks-%i.png' % brickq)
            
            # submit to qdo queue
            print('Queuing', len(B[I]), 'bricks')
            if brickq == 0:
                reqs = None
            else:
                assert(len(I) == len(reqs))
            taskids = q.add_multiple(B.brickname[I], requires=reqs)
            assert(len(taskids) == len(I))
            print('Queued', len(taskids), 'bricks')
            brick_to_task.update(dict(zip(B.brickname[I], taskids)))
        
    if not (opt.calibs or opt.forced or opt.lsb):
        sys.exit(0)

    bands = 'grz'
    log('Filters:', np.unique(T.filter))
    T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
    log('Cut to', len(T), 'CCDs in filters', bands)

    if opt.touching:
        allI = set()
        for b in B:
            wcs = wcs_for_brick(b)
            I = ccds_touching_wcs(wcs, T)
            log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
            if len(I) == 0:
                continue
            allI.update(I)
        allI = list(allI)
        allI.sort()
    elif opt.near:
        # Roughly brick radius + DECam image radius
        radius = 0.35
        allI,nil,nil = match_radec(T.ra, T.dec, B.ra, B.dec, radius, nearest=True)
    else:
        allI = np.arange(len(T))

    if opt.write_ccds:
        T[allI].writeto(opt.write_ccds)
        log('Wrote', opt.write_ccds)

    ## Be careful here -- T has been cut; we want to write out T.index.
    ## 'allI' contains indices into T.

    if opt.forced:
        log('Writing forced-photometry commands to', opt.out)
        f = open(opt.out,'w')
        log('Total of', len(allI), 'CCDs')
        for j,i in enumerate(allI):
            expstr = '%08i' % T.expnum[i]
            outfn = os.path.join('forced', expstr[:5], expstr,
                                 'decam-%s-%s-forced.fits' %
                                 (expstr, T.ccdname[i]))
            imgfn = os.path.join(survey.survey_dir, 'images',
                                 T.image_filename[i].strip())
            if (not os.path.exists(imgfn) and
                imgfn.endswith('.fz') and
                os.path.exists(imgfn[:-3])):
                imgfn = imgfn[:-3]

            #f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' %
            #        (imgfn, T.image_hdu[i], outfn))

            f.write('python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n' %
                    (T.expnum[i], T.ccdname[i], outfn))
            
        f.close()
        log('Wrote', opt.out)
        sys.exit(0)

    if opt.lsb:
        log('Writing LSB commands to', opt.out)
        f = open(opt.out,'w')
        log('Total of', len(allI), 'CCDs')
        for j,i in enumerate(allI):
            exp = T.expnum[i]
            ext = T.ccdname[i].strip()
            outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
            f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext))
        f.close()
        log('Wrote', opt.out)
        sys.exit(0)


    log('Writing calibs to', opt.out)
    f = open(opt.out,'w')
    log('Total of', len(allI), 'CCDs')

    batch = []

    def write_batch(f, batch, cmd):
        if cmd is None:
            cmd = ''
        f.write(cmd + ' '.join(batch) + '\n')

    cmd = None
    if opt.command:
        cmd = 'python legacypipe/run-calib.py '
        if opt.opt is not None:
            cmd += opt.opt + ' '
        
    for j,i in enumerate(allI):

        if opt.delete_sky or opt.delete_pvastrom:
            log(j+1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if opt.delete_sky and os.path.exists(im.skyfn):
                log('  deleting:', im.skyfn)
                os.unlink(im.skyfn)
            if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
                log('  deleting:', im.pvwcsfn)
                os.unlink(im.pvwcsfn)

        if opt.check:
            log(j+1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if not im.run_calibs(im, just_check=True):
                log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done')
                continue

        if opt.command:
            s = '%i-%s' % (T.expnum[i], T.ccdname[i])
            prefix = 'python legacypipe/run-calib.py ' + opt.opt
            #('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
            #     (T.expnum[i], T.ccdname[i]))
        else:
            s = '%i' % T.index[i]
            prefix = ''
            
        if j < 10:
            print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i],
                  'filename', T.image_filename[i])
            
        if not opt.nper:
            f.write(prefix + s + '\n')
        else:
            batch.append(s)
            if len(batch) >= opt.nper:
                write_batch(f, batch, cmd)
                batch = []

        if opt.check:
            f.flush()

    if len(batch):
        write_batch(f, batch, cmd)

    f.close()
    log('Wrote', opt.out)
    return 0
Пример #4
0
def make_pickle_file(pfn, derivs=False, agn=False):
    survey = LegacySurveyData()
    ccds = survey.get_ccds()

    brickname = '0364m042'

    bricks = survey.get_bricks()
    brick = bricks[bricks.brickname == brickname][0]
    print('Brick', brick)

    catfn = survey.find_file('tractor', brick=brickname)
    print('Reading catalog from', catfn)
    cat = fits_table(catfn)
    print(len(cat), 'catalog entries')
    cat.cut(cat.brick_primary)
    print(len(cat), 'brick primary')
    '''
    BRICKNAM     BRICKID BRICKQ    BRICKROW    BRICKCOL                      RA                     DEC
    0364m042      306043      3         343         145        36.4255910987483       -4.25000000000000

    RA1                     RA2                    DEC1                    DEC2
    36.3004172461752        36.5507649513213       -4.37500000000000       -4.12500000000000
    '''

    rlo, rhi = brick.ra1, brick.ra2
    dlo, dhi = brick.dec1, brick.dec2
    #rlo,rhi = 36.4, 36.5
    #dlo,dhi = -4.4, -4.3

    ra, dec = (rlo + rhi) / 2., (dlo + dhi) / 2.

    ## optional
    cat.cut(
        (cat.ra > rlo) * (cat.ra < rhi) * (cat.dec > dlo) * (cat.dec < dhi))
    print('Cut to', len(cat), 'catalog objects in RA,Dec box')

    lightcurves = dict([((brickname, oid), []) for oid in cat.objid])

    # close enough to equator to ignore cos(dec)
    dra = 4096 / 2. * 0.262 / 3600.
    ddec = 2048 / 2. * 0.262 / 3600.

    ccds.cut((np.abs(ccds.ra - ra) <
              (rhi - rlo) / 2. + dra) * (np.abs(ccds.dec - dec) <
                                         (dhi - dlo) / 2. + ddec))
    print('Cut to', len(ccds), 'CCDs overlapping brick')

    ### HACK
    #ccds = ccds[:50]

    for i, (expnum, ccdname) in enumerate(zip(ccds.expnum, ccds.ccdname)):
        ee = '%08i' % expnum

        flavor = 'vanilla'
        cols = [
            'brickname', 'objid', 'camera', 'expnum', 'ccdname', 'mjd',
            'filter', 'flux', 'flux_ivar'
        ]
        if derivs:
            flavor = 'derivs'
            cols.extend(
                ['flux_dra', 'flux_ddec', 'flux_dra_ivar', 'flux_ddec_ivar'])
        if agn:
            flavor = 'agn'
            cols.extend(['flux_agn', 'flux_agn_ivar'])

        fn = 'forced/%s/%s/%s/forced-decam-%s-%s.fits' % (flavor, ee[:5], ee,
                                                          ee, ccdname)
        if not os.path.exists(fn):
            print('WARNING: missing:', fn)
            continue
        T = fits_table(fn)
        print(i + 1, 'of', len(ccds), ':', len(T), 'in', fn)
        T.cut(T.brickname == brickname)
        print(len(T), 'in brick', brickname)
        found = 0
        for t in T:  #oid,expnum,ccdname,mjd,filter,flux,fluxiv in zip(T.objid, T.expnum, T.ccdname, T.mjd, T.filter, T.flux, T.flux_ivar):
            lc = lightcurves.get((t.brickname, t.objid), None)
            if lc is None:
                continue
            found += 1
            #lc.append((expnum, ccdname, mjd, filter, flux, fluxiv))
            lc.append([t.get(c) for c in cols])
        print('Matched', found, 'sources to light curves')

    #pickle_to_file(lightcurves, pfn)

    ll = {}
    for k, v in lightcurves.items():
        if len(v) == 0:
            continue
        T = fits_table()
        # T.expnum = np.array([vv[0] for vv in v])
        # T.ccdname= np.array([vv[1] for vv in v])
        # T.mjd    = np.array([vv[2] for vv in v])
        # T.filter = np.array([vv[3] for vv in v])
        # T.flux   = np.array([vv[4] for vv in v])
        # T.fluxiv = np.array([vv[5] for vv in v])
        for i, c in enumerate(cols):
            T.set(c, np.array([vv[i] for vv in v]))
        ll[k] = T
    pickle_to_file(ll, pfn)
Пример #5
0
                        '--outdir',
                        help='Set output base directory',
                        default='tractor2')
    parser.add_argument('--north',
                        help='Set Dec limits for Northern Cap surveys',
                        action='store_true')
    parser.add_argument(
        '--overwrite',
        action='store_true',
        default=False,
        help='Overwrite existing output files?  Default is to skip them.')
    opt = parser.parse_args()

    survey = LegacySurveyData(survey_dir=opt.survey_dir, output_dir=opt.outdir)

    bricks = survey.get_bricks()
    if opt.north:
        bricks.cut(bricks.dec > 30)
    else:
        bricks.cut(bricks.dec > -25)
        bricks.cut(bricks.dec < 40)

    ## HACK -- cut to COSMOS
    #bricks.cut((np.abs(bricks.ra - 150) < 2) *
    #           (np.abs(bricks.dec - 2.2) < 2))
    #print('Cut to', len(bricks), 'bricks near COSMOS')

    # Note to self: don't bother multiprocessing this; I/O bound
    for brick in bricks.brickname:
        add_depth_tag(survey, brick, overwrite=opt.overwrite)
Пример #6
0
def main():
    """Main program.
    """
    import argparse

    parser = argparse.ArgumentParser(
        description=
        "This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)."
    )
    parser.add_argument('--calibs',
                        action='store_true',
                        help='Output CCDs that need to be calibrated.')

    parser.add_argument('--nper',
                        type=int,
                        default=None,
                        help='Batch N calibs per line')

    parser.add_argument('--forced',
                        action='store_true',
                        help='Output forced-photometry commands')

    parser.add_argument('--lsb',
                        action='store_true',
                        help='Output Low-Surface-Brightness commands')

    parser.add_argument('--touching',
                        action='store_true',
                        help='Cut to only CCDs touching selected bricks')
    parser.add_argument('--near',
                        action='store_true',
                        help='Quick cut to only CCDs near selected bricks')

    parser.add_argument('--check',
                        action='store_true',
                        help='Check which calibrations actually need to run.')
    parser.add_argument('--check-coadd',
                        action='store_true',
                        help='Check which caoadds actually need to run.')
    parser.add_argument('--out',
                        help='Output filename for calibs, default %(default)s',
                        default='jobs')
    parser.add_argument('--command',
                        action='store_true',
                        help='Write out full command-line to run calib')
    parser.add_argument('--opt', help='With --command, extra options to add')

    parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
    parser.add_argument('--mindec', type=float, help='Minimum Dec to run')

    parser.add_argument('--region', help='Region to select')

    parser.add_argument('--bricks', help='Set bricks.fits file to load')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')
    parser.add_argument('--ignore_cuts',
                        action='store_true',
                        default=False,
                        help='no photometric or blacklist cuts')
    parser.add_argument('--save_to_fits',
                        action='store_true',
                        default=False,
                        help='save cut brick,ccd to fits table')
    parser.add_argument(
        '--name',
        action='store',
        default='dr3',
        help='save with this suffix, e.g. refers to ccds table')

    parser.add_argument('--delete-sky',
                        action='store_true',
                        help='Delete any existing sky calibration files')
    parser.add_argument('--delete-pvastrom',
                        action='store_true',
                        help='Delete any existing PV WCS calibration files')

    parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')

    parser.add_argument(
        '--brickq',
        type=int,
        default=None,
        help='Queue only bricks with the given "brickq" value [0 to 3]')

    parser.add_argument(
        '--brickq-deps',
        action='store_true',
        default=False,
        help='Queue bricks directly using qdo API, setting brickq dependencies'
    )
    parser.add_argument('--queue',
                        default='bricks',
                        help='With --brickq-deps, the QDO queue name to use')

    opt = parser.parse_args()

    survey = LegacySurveyData()
    if opt.bricks is not None:
        B = fits_table(opt.bricks)
        log('Read', len(B), 'from', opt.bricks)
    else:
        B = survey.get_bricks()

    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        log('Read', len(T), 'from', opt.ccds)
    else:
        T = survey.get_ccds()
        log(len(T), 'CCDs')
    T.index = np.arange(len(T))

    if opt.ignore_cuts == False:
        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
    print(len(T), 'CCDs remain')

    # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
    # plt.clf()
    # plt.hist(counts, counts.max()+1)
    # plt.savefig('bricks.png')
    # B.cut(I[counts >= 9])
    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # #plt.scatter(B.ra[I], B.dec[I], c=counts)
    # plt.savefig('bricks2.png')

    # DES Stripe82
    #rlo,rhi = 350.,360.
    # rlo,rhi = 300., 10.
    # dlo,dhi = -6., 4.
    # TINY bit
    #rlo,rhi = 350.,351.1
    #dlo,dhi = 0., 1.1

    # EDR+
    # 860 bricks
    # ~10,000 CCDs
    #rlo,rhi = 239,246
    #dlo,dhi =   5, 13

    # DR1
    #rlo,rhi = 0, 360
    # part 1
    #dlo,dhi = 25, 40
    # part 2
    #dlo,dhi = 20,25
    # part 3
    #dlo,dhi = 15,20
    # part 4
    #dlo,dhi = 10,15
    # part 5
    #dlo,dhi = 5,10
    # the rest
    #dlo,dhi = -11, 5
    #dlo,dhi = 15,25.5

    dlo, dhi = -25, 40
    rlo, rhi = 0, 360

    # Arjun says 3x3 coverage area is roughly
    # RA=240-252 DEC=6-12 (but not completely rectangular)

    # COSMOS
    #rlo,rhi = 148.9, 151.2
    #dlo,dhi = 0.9, 3.5

    # A nice well-behaved region (EDR2/3)
    # rlo,rhi = 243.6, 244.6
    # dlo,dhi = 8.1, 8.6

    # 56 bricks, ~725 CCDs
    #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
    # 240 bricks, ~3000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
    # 535 bricks, ~7000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))

    if opt.region in ['test1', 'test2', 'test3', 'test4']:
        nm = dict(
            test1='2446p115',  # weird stuff around bright star
            test2='1183p292',  # faint sources around bright galaxy
            test3='3503p005',  # DES
            test4='1163p277',  # Pollux
        )[opt.region]

        B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
        log('Cut to', len(B), 'bricks')
        log(B.ra, B.dec)
        dlo, dhi = -90, 90
        rlo, rhi = 0, 360

    elif opt.region == 'edr':
        # EDR:
        # 535 bricks, ~7000 CCDs
        rlo, rhi = 240, 245
        dlo, dhi = 5, 12

    elif opt.region == 'edrplus':
        rlo, rhi = 235, 248
        dlo, dhi = 5, 15

    elif opt.region == 'edr-south':
        rlo, rhi = 240, 245
        dlo, dhi = 5, 10

    elif opt.region == 'cosmos1':
        # 16 bricks in the core of the COSMOS field.
        rlo, rhi = 149.75, 150.75
        dlo, dhi = 1.6, 2.6

    elif opt.region == 'pristine':
        # Stream?
        rlo, rhi = 240, 250
        dlo, dhi = 10, 15

    elif opt.region == 'des':
        dlo, dhi = -6., 4.
        rlo, rhi = 317., 7.

        T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
        log('Cut to', len(T), 'CCDs with "CPDES82" in filename')

    elif opt.region == 'subdes':
        rlo, rhi = 320., 360.
        dlo, dhi = -1.25, 1.25

    elif opt.region == 'northwest':
        rlo, rhi = 240, 360
        dlo, dhi = 20, 40
    elif opt.region == 'north':
        rlo, rhi = 120, 240
        dlo, dhi = 20, 40
    elif opt.region == 'northeast':
        rlo, rhi = 0, 120
        dlo, dhi = 20, 40
    elif opt.region == 'southwest':
        rlo, rhi = 240, 360
        dlo, dhi = -20, 0
    elif opt.region == 'south':
        rlo, rhi = 120, 240
        dlo, dhi = -20, 0
    elif opt.region == 'southeast':
        rlo, rhi = 0, 120
        dlo, dhi = -20, 0
    elif opt.region == 'southsoutheast':
        rlo, rhi = 0, 120
        dlo, dhi = -20, -10
    elif opt.region == 'midwest':
        rlo, rhi = 240, 360
        dlo, dhi = 0, 20
    elif opt.region == 'middle':
        rlo, rhi = 120, 240
        dlo, dhi = 0, 20
    elif opt.region == 'mideast':
        rlo, rhi = 0, 120
        dlo, dhi = 0, 20

    elif opt.region == 'grz':
        # Bricks with grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
        log('Cut to', len(B), 'bricks with grz coverage')

    elif opt.region == 'nogrz':
        # Bricks without grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
        log('Cut to', len(B), 'bricks withOUT grz coverage')
    elif opt.region == 'deep2':
        rlo, rhi = 250, 260
        dlo, dhi = 30, 35

    elif opt.region == 'deep2f3':
        rlo, rhi = 351.25, 353.75
        dlo, dhi = 0, 0.5

    elif opt.region == 'virgo':
        rlo, rhi = 185, 190
        dlo, dhi = 10, 15

    elif opt.region == 'virgo2':
        rlo, rhi = 182, 192
        dlo, dhi = 8, 18

    elif opt.region == 'lsb':
        rlo, rhi = 147.2, 147.8
        dlo, dhi = -0.4, 0.4

    elif opt.region == 'eboss-sgc':
        # generous boundaries to make sure get all relevant images
        # RA -45 to +45
        # Dec -5 to +7
        rlo, rhi = 310., 50.
        dlo, dhi = -6., 6.

    elif opt.region == 'eboss-ngc':
        # generous boundaries to make sure get all relevant images
        # NGC ELGs
        # RA 115 to 175
        # Dec 15 to  30
        rlo, rhi = 122., 177.
        dlo, dhi = 12., 32.

    elif opt.region == 'mzls':
        dlo, dhi = 30., 90.
    elif opt.region == 'dr4-bootes':
        # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
        #dlo,dhi = 34., 35.
        #rlo,rhi = 209.5, 210.5
        dlo, dhi = 33., 36.
        rlo, rhi = 216.5, 219.5

    if opt.mindec is not None:
        dlo = opt.mindec
    if opt.maxdec is not None:
        dhi = opt.maxdec

    if rlo < rhi:
        B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi))
    else:  # RA wrap
        B.cut(
            np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) *
            (B.dec <= dhi))
    log(len(B), 'bricks in range')
    #for name in B.get('brickname'):
    #print(name)
    B.writeto('bricks-cut.fits')

    I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize)
    keep = np.zeros(len(B), bool)
    for i in I:
        keep[i] = True
    B.cut(keep)
    log('Cut to', len(B), 'bricks near CCDs')

    plt.clf()
    plt.plot(B.ra, B.dec, 'b.')
    plt.title('DR3 bricks')
    plt.axis([360, 0, np.min(B.dec) - 1, np.max(B.dec) + 1])
    plt.savefig('bricks.png')

    if opt.brickq is not None:
        B.cut(B.brickq == opt.brickq)
        log('Cut to', len(B), 'with brickq =', opt.brickq)

    if opt.touching:
        keep = np.zeros(len(T), bool)
        for j in J:
            keep[j] = True
        T.cut(keep)
        log('Cut to', len(T), 'CCDs near bricks')

    # Aside -- how many near DR1=1 CCDs?
    if False:
        T2 = D.get_ccds()
        log(len(T2), 'CCDs')
        T2.cut(T2.dr1 == 1)
        log(len(T2), 'CCDs marked DR1=1')
        log(len(B), 'bricks in range')
        I, J, d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize)
        keep = np.zeros(len(B), bool)
        for i in I:
            keep[i] = True
        B2 = B[keep]
        log('Total of', len(B2), 'bricks near CCDs with DR1=1')
        for band in 'grz':
            Tb = T2[T2.filter == band]
            log(len(Tb), 'in filter', band)
            I, J, d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec,
                                  survey.bricksize)
            good = np.zeros(len(B2), np.uint8)
            for i in I:
                good[i] = 1
            B2.set('has_' + band, good)

        B2.writeto('survey-bricks-in-dr1.fits')
        sys.exit(0)

    # sort by dec decreasing
    #B.cut(np.argsort(-B.dec))
    # RA increasing
    B.cut(np.argsort(B.ra))

    for b in B:
        if opt.check:
            fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3],
                                                      b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue
        if opt.check_coadd:
            fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (
                b.brickname[:3], b.brickname, b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue

        #print(b.brickname)

    if opt.save_to_fits:
        assert (opt.touching)
        # Write cut tables to file
        for tab, typ in zip([B, T], ['bricks', 'ccds']):
            fn = '%s-%s-cut.fits' % (typ, opt.region)
            if os.path.exists(fn):
                os.remove(fn)
            tab.writeto(fn)
            print('Wrote %s' % fn)
        # Write text files listing ccd and filename names
        nm1, nm2 = 'ccds-%s.txt' % opt.region, 'filenames-%s.txt' % opt.region
        if os.path.exists(nm1):
            os.remove(nm1)
        if os.path.exists(nm2):
            os.remove(nm2)
        f1, f2 = open(nm1, 'w'), open(nm2, 'w')
        fns = list(set(T.get('image_filename')))
        for fn in fns:
            f2.write('%s\n' % fn.strip())
        for ti in T:
            f1.write('%s\n' % ti.get('image_filename').strip())
        f1.close()
        f2.close()
        print('Wrote *-names.txt')

    if opt.brickq_deps:
        import qdo
        from legacypipe.survey import on_bricks_dependencies

        #... find Queue...
        q = qdo.connect(opt.queue, create_ok=True)
        print('Connected to QDO queue', opt.queue, q)
        brick_to_task = dict()

        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
        print(len(T), 'CCDs remaining')

        T.wra = T.ra + (T.ra > 180) * -360
        wra = rlo - 360
        plt.clf()
        plt.plot(T.wra, T.dec, 'b.')
        ax = [wra, rhi, dlo, dhi]
        plt.axis(ax)
        plt.title('CCDs')
        plt.savefig('q-ccds.png')

        B.wra = B.ra + (B.ra > 180) * -360

        # this slight overestimate (for DECam images) is fine
        radius = 0.3
        Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius, indexlist=True)
        ikeep = []
        for ib, (b, Iccd) in enumerate(zip(B, Iccds)):
            if Iccd is None or len(Iccd) == 0:
                print('No matched CCDs to brick', b.brickname)
                continue
            wcs = wcs_for_brick(b)
            cI = ccds_touching_wcs(wcs, T[np.array(Iccd)])
            print(len(cI), 'CCDs touching brick', b.brickname)
            if len(cI) == 0:
                continue
            ikeep.append(ib)
        B.cut(np.array(ikeep))
        print('Cut to', len(B), 'bricks touched by CCDs')

        for brickq in range(4):
            I = np.flatnonzero(B.brickq == brickq)
            print(len(I), 'bricks with brickq =', brickq)

            J = np.flatnonzero(B.brickq < brickq)
            preB = B[J]
            reqs = []
            if brickq > 0:
                for b in B[I]:
                    # find brick dependencies
                    brickdeps = on_bricks_dependencies(b, survey, bricks=preB)
                    # convert to task ids
                    taskdeps = [
                        brick_to_task.get(b.brickname, None) for b in brickdeps
                    ]
                    # If we dropped a dependency brick from a previous brickq because
                    # of no overlapping CCDs, it won't appear in the brick_to_task map.
                    taskdeps = [t for t in taskdeps if t is not None]
                    reqs.append(taskdeps)

            plt.clf()
            plt.plot(B.wra, B.dec, '.', color='0.5')
            plt.plot(B.wra[I], B.dec[I], 'b.')
            plt.axis(ax)
            plt.title('Bricks: brickq=%i' % brickq)
            plt.savefig('q-bricks-%i.png' % brickq)

            # submit to qdo queue
            print('Queuing', len(B[I]), 'bricks')
            if brickq == 0:
                reqs = None
            else:
                assert (len(I) == len(reqs))
            taskids = q.add_multiple(B.brickname[I], requires=reqs)
            assert (len(taskids) == len(I))
            print('Queued', len(taskids), 'bricks')
            brick_to_task.update(dict(zip(B.brickname[I], taskids)))

    if not (opt.calibs or opt.forced or opt.lsb):
        sys.exit(0)

    bands = 'grz'
    log('Filters:', np.unique(T.filter))
    T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
    log('Cut to', len(T), 'CCDs in filters', bands)

    if opt.touching:
        allI = set()
        for b in B:
            wcs = wcs_for_brick(b)
            I = ccds_touching_wcs(wcs, T)
            log(len(I), 'CCDs for brick', b.brickid,
                'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
            if len(I) == 0:
                continue
            allI.update(I)
        allI = list(allI)
        allI.sort()
    elif opt.near:
        # Roughly brick radius + DECam image radius
        radius = 0.35
        allI, nil, nil = match_radec(T.ra,
                                     T.dec,
                                     B.ra,
                                     B.dec,
                                     radius,
                                     nearest=True)
    else:
        allI = np.arange(len(T))

    if opt.write_ccds:
        T[allI].writeto(opt.write_ccds)
        log('Wrote', opt.write_ccds)

    ## Be careful here -- T has been cut; we want to write out T.index.
    ## 'allI' contains indices into T.

    if opt.forced:
        log('Writing forced-photometry commands to', opt.out)
        f = open(opt.out, 'w')
        log('Total of', len(allI), 'CCDs')
        for j, i in enumerate(allI):
            expstr = '%08i' % T.expnum[i]
            outfn = os.path.join(
                'forced', expstr[:5], expstr,
                'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i]))
            imgfn = os.path.join(survey.survey_dir, 'images',
                                 T.image_filename[i].strip())
            if (not os.path.exists(imgfn) and imgfn.endswith('.fz')
                    and os.path.exists(imgfn[:-3])):
                imgfn = imgfn[:-3]

            #f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' %
            #        (imgfn, T.image_hdu[i], outfn))

            f.write(
                'python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n'
                % (T.expnum[i], T.ccdname[i], outfn))

        f.close()
        log('Wrote', opt.out)
        sys.exit(0)

    if opt.lsb:
        log('Writing LSB commands to', opt.out)
        f = open(opt.out, 'w')
        log('Total of', len(allI), 'CCDs')
        for j, i in enumerate(allI):
            exp = T.expnum[i]
            ext = T.ccdname[i].strip()
            outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
            f.write(
                'python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n'
                % (exp, ext, outfn, exp, ext))
        f.close()
        log('Wrote', opt.out)
        sys.exit(0)

    log('Writing calibs to', opt.out)
    f = open(opt.out, 'w')
    log('Total of', len(allI), 'CCDs')

    batch = []

    def write_batch(f, batch, cmd):
        if cmd is None:
            cmd = ''
        f.write(cmd + ' '.join(batch) + '\n')

    cmd = None
    if opt.command:
        cmd = 'python legacypipe/run-calib.py '
        if opt.opt is not None:
            cmd += opt.opt + ' '

    for j, i in enumerate(allI):

        if opt.delete_sky or opt.delete_pvastrom:
            log(j + 1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if opt.delete_sky and os.path.exists(im.skyfn):
                log('  deleting:', im.skyfn)
                os.unlink(im.skyfn)
            if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
                log('  deleting:', im.pvwcsfn)
                os.unlink(im.pvwcsfn)

        if opt.check:
            log(j + 1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if not im.run_calibs(im, just_check=True):
                log('Calibs for', im.expnum, im.ccdname, im.calname,
                    'already done')
                continue

        if opt.command:
            s = '%i-%s' % (T.expnum[i], T.ccdname[i])
            prefix = 'python legacypipe/run-calib.py ' + opt.opt
            #('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
            #     (T.expnum[i], T.ccdname[i]))
        else:
            s = '%i' % T.index[i]
            prefix = ''

        if j < 10:
            print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname',
                  T.ccdname[i], 'filename', T.image_filename[i])

        if not opt.nper:
            f.write(prefix + s + '\n')
        else:
            batch.append(s)
            if len(batch) >= opt.nper:
                write_batch(f, batch, cmd)
                batch = []

        if opt.check:
            f.flush()

    if len(batch):
        write_batch(f, batch, cmd)

    f.close()
    log('Wrote', opt.out)
    return 0
Пример #7
0
    
    parser = argparse.ArgumentParser()
    parser.add_argument('--survey-dir', type=str, default=None,
                        help='Override the $LEGACY_SURVEY_DIR environment variable')
    parser.add_argument('-d', '--outdir', help='Set output base directory',
                        default='tractor2')
    parser.add_argument('--north', help='Set Dec limits for Northern Cap surveys',
                        action='store_true')
    parser.add_argument('--overwrite', action='store_true', default=False,
                        help='Overwrite existing output files?  Default is to skip them.')
    opt = parser.parse_args()

    survey = LegacySurveyData(survey_dir=opt.survey_dir,
                              output_dir=opt.outdir)

    bricks = survey.get_bricks()
    if opt.north:
        bricks.cut(bricks.dec > 30)
    else:
        bricks.cut(bricks.dec > -25)
        bricks.cut(bricks.dec <  40)

    ## HACK -- cut to COSMOS
    #bricks.cut((np.abs(bricks.ra - 150) < 2) *
    #           (np.abs(bricks.dec - 2.2) < 2))
    #print('Cut to', len(bricks), 'bricks near COSMOS')

    # Note to self: don't bother multiprocessing this; I/O bound
    for brick in bricks.brickname:
        add_depth_tag(survey, brick, overwrite=opt.overwrite)