コード例 #1
0
def main():

    parser = argparse.ArgumentParser()
    parser.add_argument('--build-sample',
                        action='store_true',
                        help='Build the sample.')
    parser.add_argument('--jpg-cutouts',
                        action='store_true',
                        help='Get jpg cutouts from the viewer.')
    parser.add_argument('--ccd-cutouts',
                        action='store_true',
                        help='Get CCD cutouts of each galaxy.')
    parser.add_argument('--runbrick',
                        action='store_true',
                        help='Run the pipeline.')
    parser.add_argument('--build-webpage',
                        action='store_true',
                        help='(Re)build the web content.')
    args = parser.parse_args()

    # Top-level directory
    key = 'LEGACY_SURVEY_LARGE_GALAXIES'
    if key not in os.environ:
        print('Required ${} environment variable not set'.format(key))
        return 0
    largedir = os.getenv(key)
    samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')

    # --------------------------------------------------
    # Build the sample of large galaxies based on the available imaging.
    if args.build_sample:

        # Read the parent catalog.
        cat = read_rc3()

        # Create a simple WCS object for each object and find all the CCDs
        # touching that WCS footprint.
        survey = LegacySurveyData(version='dr2')  # hack!
        allccds = survey.get_ccds()
        keep = np.concatenate((survey.apply_blacklist(allccds),
                               survey.photometric_ccds(allccds)))
        allccds.cut(keep)

        ccdlist = []
        outcat = []
        for gal in cat:
            galwcs = _simplewcs(gal)

            ccds1 = allccds[ccds_touching_wcs(galwcs, allccds)]
            ccds1 = ccds1[_uniqccds(ccds1)]

            if len(
                    ccds1
            ) > 0 and 'g' in ccds1.filter and 'r' in ccds1.filter and 'z' in ccds1.filter:
                print('Found {} CCDs for {}, D(25)={:.4f}'.format(
                    len(ccds1), gal['GALAXY'], gal['RADIUS']))

                ccdsfile = os.path.join(
                    largedir, 'ccds',
                    '{}-ccds.fits'.format(gal['GALAXY'].strip().lower()))
                print('  Writing {}'.format(ccdsfile))
                if os.path.isfile(ccdsfile):
                    os.remove(ccdsfile)
                ccds1.writeto(ccdsfile)

                ccdlist.append(ccds1)
                if len(outcat) == 0:
                    outcat = gal
                else:
                    outcat = vstack((outcat, gal))
                #if gal['GALAXY'] == 'MCG5-19-36':
                #    pdb.set_trace()

        # Write out the final catalog.
        samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')
        if os.path.isfile(samplefile):
            os.remove(samplefile)
        print('Writing {}'.format(samplefile))
        outcat.write(samplefile)
        print(outcat)

        # Do we need to transfer any of the data to nyx?
        _getfiles(merge_tables(ccdlist))

    # --------------------------------------------------
    # Get data, model, and residual cutouts from the legacysurvey viewer.  Also
    # get thumbnails that are lower resolution.
    if args.jpg_cutouts:
        thumbsize = 100
        sample = fits.getdata(samplefile, 1)
        for gal in sample:
            size = np.ceil(10 * gal['RADIUS'] / PIXSCALE)
            thumbpixscale = PIXSCALE * size / thumbsize

            #imageurl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
            #  '&pixscale={:.3f}&size={:g}'.format(PIXSCALE, size)
            #imagejpg = os.path.join(largedir, 'cutouts', gal['GALAXY'].strip().lower()+'-image.jpg')
            #if os.path.isfile(imagejpg):
            #    os.remove(imagejpg)
            #os.system('wget --continue -O {:s} "{:s}"' .format(imagejpg, imageurl))

            thumburl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
              '&pixscale={:.3f}&size={:g}'.format(thumbpixscale, thumbsize)
            thumbjpg = os.path.join(
                largedir, 'cutouts',
                gal['GALAXY'].strip().lower() + '-image-thumb.jpg')
            if os.path.isfile(thumbjpg):
                os.remove(thumbjpg)
            os.system('wget --continue -O {:s} "{:s}"'.format(
                thumbjpg, thumburl))

    # --------------------------------------------------
    # (Re)build the webpage.
    if args.build_webpage:

        # index.html
        html = open(os.path.join(largedir, 'index.html'), 'w')
        html.write('<html><body>\n')
        html.write('<h1>Sample of Large Galaxies</h1>\n')
        html.write('<table border="2" width="30%">\n')
        html.write('<tbody>\n')
        sample = fits.getdata(samplefile, 1)
        for gal in sample:
            # Add coordinates and sizes here.
            galaxy = gal['GALAXY'].strip().lower()
            html.write('<tr>\n')
            html.write('<td><a href="html/{}.html">{}</a></td>\n'.format(
                galaxy, galaxy.upper()))
            html.write(
                '<td><a href="http://legacysurvey.org/viewer/?ra={:.6f}&dec={:.6f}" target="_blank"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'
                .format(gal['RA'], gal['DEC'], galaxy, galaxy.upper()))
            #           html.write('<td><a href="html/{}.html"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'.format(galaxy, galaxy, galaxy.upper()))
            html.write('</tr>\n')
        html.write('</tbody>\n')
        html.write('</table>\n')
        html.write('</body></html>\n')
        html.close()

        sys.exit(1)

        # individual galaxy pages
        for gal in sample[:3]:
            galaxy = gal['GALAXY'].strip().lower()
            html = open(os.path.join(largedir, 'html/{}.html'.format(galaxy)),
                        'w')
            html.write('<html><body>\n')
            html.write(
                '<a href=../cutouts/{}.jpg><img src=../cutouts/{}-image.jpg alt={} /></a>\n'
                .format(galaxy, galaxy, galaxy, galaxy.upper()))
            html.write('</body></html>\n')
            html.close()

    # --------------------------------------------------
    # Get cutouts of all the CCDs for each galaxy.
    if args.ccd_cutouts:
        sample = fits.getdata(samplefile, 1)

        for gal in sample[1:2]:
            galaxy = gal['GALAXY'].strip().lower()
            ccdsfile = os.path.join(largedir, 'ccds',
                                    '{}-ccds.fits'.format(galaxy))
            ccds = fits.getdata(ccdsfile)

            pdb.set_trace()

    # --------------------------------------------------
    # Run the pipeline.
    if args.runbrick:
        sample = fits.getdata(samplefile, 1)

        for gal in sample[1:2]:
            galaxy = gal['GALAXY'].strip().lower()
            diam = 10 * np.ceil(gal['RADIUS'] / PIXSCALE).astype(
                'int16')  # [pixels]

            # Note: zoom is relative to the center of an imaginary brick with
            # dimensions (0, 3600, 0, 3600).
            survey = LegacySurveyData(version='dr2', output_dir=largedir)
            run_brick(None,
                      survey,
                      radec=(gal['RA'], gal['DEC']),
                      blobxy=zip([diam / 2], [diam / 2]),
                      threads=1,
                      zoom=(1800 - diam / 2, 1800 + diam / 2, 1800 - diam / 2,
                            1800 + diam / 2),
                      wise=False,
                      forceAll=True,
                      writePickles=False,
                      do_calibs=False,
                      write_metrics=False,
                      pixPsf=True,
                      splinesky=True,
                      early_coadds=True,
                      stages=['writecat'],
                      ceres=False)

            pdb.set_trace()
コード例 #2
0
def main():

    parser = argparse.ArgumentParser()
    parser.add_argument('--build-sample', action='store_true', help='Build the sample.')
    parser.add_argument('--jpg-cutouts', action='store_true', help='Get jpg cutouts from the viewer.')
    parser.add_argument('--ccd-cutouts', action='store_true', help='Get CCD cutouts of each galaxy.')
    parser.add_argument('--runbrick', action='store_true', help='Run the pipeline.')
    parser.add_argument('--build-webpage', action='store_true', help='(Re)build the web content.')
    args = parser.parse_args()

    # Top-level directory
    key = 'LEGACY_SURVEY_LARGE_GALAXIES'
    if key not in os.environ:
        print('Required ${} environment variable not set'.format(key))
        return 0
    largedir = os.getenv(key)
    samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')

    # --------------------------------------------------
    # Build the sample of large galaxies based on the available imaging.
    if args.build_sample:

        # Read the parent catalog.
        cat = read_rc3()
        
        # Create a simple WCS object for each object and find all the CCDs
        # touching that WCS footprint.
        survey = LegacySurveyData(version='dr2') # hack!
        allccds = survey.get_ccds()
        keep = np.concatenate((survey.apply_blacklist(allccds),
                               survey.photometric_ccds(allccds)))
        allccds.cut(keep)

        ccdlist = []
        outcat = []
        for gal in cat:
            galwcs = _simplewcs(gal)

            ccds1 = allccds[ccds_touching_wcs(galwcs, allccds)]
            ccds1 = ccds1[_uniqccds(ccds1)]
            
            if len(ccds1) > 0 and 'g' in ccds1.filter and 'r' in ccds1.filter and 'z' in ccds1.filter:
                print('Found {} CCDs for {}, D(25)={:.4f}'.format(
                    len(ccds1), gal['GALAXY'], gal['RADIUS']))
                
                ccdsfile = os.path.join(largedir, 'ccds', '{}-ccds.fits'.format(gal['GALAXY'].strip().lower()))
                print('  Writing {}'.format(ccdsfile))
                if os.path.isfile(ccdsfile):
                    os.remove(ccdsfile)
                ccds1.writeto(ccdsfile)
                
                ccdlist.append(ccds1)
                if len(outcat) == 0:
                    outcat = gal
                else:
                    outcat = vstack((outcat, gal))
                #if gal['GALAXY'] == 'MCG5-19-36':
                #    pdb.set_trace()

        # Write out the final catalog.
        samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')
        if os.path.isfile(samplefile):
            os.remove(samplefile)
        print('Writing {}'.format(samplefile))
        outcat.write(samplefile)
        print(outcat)

        # Do we need to transfer any of the data to nyx?
        _getfiles(merge_tables(ccdlist))

    # --------------------------------------------------
    # Get data, model, and residual cutouts from the legacysurvey viewer.  Also
    # get thumbnails that are lower resolution.
    if args.jpg_cutouts:
        thumbsize = 100
        sample = fits.getdata(samplefile, 1)
        for gal in sample:
            size = np.ceil(10*gal['RADIUS']/PIXSCALE)
            thumbpixscale = PIXSCALE*size/thumbsize

            #imageurl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
            #  '&pixscale={:.3f}&size={:g}'.format(PIXSCALE, size)
            #imagejpg = os.path.join(largedir, 'cutouts', gal['GALAXY'].strip().lower()+'-image.jpg')
            #if os.path.isfile(imagejpg):
            #    os.remove(imagejpg)
            #os.system('wget --continue -O {:s} "{:s}"' .format(imagejpg, imageurl))

            thumburl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
              '&pixscale={:.3f}&size={:g}'.format(thumbpixscale, thumbsize)
            thumbjpg = os.path.join(largedir, 'cutouts', gal['GALAXY'].strip().lower()+'-image-thumb.jpg')
            if os.path.isfile(thumbjpg):
                os.remove(thumbjpg)
            os.system('wget --continue -O {:s} "{:s}"' .format(thumbjpg, thumburl))

    # --------------------------------------------------
    # (Re)build the webpage.
    if args.build_webpage:

        # index.html
        html = open(os.path.join(largedir, 'index.html'), 'w')
        html.write('<html><body>\n')
        html.write('<h1>Sample of Large Galaxies</h1>\n')
        html.write('<table border="2" width="30%">\n')
        html.write('<tbody>\n')
        sample = fits.getdata(samplefile, 1)
        for gal in sample:
            # Add coordinates and sizes here.
            galaxy = gal['GALAXY'].strip().lower()
            html.write('<tr>\n')
            html.write('<td><a href="html/{}.html">{}</a></td>\n'.format(galaxy, galaxy.upper()))
            html.write('<td><a href="http://legacysurvey.org/viewer/?ra={:.6f}&dec={:.6f}" target="_blank"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'.format(gal['RA'], gal['DEC'], galaxy, galaxy.upper()))
#           html.write('<td><a href="html/{}.html"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'.format(galaxy, galaxy, galaxy.upper()))
            html.write('</tr>\n')
        html.write('</tbody>\n')
        html.write('</table>\n')
        html.write('</body></html>\n')
        html.close()

        sys.exit(1)
    
        # individual galaxy pages
        for gal in sample[:3]:
            galaxy = gal['GALAXY'].strip().lower()
            html = open(os.path.join(largedir, 'html/{}.html'.format(galaxy)), 'w')
            html.write('<html><body>\n')
            html.write('<a href=../cutouts/{}.jpg><img src=../cutouts/{}-image.jpg alt={} /></a>\n'.format(galaxy, galaxy, galaxy, galaxy.upper()))
            html.write('</body></html>\n')
            html.close()

    # --------------------------------------------------
    # Get cutouts of all the CCDs for each galaxy.
    if args.ccd_cutouts:
        sample = fits.getdata(samplefile, 1)

        for gal in sample[1:2]:
            galaxy = gal['GALAXY'].strip().lower()
            ccdsfile = os.path.join(largedir, 'ccds', '{}-ccds.fits'.format(galaxy))
            ccds = fits.getdata(ccdsfile)

            pdb.set_trace()

    # --------------------------------------------------
    # Run the pipeline.
    if args.runbrick:
        sample = fits.getdata(samplefile, 1)

        for gal in sample[1:2]:
            galaxy = gal['GALAXY'].strip().lower()
            diam = 10*np.ceil(gal['RADIUS']/PIXSCALE).astype('int16') # [pixels]

            # Note: zoom is relative to the center of an imaginary brick with
            # dimensions (0, 3600, 0, 3600).
            survey = LegacySurveyData(version='dr2', output_dir=largedir)
            run_brick(None, survey, radec=(gal['RA'], gal['DEC']), blobxy=zip([diam/2], [diam/2]), 
                      threads=1, zoom=(1800-diam/2, 1800+diam/2, 1800-diam/2, 1800+diam/2),
                      wise=False, forceAll=True, writePickles=False, do_calibs=False,
                      write_metrics=False, pixPsf=True, splinesky=True, 
                      early_coadds=True, stages=['writecat'], ceres=False)

            pdb.set_trace()
コード例 #3
0
def main(outfn='ccds-annotated.fits', ccds=None):
    survey = LegacySurveyData(ccds=ccds)
    if ccds is None:
        ccds = survey.get_ccds()

    # File from the "observing" svn repo:
    # https://desi.lbl.gov/svn/decam/code/observing/trunk
    tiles = fits_table('decam-tiles_obstatus.fits')

    I = survey.photometric_ccds(ccds)
    ccds.photometric = np.zeros(len(ccds), bool)
    ccds.photometric[I] = True

    I = survey.apply_blacklist(ccds)
    ccds.blacklist_ok = np.zeros(len(ccds), bool)
    ccds.blacklist_ok[I] = True

    ccds.good_region = np.empty((len(ccds), 4), np.int16)
    ccds.good_region[:,:] = -1

    ccds.ra0  = np.zeros(len(ccds), np.float64)
    ccds.dec0 = np.zeros(len(ccds), np.float64)
    ccds.ra1  = np.zeros(len(ccds), np.float64)
    ccds.dec1 = np.zeros(len(ccds), np.float64)
    ccds.ra2  = np.zeros(len(ccds), np.float64)
    ccds.dec2 = np.zeros(len(ccds), np.float64)
    ccds.ra3  = np.zeros(len(ccds), np.float64)
    ccds.dec3 = np.zeros(len(ccds), np.float64)

    ccds.dra  = np.zeros(len(ccds), np.float32)
    ccds.ddec = np.zeros(len(ccds), np.float32)
    ccds.ra_center  = np.zeros(len(ccds), np.float64)
    ccds.dec_center = np.zeros(len(ccds), np.float64)

    ccds.sig1 = np.zeros(len(ccds), np.float32)

    ccds.meansky = np.zeros(len(ccds), np.float32)
    ccds.stdsky  = np.zeros(len(ccds), np.float32)
    ccds.maxsky  = np.zeros(len(ccds), np.float32)
    ccds.minsky  = np.zeros(len(ccds), np.float32)

    ccds.pixscale_mean = np.zeros(len(ccds), np.float32)
    ccds.pixscale_std  = np.zeros(len(ccds), np.float32)
    ccds.pixscale_max  = np.zeros(len(ccds), np.float32)
    ccds.pixscale_min  = np.zeros(len(ccds), np.float32)

    ccds.psfnorm_mean = np.zeros(len(ccds), np.float32)
    ccds.psfnorm_std  = np.zeros(len(ccds), np.float32)
    ccds.galnorm_mean = np.zeros(len(ccds), np.float32)
    ccds.galnorm_std  = np.zeros(len(ccds), np.float32)

    gaussgalnorm = np.zeros(len(ccds), np.float32)

    # 2nd moments
    ccds.psf_mx2 = np.zeros(len(ccds), np.float32)
    ccds.psf_my2 = np.zeros(len(ccds), np.float32)
    ccds.psf_mxy = np.zeros(len(ccds), np.float32)
    #
    ccds.psf_a = np.zeros(len(ccds), np.float32)
    ccds.psf_b = np.zeros(len(ccds), np.float32)
    ccds.psf_theta = np.zeros(len(ccds), np.float32)
    ccds.psf_ell   = np.zeros(len(ccds), np.float32)

    ccds.humidity = np.zeros(len(ccds), np.float32)
    ccds.outtemp  = np.zeros(len(ccds), np.float32)

    ccds.tileid   = np.zeros(len(ccds), np.int32)
    ccds.tilepass = np.zeros(len(ccds), np.uint8)
    ccds.tileebv  = np.zeros(len(ccds), np.float32)

    plvers = []

    for iccd,ccd in enumerate(ccds):
        im = survey.get_image_object(ccd)
        print('Reading CCD %i of %i:' % (iccd+1, len(ccds)), im)

        X = im.get_good_image_subregion()
        for i,x in enumerate(X):
            if x is not None:
                ccds.good_region[iccd,i] = x

        W,H = ccd.width, ccd.height

        psf = None
        wcs = None
        sky = None
        try:
            tim = im.get_tractor_image(pixPsf=True, splinesky=True, subsky=False,
                                       pixels=False, dq=False, invvar=False)

        except:
            import traceback
            traceback.print_exc()
            plvers.append('')
            continue

        if tim is None:
            plvers.append('')
            continue

        psf = tim.psf
        wcs = tim.wcs.wcs
        sky = tim.sky
        hdr = tim.primhdr

        # print('Got PSF', psf)
        # print('Got sky', type(sky))
        # print('Got WCS', wcs)

        ccds.humidity[iccd] = hdr.get('HUMIDITY')
        ccds.outtemp[iccd]  = hdr.get('OUTTEMP')

        ccds.sig1[iccd] = tim.sig1
        plvers.append(tim.plver)

        # parse 'DECaLS_15150_r' to get tile number
        obj = ccd.object.strip()
        words = obj.split('_')
        tile = None
        if len(words) == 3 and words[0] == 'DECaLS':
            try:
                tileid = int(words[1])
                tile = tiles[tileid - 1]
                if tile.tileid != tileid:
                    I = np.flatnonzero(tile.tileid == tileid)
                    tile = tiles[I[0]]
            except:
                pass

        if tile is not None:
            ccds.tileid  [iccd] = tile.tileid
            ccds.tilepass[iccd] = tile.get('pass')
            ccds.tileebv [iccd] = tile.ebv_med

        # Instantiate PSF on a grid
        S = 32
        xx = np.linspace(1+S, W-S, 5)
        yy = np.linspace(1+S, H-S, 5)
        xx,yy = np.meshgrid(xx, yy)
        psfnorms = []
        galnorms = []
        for x,y in zip(xx.ravel(), yy.ravel()):
            p = im.psf_norm(tim, x=x, y=y)
            g = im.galaxy_norm(tim, x=x, y=y)
            psfnorms.append(p)
            galnorms.append(g)
        ccds.psfnorm_mean[iccd] = np.mean(psfnorms)
        ccds.psfnorm_std [iccd] = np.std (psfnorms)
        ccds.galnorm_mean[iccd] = np.mean(galnorms)
        ccds.galnorm_std [iccd] = np.std (galnorms)

        # PSF in center of field
        cx,cy = (W+1)/2., (H+1)/2.
        p = psf.getPointSourcePatch(cx, cy).patch
        ph,pw = p.shape
        px,py = np.meshgrid(np.arange(pw), np.arange(ph))
        psum = np.sum(p)
        # print('psum', psum)
        p /= psum
        # centroids
        cenx = np.sum(p * px)
        ceny = np.sum(p * py)
        # print('cenx,ceny', cenx,ceny)
        # second moments
        x2 = np.sum(p * (px - cenx)**2)
        y2 = np.sum(p * (py - ceny)**2)
        xy = np.sum(p * (px - cenx)*(py - ceny))
        # semi-major/minor axes and position angle
        theta = np.rad2deg(np.arctan2(2 * xy, x2 - y2) / 2.)
        theta = np.abs(theta) * np.sign(xy)
        s = np.sqrt(((x2 - y2)/2.)**2 + xy**2)
        a = np.sqrt((x2 + y2) / 2. + s)
        b = np.sqrt((x2 + y2) / 2. - s)
        ell = 1. - b/a

        # print('PSF second moments', x2, y2, xy)
        # print('PSF position angle', theta)
        # print('PSF semi-axes', a, b)
        # print('PSF ellipticity', ell)

        ccds.psf_mx2[iccd] = x2
        ccds.psf_my2[iccd] = y2
        ccds.psf_mxy[iccd] = xy
        ccds.psf_a[iccd] = a
        ccds.psf_b[iccd] = b
        ccds.psf_theta[iccd] = theta
        ccds.psf_ell  [iccd] = ell

        print('Computing Gaussian approximate PSF quantities...')
        # Galaxy norm using Gaussian approximation of PSF.
        realpsf = tim.psf
        tim.psf = im.read_psf_model(0, 0, gaussPsf=True,
                                    psf_sigma=tim.psf_sigma)
        gaussgalnorm[iccd] = im.galaxy_norm(tim, x=cx, y=cy)
        tim.psf = realpsf
        
        # Sky -- evaluate on a grid (every ~10th pixel)
        skygrid = sky.evaluateGrid(np.linspace(0, ccd.width-1,  int(1+ccd.width/10)),
                                   np.linspace(0, ccd.height-1, int(1+ccd.height/10)))
        ccds.meansky[iccd] = np.mean(skygrid)
        ccds.stdsky[iccd]  = np.std(skygrid)
        ccds.maxsky[iccd]  = skygrid.max()
        ccds.minsky[iccd]  = skygrid.min()

        # WCS
        ccds.ra0[iccd],ccds.dec0[iccd] = wcs.pixelxy2radec(1, 1)
        ccds.ra1[iccd],ccds.dec1[iccd] = wcs.pixelxy2radec(1, H)
        ccds.ra2[iccd],ccds.dec2[iccd] = wcs.pixelxy2radec(W, H)
        ccds.ra3[iccd],ccds.dec3[iccd] = wcs.pixelxy2radec(W, 1)

        midx, midy = (W+1)/2., (H+1)/2.
        rc,dc  = wcs.pixelxy2radec(midx, midy)
        ra,dec = wcs.pixelxy2radec([1,W,midx,midx], [midy,midy,1,H])
        ccds.dra [iccd] = max(degrees_between(ra, dc+np.zeros_like(ra),
                                              rc, dc))
        ccds.ddec[iccd] = max(degrees_between(rc+np.zeros_like(dec), dec,
                                              rc, dc))
        ccds.ra_center [iccd] = rc
        ccds.dec_center[iccd] = dc

        # Compute scale change across the chip
        # how many pixels to step
        step = 10
        xx = np.linspace(1+step, W-step, 5)
        yy = np.linspace(1+step, H-step, 5)
        xx,yy = np.meshgrid(xx, yy)
        pixscale = []
        for x,y in zip(xx.ravel(), yy.ravel()):
            sx = [x-step, x-step, x+step, x+step, x-step]
            sy = [y-step, y+step, y+step, y-step, y-step]
            sr,sd = wcs.pixelxy2radec(sx, sy)
            rc,dc = wcs.pixelxy2radec(x, y)
            # project around a tiny little TAN WCS at (x,y), with 1" pixels
            locwcs = Tan(rc, dc, 0., 0., 1./3600, 0., 0., 1./3600, 1., 1.)
            ok,lx,ly = locwcs.radec2pixelxy(sr, sd)
            #print('local x,y:', lx, ly)
            A = polygon_area((lx, ly))
            pixscale.append(np.sqrt(A / (2*step)**2))
        # print('Pixel scales:', pixscale)
        ccds.pixscale_mean[iccd] = np.mean(pixscale)
        ccds.pixscale_min[iccd] = min(pixscale)
        ccds.pixscale_max[iccd] = max(pixscale)
        ccds.pixscale_std[iccd] = np.std(pixscale)

    ccds.plver = np.array(plvers)

    sfd = tractor.sfd.SFDMap()
    allbands = 'ugrizY'
    filts = ['%s %s' % ('DES', f) for f in allbands]
    wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4']
    ebv,ext = sfd.extinction(filts + wisebands, ccds.ra_center,
                             ccds.dec_center, get_ebv=True)
    ext = ext.astype(np.float32)
    ccds.ebv = ebv.astype(np.float32)
    ccds.decam_extinction = ext[:,:len(allbands)]
    ccds.wise_extinction = ext[:,len(allbands):]

    # Depth
    detsig1 = ccds.sig1 / ccds.psfnorm_mean
    depth = 5. * detsig1
    # that's flux in nanomaggies -- convert to mag
    ccds.psfdepth = -2.5 * (np.log10(depth) - 9)

    detsig1 = ccds.sig1 / ccds.galnorm_mean
    depth = 5. * detsig1
    # that's flux in nanomaggies -- convert to mag
    ccds.galdepth = -2.5 * (np.log10(depth) - 9)
    
    # Depth using Gaussian FWHM.
    psf_sigma = ccds.fwhm / 2.35
    gnorm = 1./(2. * np.sqrt(np.pi) * psf_sigma)
    detsig1 = ccds.sig1 / gnorm
    depth = 5. * detsig1
    # that's flux in nanomaggies -- convert to mag
    ccds.gausspsfdepth = -2.5 * (np.log10(depth) - 9)

    # Gaussian galaxy depth
    detsig1 = ccds.sig1 / gaussgalnorm
    depth = 5. * detsig1
    # that's flux in nanomaggies -- convert to mag
    ccds.gaussgaldepth = -2.5 * (np.log10(depth) - 9)

    ccds.writeto(outfn)
コード例 #4
0
def main():
    """Main program.
    """
    import argparse

    parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).")
    parser.add_argument('--calibs', action='store_true',
                      help='Output CCDs that need to be calibrated.')

    parser.add_argument('--nper', type=int, default=None,
                      help='Batch N calibs per line')

    parser.add_argument('--forced', action='store_true',
                      help='Output forced-photometry commands')

    parser.add_argument('--lsb', action='store_true',
                      help='Output Low-Surface-Brightness commands')

    parser.add_argument('--touching', action='store_true',
                      help='Cut to only CCDs touching selected bricks')
    parser.add_argument('--near', action='store_true',
                      help='Quick cut to only CCDs near selected bricks')

    parser.add_argument('--check', action='store_true',
                      help='Check which calibrations actually need to run.')
    parser.add_argument('--check-coadd', action='store_true',
                      help='Check which caoadds actually need to run.')
    parser.add_argument('--out', help='Output filename for calibs, default %(default)s',
                      default='jobs')
    parser.add_argument('--command', action='store_true',
                      help='Write out full command-line to run calib')
    parser.add_argument('--opt', help='With --command, extra options to add')
    
    parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
    parser.add_argument('--mindec', type=float, help='Minimum Dec to run')

    parser.add_argument('--region', help='Region to select')

    parser.add_argument('--bricks', help='Set bricks.fits file to load')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')
    parser.add_argument('--ignore_cuts', action='store_true',default=False,help='no photometric or blacklist cuts')
    parser.add_argument('--save_to_fits', action='store_true',default=False,help='save cut brick,ccd to fits table')
    parser.add_argument('--name', action='store',default='dr3',help='save with this suffix, e.g. refers to ccds table')

    parser.add_argument('--delete-sky', action='store_true',
                      help='Delete any existing sky calibration files')
    parser.add_argument('--delete-pvastrom', action='store_true',
                      help='Delete any existing PV WCS calibration files')

    parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')

    parser.add_argument('--brickq', type=int, default=None,
                        help='Queue only bricks with the given "brickq" value [0 to 3]')

    parser.add_argument('--brickq-deps', action='store_true', default=False,
                        help='Queue bricks directly using qdo API, setting brickq dependencies')
    parser.add_argument('--queue', default='bricks',
                        help='With --brickq-deps, the QDO queue name to use')
    
    opt = parser.parse_args()

    survey = LegacySurveyData()
    if opt.bricks is not None:
        B = fits_table(opt.bricks)
        log('Read', len(B), 'from', opt.bricks)
    else:
        B = survey.get_bricks()

    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        log('Read', len(T), 'from', opt.ccds)
    else:
        T = survey.get_ccds()
        log(len(T), 'CCDs')
    T.index = np.arange(len(T))

    if opt.ignore_cuts == False:
        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
    print(len(T), 'CCDs remain')

    # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
    # plt.clf()
    # plt.hist(counts, counts.max()+1)
    # plt.savefig('bricks.png')
    # B.cut(I[counts >= 9])
    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # #plt.scatter(B.ra[I], B.dec[I], c=counts)
    # plt.savefig('bricks2.png')


    # DES Stripe82
    #rlo,rhi = 350.,360.
    # rlo,rhi = 300., 10.
    # dlo,dhi = -6., 4.
    # TINY bit
    #rlo,rhi = 350.,351.1
    #dlo,dhi = 0., 1.1

    # EDR+
    # 860 bricks
    # ~10,000 CCDs
    #rlo,rhi = 239,246
    #dlo,dhi =   5, 13

    # DR1
    #rlo,rhi = 0, 360
    # part 1
    #dlo,dhi = 25, 40
    # part 2
    #dlo,dhi = 20,25
    # part 3
    #dlo,dhi = 15,20
    # part 4
    #dlo,dhi = 10,15
    # part 5
    #dlo,dhi = 5,10
    # the rest
    #dlo,dhi = -11, 5
    #dlo,dhi = 15,25.5

    dlo,dhi = -25, 40
    rlo,rhi = 0, 360

    # Arjun says 3x3 coverage area is roughly
    # RA=240-252 DEC=6-12 (but not completely rectangular)

    # COSMOS
    #rlo,rhi = 148.9, 151.2
    #dlo,dhi = 0.9, 3.5

    # A nice well-behaved region (EDR2/3)
    # rlo,rhi = 243.6, 244.6
    # dlo,dhi = 8.1, 8.6

    # 56 bricks, ~725 CCDs
    #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
    # 240 bricks, ~3000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
    # 535 bricks, ~7000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))


    if opt.region in ['test1', 'test2', 'test3', 'test4']:
        nm = dict(test1='2446p115', # weird stuff around bright star
                  test2='1183p292', # faint sources around bright galaxy
                  test3='3503p005', # DES
                  test4='1163p277', # Pollux
                  )[opt.region]

        B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
        log('Cut to', len(B), 'bricks')
        log(B.ra, B.dec)
        dlo,dhi = -90,90
        rlo,rhi = 0, 360

    elif opt.region == 'edr':
        # EDR:
        # 535 bricks, ~7000 CCDs
        rlo,rhi = 240,245
        dlo,dhi =   5, 12

    elif opt.region == 'edrplus':
        rlo,rhi = 235,248
        dlo,dhi =   5, 15

    elif opt.region == 'edr-south':
        rlo,rhi = 240,245
        dlo,dhi =   5, 10

    elif opt.region == 'cosmos1':
        # 16 bricks in the core of the COSMOS field.
        rlo,rhi = 149.75, 150.75
        dlo,dhi = 1.6, 2.6

    elif opt.region == 'pristine':
        # Stream?
        rlo,rhi = 240,250
        dlo,dhi = 10,15

    elif opt.region == 'des':
        dlo, dhi = -6., 4.
        rlo, rhi = 317., 7.

        T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
        log('Cut to', len(T), 'CCDs with "CPDES82" in filename')

    elif opt.region == 'subdes':
        rlo,rhi = 320., 360.
        dlo,dhi = -1.25, 1.25

    elif opt.region == 'northwest':
        rlo,rhi = 240,360
        dlo,dhi = 20,40
    elif opt.region == 'north':
        rlo,rhi = 120,240
        dlo,dhi = 20,40
    elif opt.region == 'northeast':
        rlo,rhi = 0,120
        dlo,dhi = 20,40
    elif opt.region == 'southwest':
        rlo,rhi = 240,360
        dlo,dhi = -20,0
    elif opt.region == 'south':
        rlo,rhi = 120,240
        dlo,dhi = -20,0
    elif opt.region == 'southeast':
        rlo,rhi = 0,120
        dlo,dhi = -20,0
    elif opt.region == 'southsoutheast':
        rlo,rhi = 0,120
        dlo,dhi = -20,-10
    elif opt.region == 'midwest':
        rlo,rhi = 240,360
        dlo,dhi = 0,20
    elif opt.region == 'middle':
        rlo,rhi = 120,240
        dlo,dhi = 0,20
    elif opt.region == 'mideast':
        rlo,rhi = 0,120
        dlo,dhi = 0,20

    elif opt.region == 'grz':
        # Bricks with grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
        log('Cut to', len(B), 'bricks with grz coverage')

    elif opt.region == 'nogrz':
        # Bricks without grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
        log('Cut to', len(B), 'bricks withOUT grz coverage')
    elif opt.region == 'deep2':
        rlo,rhi = 250,260
        dlo,dhi = 30,35

    elif opt.region == 'deep2f3':
        rlo,rhi = 351.25, 353.75
        dlo,dhi = 0, 0.5

    elif opt.region == 'virgo':
        rlo,rhi = 185,190
        dlo,dhi =  10, 15

    elif opt.region == 'virgo2':
        rlo,rhi = 182,192
        dlo,dhi =   8, 18

    elif opt.region == 'lsb':
        rlo,rhi = 147.2, 147.8
        dlo,dhi = -0.4, 0.4

    elif opt.region == 'eboss-elg':
        # RA -45 to +45
        # Dec -5 to +7
        rlo,rhi = 315., 45.
        dlo,dhi = -5., 7.

    elif opt.region == 'eboss-ngc':
        # NGC ELGs
        # RA 115 to 175
        # Dec 15 to  30
        rlo,rhi = 115., 175.
        dlo,dhi =  15.,  30.

    elif opt.region == 'mzls':
        dlo,dhi = 30., 90.
    elif opt.region == 'dr4-bootes':
        # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched 
        #dlo,dhi = 34., 35.
        #rlo,rhi = 209.5, 210.5
        dlo,dhi = 33., 36.
        rlo,rhi = 216.5, 219.5

        
    if opt.mindec is not None:
        dlo = opt.mindec
    if opt.maxdec is not None:
        dhi = opt.maxdec

    if rlo < rhi:
        B.cut((B.ra >= rlo) * (B.ra <= rhi) *
              (B.dec >= dlo) * (B.dec <= dhi))
    else: # RA wrap
        B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) *
              (B.dec >= dlo) * (B.dec <= dhi))
    log(len(B), 'bricks in range')
    for name in B.get('brickname'):
        print(name)
    B.writeto('bricks-cut.fits')

    I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize)
    keep = np.zeros(len(B), bool)
    for i in I:
        keep[i] = True
    B.cut(keep)
    log('Cut to', len(B), 'bricks near CCDs')

    plt.clf()
    plt.plot(B.ra, B.dec, 'b.')
    plt.title('DR3 bricks')
    plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
    plt.savefig('bricks.png')

    if opt.brickq is not None:
        B.cut(B.brickq == opt.brickq)
        log('Cut to', len(B), 'with brickq =', opt.brickq)
    
    if opt.touching:
        keep = np.zeros(len(T), bool)
        for j in J:
            keep[j] = True
        T.cut(keep)
        log('Cut to', len(T), 'CCDs near bricks')

    # Aside -- how many near DR1=1 CCDs?
    if False:
        T2 = D.get_ccds()
        log(len(T2), 'CCDs')
        T2.cut(T2.dr1 == 1)
        log(len(T2), 'CCDs marked DR1=1')
        log(len(B), 'bricks in range')
        I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize)
        keep = np.zeros(len(B), bool)
        for i in I:
            keep[i] = True
        B2 = B[keep]
        log('Total of', len(B2), 'bricks near CCDs with DR1=1')
        for band in 'grz':
            Tb = T2[T2.filter == band]
            log(len(Tb), 'in filter', band)
            I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, survey.bricksize)
            good = np.zeros(len(B2), np.uint8)
            for i in I:
                good[i] = 1
            B2.set('has_' + band, good)

        B2.writeto('survey-bricks-in-dr1.fits')
        sys.exit(0)

    # sort by dec decreasing
    #B.cut(np.argsort(-B.dec))
    # RA increasing
    B.cut(np.argsort(B.ra))

    for b in B:
        if opt.check:
            fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue
        if opt.check_coadd:
            fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue

        print(b.brickname)

    if opt.save_to_fits:
        assert(opt.touching)
        # Write cut tables to file
        for tab,typ in zip([B,T],['bricks','ccds']):
            fn='%s-%s-cut.fits' % (typ,opt.name)
            if os.path.exists(fn):
                os.remove(fn)
            tab.writeto(fn)
            print('Wrote %s' % fn)
        # Write text files listing ccd and filename names
        nm1,nm2= 'ccds-%s.txt'% opt.name,'filenames-%s.txt' % opt.name
        if os.path.exists(nm1):
            os.remove(nm1)
        if os.path.exists(nm2):
            os.remove(nm2)
        f1,f2=open(nm1,'w'),open(nm2,'w')
        fns= list(set(T.get('image_filename')))
        for fn in fns:
            f2.write('%s\n' % fn.strip())
        for ti in T:
            f1.write('%s\n' % ti.get('image_filename').strip())
        f1.close()
        f2.close()
        print('Wrote *-names.txt')
    

    if opt.brickq_deps:
        import qdo
        from legacypipe.survey import on_bricks_dependencies

        #... find Queue...
        q = qdo.connect(opt.queue, create_ok=True)
        print('Connected to QDO queue', opt.queue, q)
        brick_to_task = dict()

        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
        print(len(T), 'CCDs remaining')

        T.wra = T.ra + (T.ra > 180) * -360
        wra = rlo - 360
        plt.clf()
        plt.plot(T.wra, T.dec, 'b.')
        ax = [wra, rhi, dlo, dhi]
        plt.axis(ax)
        plt.title('CCDs')
        plt.savefig('q-ccds.png')

        B.wra = B.ra + (B.ra > 180) * -360

        # this slight overestimate (for DECam images) is fine
        radius = 0.3
        Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius,
                            indexlist=True)
        ikeep = []
        for ib,(b,Iccd) in enumerate(zip(B, Iccds)):
            if Iccd is None or len(Iccd) == 0:
                print('No matched CCDs to brick', b.brickname)
                continue
            wcs = wcs_for_brick(b)
            cI = ccds_touching_wcs(wcs, T[np.array(Iccd)])
            print(len(cI), 'CCDs touching brick', b.brickname)
            if len(cI) == 0:
                continue
            ikeep.append(ib)
        B.cut(np.array(ikeep))
        print('Cut to', len(B), 'bricks touched by CCDs')
        
        for brickq in range(4):
            I = np.flatnonzero(B.brickq == brickq)
            print(len(I), 'bricks with brickq =', brickq)

            J = np.flatnonzero(B.brickq < brickq)
            preB = B[J]
            reqs = []
            if brickq > 0:
                for b in B[I]:
                    # find brick dependencies
                    brickdeps = on_bricks_dependencies(b, survey, bricks=preB)
                    # convert to task ids
                    taskdeps = [brick_to_task.get(b.brickname,None) for b in brickdeps]
                    # If we dropped a dependency brick from a previous brickq because
                    # of no overlapping CCDs, it won't appear in the brick_to_task map.
                    taskdeps = [t for t in taskdeps if t is not None]
                    reqs.append(taskdeps)

            plt.clf()
            plt.plot(B.wra, B.dec, '.', color='0.5')
            plt.plot(B.wra[I], B.dec[I], 'b.')
            plt.axis(ax)
            plt.title('Bricks: brickq=%i' % brickq)
            plt.savefig('q-bricks-%i.png' % brickq)
            
            # submit to qdo queue
            print('Queuing', len(B[I]), 'bricks')
            if brickq == 0:
                reqs = None
            else:
                assert(len(I) == len(reqs))
            taskids = q.add_multiple(B.brickname[I], requires=reqs)
            assert(len(taskids) == len(I))
            print('Queued', len(taskids), 'bricks')
            brick_to_task.update(dict(zip(B.brickname[I], taskids)))
        
    if not (opt.calibs or opt.forced or opt.lsb):
        sys.exit(0)

    bands = 'grz'
    log('Filters:', np.unique(T.filter))
    T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
    log('Cut to', len(T), 'CCDs in filters', bands)

    if opt.touching:
        allI = set()
        for b in B:
            wcs = wcs_for_brick(b)
            I = ccds_touching_wcs(wcs, T)
            log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
            if len(I) == 0:
                continue
            allI.update(I)
        allI = list(allI)
        allI.sort()
    elif opt.near:
        # Roughly brick radius + DECam image radius
        radius = 0.35
        allI,nil,nil = match_radec(T.ra, T.dec, B.ra, B.dec, radius, nearest=True)
    else:
        allI = np.arange(len(T))

    if opt.write_ccds:
        T[allI].writeto(opt.write_ccds)
        log('Wrote', opt.write_ccds)

    ## Be careful here -- T has been cut; we want to write out T.index.
    ## 'allI' contains indices into T.

    if opt.forced:
        log('Writing forced-photometry commands to', opt.out)
        f = open(opt.out,'w')
        log('Total of', len(allI), 'CCDs')
        for j,i in enumerate(allI):
            expstr = '%08i' % T.expnum[i]
            outfn = os.path.join('forced', expstr[:5], expstr,
                                 'decam-%s-%s-forced.fits' %
                                 (expstr, T.ccdname[i]))
            imgfn = os.path.join(survey.survey_dir, 'images',
                                 T.image_filename[i].strip())
            if (not os.path.exists(imgfn) and
                imgfn.endswith('.fz') and
                os.path.exists(imgfn[:-3])):
                imgfn = imgfn[:-3]

            #f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' %
            #        (imgfn, T.image_hdu[i], outfn))

            f.write('python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n' %
                    (T.expnum[i], T.ccdname[i], outfn))
            
        f.close()
        log('Wrote', opt.out)
        sys.exit(0)

    if opt.lsb:
        log('Writing LSB commands to', opt.out)
        f = open(opt.out,'w')
        log('Total of', len(allI), 'CCDs')
        for j,i in enumerate(allI):
            exp = T.expnum[i]
            ext = T.ccdname[i].strip()
            outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
            f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext))
        f.close()
        log('Wrote', opt.out)
        sys.exit(0)


    log('Writing calibs to', opt.out)
    f = open(opt.out,'w')
    log('Total of', len(allI), 'CCDs')

    batch = []

    def write_batch(f, batch, cmd):
        if cmd is None:
            cmd = ''
        f.write(cmd + ' '.join(batch) + '\n')

    cmd = None
    if opt.command:
        cmd = 'python legacypipe/run-calib.py '
        if opt.opt is not None:
            cmd += opt.opt + ' '
        
    for j,i in enumerate(allI):

        if opt.delete_sky or opt.delete_pvastrom:
            log(j+1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if opt.delete_sky and os.path.exists(im.skyfn):
                log('  deleting:', im.skyfn)
                os.unlink(im.skyfn)
            if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
                log('  deleting:', im.pvwcsfn)
                os.unlink(im.pvwcsfn)

        if opt.check:
            log(j+1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if not im.run_calibs(im, just_check=True):
                log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done')
                continue

        if opt.command:
            s = '%i-%s' % (T.expnum[i], T.ccdname[i])
            prefix = 'python legacypipe/run-calib.py ' + opt.opt
            #('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
            #     (T.expnum[i], T.ccdname[i]))
        else:
            s = '%i' % T.index[i]
            prefix = ''
            
        if j < 10:
            print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i],
                  'filename', T.image_filename[i])
            
        if not opt.nper:
            f.write(prefix + s + '\n')
        else:
            batch.append(s)
            if len(batch) >= opt.nper:
                write_batch(f, batch, cmd)
                batch = []

        if opt.check:
            f.flush()

    if len(batch):
        write_batch(f, batch, cmd)

    f.close()
    log('Wrote', opt.out)
    return 0
コード例 #5
0
ファイル: queue-calibs.py プロジェクト: gaztanaga/legacypipe
def main():
    """Main program.
    """
    import argparse

    parser = argparse.ArgumentParser(
        description=
        "This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)."
    )
    parser.add_argument('--calibs',
                        action='store_true',
                        help='Output CCDs that need to be calibrated.')

    parser.add_argument('--nper',
                        type=int,
                        default=None,
                        help='Batch N calibs per line')

    parser.add_argument('--forced',
                        action='store_true',
                        help='Output forced-photometry commands')

    parser.add_argument('--lsb',
                        action='store_true',
                        help='Output Low-Surface-Brightness commands')

    parser.add_argument('--touching',
                        action='store_true',
                        help='Cut to only CCDs touching selected bricks')
    parser.add_argument('--near',
                        action='store_true',
                        help='Quick cut to only CCDs near selected bricks')

    parser.add_argument('--check',
                        action='store_true',
                        help='Check which calibrations actually need to run.')
    parser.add_argument('--check-coadd',
                        action='store_true',
                        help='Check which caoadds actually need to run.')
    parser.add_argument('--out',
                        help='Output filename for calibs, default %(default)s',
                        default='jobs')
    parser.add_argument('--command',
                        action='store_true',
                        help='Write out full command-line to run calib')
    parser.add_argument('--opt', help='With --command, extra options to add')

    parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
    parser.add_argument('--mindec', type=float, help='Minimum Dec to run')

    parser.add_argument('--region', help='Region to select')

    parser.add_argument('--bricks', help='Set bricks.fits file to load')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')
    parser.add_argument('--ignore_cuts',
                        action='store_true',
                        default=False,
                        help='no photometric or blacklist cuts')
    parser.add_argument('--save_to_fits',
                        action='store_true',
                        default=False,
                        help='save cut brick,ccd to fits table')
    parser.add_argument(
        '--name',
        action='store',
        default='dr3',
        help='save with this suffix, e.g. refers to ccds table')

    parser.add_argument('--delete-sky',
                        action='store_true',
                        help='Delete any existing sky calibration files')
    parser.add_argument('--delete-pvastrom',
                        action='store_true',
                        help='Delete any existing PV WCS calibration files')

    parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')

    parser.add_argument(
        '--brickq',
        type=int,
        default=None,
        help='Queue only bricks with the given "brickq" value [0 to 3]')

    parser.add_argument(
        '--brickq-deps',
        action='store_true',
        default=False,
        help='Queue bricks directly using qdo API, setting brickq dependencies'
    )
    parser.add_argument('--queue',
                        default='bricks',
                        help='With --brickq-deps, the QDO queue name to use')

    opt = parser.parse_args()

    survey = LegacySurveyData()
    if opt.bricks is not None:
        B = fits_table(opt.bricks)
        log('Read', len(B), 'from', opt.bricks)
    else:
        B = survey.get_bricks()

    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        log('Read', len(T), 'from', opt.ccds)
    else:
        T = survey.get_ccds()
        log(len(T), 'CCDs')
    T.index = np.arange(len(T))

    if opt.ignore_cuts == False:
        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
    print(len(T), 'CCDs remain')

    # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
    # plt.clf()
    # plt.hist(counts, counts.max()+1)
    # plt.savefig('bricks.png')
    # B.cut(I[counts >= 9])
    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # #plt.scatter(B.ra[I], B.dec[I], c=counts)
    # plt.savefig('bricks2.png')

    # DES Stripe82
    #rlo,rhi = 350.,360.
    # rlo,rhi = 300., 10.
    # dlo,dhi = -6., 4.
    # TINY bit
    #rlo,rhi = 350.,351.1
    #dlo,dhi = 0., 1.1

    # EDR+
    # 860 bricks
    # ~10,000 CCDs
    #rlo,rhi = 239,246
    #dlo,dhi =   5, 13

    # DR1
    #rlo,rhi = 0, 360
    # part 1
    #dlo,dhi = 25, 40
    # part 2
    #dlo,dhi = 20,25
    # part 3
    #dlo,dhi = 15,20
    # part 4
    #dlo,dhi = 10,15
    # part 5
    #dlo,dhi = 5,10
    # the rest
    #dlo,dhi = -11, 5
    #dlo,dhi = 15,25.5

    dlo, dhi = -25, 40
    rlo, rhi = 0, 360

    # Arjun says 3x3 coverage area is roughly
    # RA=240-252 DEC=6-12 (but not completely rectangular)

    # COSMOS
    #rlo,rhi = 148.9, 151.2
    #dlo,dhi = 0.9, 3.5

    # A nice well-behaved region (EDR2/3)
    # rlo,rhi = 243.6, 244.6
    # dlo,dhi = 8.1, 8.6

    # 56 bricks, ~725 CCDs
    #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
    # 240 bricks, ~3000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
    # 535 bricks, ~7000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))

    if opt.region in ['test1', 'test2', 'test3', 'test4']:
        nm = dict(
            test1='2446p115',  # weird stuff around bright star
            test2='1183p292',  # faint sources around bright galaxy
            test3='3503p005',  # DES
            test4='1163p277',  # Pollux
        )[opt.region]

        B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
        log('Cut to', len(B), 'bricks')
        log(B.ra, B.dec)
        dlo, dhi = -90, 90
        rlo, rhi = 0, 360

    elif opt.region == 'edr':
        # EDR:
        # 535 bricks, ~7000 CCDs
        rlo, rhi = 240, 245
        dlo, dhi = 5, 12

    elif opt.region == 'edrplus':
        rlo, rhi = 235, 248
        dlo, dhi = 5, 15

    elif opt.region == 'edr-south':
        rlo, rhi = 240, 245
        dlo, dhi = 5, 10

    elif opt.region == 'cosmos1':
        # 16 bricks in the core of the COSMOS field.
        rlo, rhi = 149.75, 150.75
        dlo, dhi = 1.6, 2.6

    elif opt.region == 'pristine':
        # Stream?
        rlo, rhi = 240, 250
        dlo, dhi = 10, 15

    elif opt.region == 'des':
        dlo, dhi = -6., 4.
        rlo, rhi = 317., 7.

        T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
        log('Cut to', len(T), 'CCDs with "CPDES82" in filename')

    elif opt.region == 'subdes':
        rlo, rhi = 320., 360.
        dlo, dhi = -1.25, 1.25

    elif opt.region == 'northwest':
        rlo, rhi = 240, 360
        dlo, dhi = 20, 40
    elif opt.region == 'north':
        rlo, rhi = 120, 240
        dlo, dhi = 20, 40
    elif opt.region == 'northeast':
        rlo, rhi = 0, 120
        dlo, dhi = 20, 40
    elif opt.region == 'southwest':
        rlo, rhi = 240, 360
        dlo, dhi = -20, 0
    elif opt.region == 'south':
        rlo, rhi = 120, 240
        dlo, dhi = -20, 0
    elif opt.region == 'southeast':
        rlo, rhi = 0, 120
        dlo, dhi = -20, 0
    elif opt.region == 'southsoutheast':
        rlo, rhi = 0, 120
        dlo, dhi = -20, -10
    elif opt.region == 'midwest':
        rlo, rhi = 240, 360
        dlo, dhi = 0, 20
    elif opt.region == 'middle':
        rlo, rhi = 120, 240
        dlo, dhi = 0, 20
    elif opt.region == 'mideast':
        rlo, rhi = 0, 120
        dlo, dhi = 0, 20

    elif opt.region == 'grz':
        # Bricks with grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
        log('Cut to', len(B), 'bricks with grz coverage')

    elif opt.region == 'nogrz':
        # Bricks without grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
        log('Cut to', len(B), 'bricks withOUT grz coverage')
    elif opt.region == 'deep2':
        rlo, rhi = 250, 260
        dlo, dhi = 30, 35

    elif opt.region == 'deep2f3':
        rlo, rhi = 351.25, 353.75
        dlo, dhi = 0, 0.5

    elif opt.region == 'virgo':
        rlo, rhi = 185, 190
        dlo, dhi = 10, 15

    elif opt.region == 'virgo2':
        rlo, rhi = 182, 192
        dlo, dhi = 8, 18

    elif opt.region == 'lsb':
        rlo, rhi = 147.2, 147.8
        dlo, dhi = -0.4, 0.4

    elif opt.region == 'eboss-sgc':
        # generous boundaries to make sure get all relevant images
        # RA -45 to +45
        # Dec -5 to +7
        rlo, rhi = 310., 50.
        dlo, dhi = -6., 6.

    elif opt.region == 'eboss-ngc':
        # generous boundaries to make sure get all relevant images
        # NGC ELGs
        # RA 115 to 175
        # Dec 15 to  30
        rlo, rhi = 122., 177.
        dlo, dhi = 12., 32.

    elif opt.region == 'mzls':
        dlo, dhi = 30., 90.
    elif opt.region == 'dr4-bootes':
        # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
        #dlo,dhi = 34., 35.
        #rlo,rhi = 209.5, 210.5
        dlo, dhi = 33., 36.
        rlo, rhi = 216.5, 219.5

    if opt.mindec is not None:
        dlo = opt.mindec
    if opt.maxdec is not None:
        dhi = opt.maxdec

    if rlo < rhi:
        B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi))
    else:  # RA wrap
        B.cut(
            np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) *
            (B.dec <= dhi))
    log(len(B), 'bricks in range')
    #for name in B.get('brickname'):
    #print(name)
    B.writeto('bricks-cut.fits')

    I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize)
    keep = np.zeros(len(B), bool)
    for i in I:
        keep[i] = True
    B.cut(keep)
    log('Cut to', len(B), 'bricks near CCDs')

    plt.clf()
    plt.plot(B.ra, B.dec, 'b.')
    plt.title('DR3 bricks')
    plt.axis([360, 0, np.min(B.dec) - 1, np.max(B.dec) + 1])
    plt.savefig('bricks.png')

    if opt.brickq is not None:
        B.cut(B.brickq == opt.brickq)
        log('Cut to', len(B), 'with brickq =', opt.brickq)

    if opt.touching:
        keep = np.zeros(len(T), bool)
        for j in J:
            keep[j] = True
        T.cut(keep)
        log('Cut to', len(T), 'CCDs near bricks')

    # Aside -- how many near DR1=1 CCDs?
    if False:
        T2 = D.get_ccds()
        log(len(T2), 'CCDs')
        T2.cut(T2.dr1 == 1)
        log(len(T2), 'CCDs marked DR1=1')
        log(len(B), 'bricks in range')
        I, J, d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize)
        keep = np.zeros(len(B), bool)
        for i in I:
            keep[i] = True
        B2 = B[keep]
        log('Total of', len(B2), 'bricks near CCDs with DR1=1')
        for band in 'grz':
            Tb = T2[T2.filter == band]
            log(len(Tb), 'in filter', band)
            I, J, d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec,
                                  survey.bricksize)
            good = np.zeros(len(B2), np.uint8)
            for i in I:
                good[i] = 1
            B2.set('has_' + band, good)

        B2.writeto('survey-bricks-in-dr1.fits')
        sys.exit(0)

    # sort by dec decreasing
    #B.cut(np.argsort(-B.dec))
    # RA increasing
    B.cut(np.argsort(B.ra))

    for b in B:
        if opt.check:
            fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3],
                                                      b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue
        if opt.check_coadd:
            fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (
                b.brickname[:3], b.brickname, b.brickname)
            if os.path.exists(fn):
                print('Exists:', fn, file=sys.stderr)
                continue

        #print(b.brickname)

    if opt.save_to_fits:
        assert (opt.touching)
        # Write cut tables to file
        for tab, typ in zip([B, T], ['bricks', 'ccds']):
            fn = '%s-%s-cut.fits' % (typ, opt.region)
            if os.path.exists(fn):
                os.remove(fn)
            tab.writeto(fn)
            print('Wrote %s' % fn)
        # Write text files listing ccd and filename names
        nm1, nm2 = 'ccds-%s.txt' % opt.region, 'filenames-%s.txt' % opt.region
        if os.path.exists(nm1):
            os.remove(nm1)
        if os.path.exists(nm2):
            os.remove(nm2)
        f1, f2 = open(nm1, 'w'), open(nm2, 'w')
        fns = list(set(T.get('image_filename')))
        for fn in fns:
            f2.write('%s\n' % fn.strip())
        for ti in T:
            f1.write('%s\n' % ti.get('image_filename').strip())
        f1.close()
        f2.close()
        print('Wrote *-names.txt')

    if opt.brickq_deps:
        import qdo
        from legacypipe.survey import on_bricks_dependencies

        #... find Queue...
        q = qdo.connect(opt.queue, create_ok=True)
        print('Connected to QDO queue', opt.queue, q)
        brick_to_task = dict()

        I = survey.photometric_ccds(T)
        print(len(I), 'CCDs are photometric')
        T.cut(I)
        I = survey.apply_blacklist(T)
        print(len(I), 'CCDs are not blacklisted')
        T.cut(I)
        print(len(T), 'CCDs remaining')

        T.wra = T.ra + (T.ra > 180) * -360
        wra = rlo - 360
        plt.clf()
        plt.plot(T.wra, T.dec, 'b.')
        ax = [wra, rhi, dlo, dhi]
        plt.axis(ax)
        plt.title('CCDs')
        plt.savefig('q-ccds.png')

        B.wra = B.ra + (B.ra > 180) * -360

        # this slight overestimate (for DECam images) is fine
        radius = 0.3
        Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius, indexlist=True)
        ikeep = []
        for ib, (b, Iccd) in enumerate(zip(B, Iccds)):
            if Iccd is None or len(Iccd) == 0:
                print('No matched CCDs to brick', b.brickname)
                continue
            wcs = wcs_for_brick(b)
            cI = ccds_touching_wcs(wcs, T[np.array(Iccd)])
            print(len(cI), 'CCDs touching brick', b.brickname)
            if len(cI) == 0:
                continue
            ikeep.append(ib)
        B.cut(np.array(ikeep))
        print('Cut to', len(B), 'bricks touched by CCDs')

        for brickq in range(4):
            I = np.flatnonzero(B.brickq == brickq)
            print(len(I), 'bricks with brickq =', brickq)

            J = np.flatnonzero(B.brickq < brickq)
            preB = B[J]
            reqs = []
            if brickq > 0:
                for b in B[I]:
                    # find brick dependencies
                    brickdeps = on_bricks_dependencies(b, survey, bricks=preB)
                    # convert to task ids
                    taskdeps = [
                        brick_to_task.get(b.brickname, None) for b in brickdeps
                    ]
                    # If we dropped a dependency brick from a previous brickq because
                    # of no overlapping CCDs, it won't appear in the brick_to_task map.
                    taskdeps = [t for t in taskdeps if t is not None]
                    reqs.append(taskdeps)

            plt.clf()
            plt.plot(B.wra, B.dec, '.', color='0.5')
            plt.plot(B.wra[I], B.dec[I], 'b.')
            plt.axis(ax)
            plt.title('Bricks: brickq=%i' % brickq)
            plt.savefig('q-bricks-%i.png' % brickq)

            # submit to qdo queue
            print('Queuing', len(B[I]), 'bricks')
            if brickq == 0:
                reqs = None
            else:
                assert (len(I) == len(reqs))
            taskids = q.add_multiple(B.brickname[I], requires=reqs)
            assert (len(taskids) == len(I))
            print('Queued', len(taskids), 'bricks')
            brick_to_task.update(dict(zip(B.brickname[I], taskids)))

    if not (opt.calibs or opt.forced or opt.lsb):
        sys.exit(0)

    bands = 'grz'
    log('Filters:', np.unique(T.filter))
    T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
    log('Cut to', len(T), 'CCDs in filters', bands)

    if opt.touching:
        allI = set()
        for b in B:
            wcs = wcs_for_brick(b)
            I = ccds_touching_wcs(wcs, T)
            log(len(I), 'CCDs for brick', b.brickid,
                'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
            if len(I) == 0:
                continue
            allI.update(I)
        allI = list(allI)
        allI.sort()
    elif opt.near:
        # Roughly brick radius + DECam image radius
        radius = 0.35
        allI, nil, nil = match_radec(T.ra,
                                     T.dec,
                                     B.ra,
                                     B.dec,
                                     radius,
                                     nearest=True)
    else:
        allI = np.arange(len(T))

    if opt.write_ccds:
        T[allI].writeto(opt.write_ccds)
        log('Wrote', opt.write_ccds)

    ## Be careful here -- T has been cut; we want to write out T.index.
    ## 'allI' contains indices into T.

    if opt.forced:
        log('Writing forced-photometry commands to', opt.out)
        f = open(opt.out, 'w')
        log('Total of', len(allI), 'CCDs')
        for j, i in enumerate(allI):
            expstr = '%08i' % T.expnum[i]
            outfn = os.path.join(
                'forced', expstr[:5], expstr,
                'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i]))
            imgfn = os.path.join(survey.survey_dir, 'images',
                                 T.image_filename[i].strip())
            if (not os.path.exists(imgfn) and imgfn.endswith('.fz')
                    and os.path.exists(imgfn[:-3])):
                imgfn = imgfn[:-3]

            #f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' %
            #        (imgfn, T.image_hdu[i], outfn))

            f.write(
                'python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n'
                % (T.expnum[i], T.ccdname[i], outfn))

        f.close()
        log('Wrote', opt.out)
        sys.exit(0)

    if opt.lsb:
        log('Writing LSB commands to', opt.out)
        f = open(opt.out, 'w')
        log('Total of', len(allI), 'CCDs')
        for j, i in enumerate(allI):
            exp = T.expnum[i]
            ext = T.ccdname[i].strip()
            outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
            f.write(
                'python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n'
                % (exp, ext, outfn, exp, ext))
        f.close()
        log('Wrote', opt.out)
        sys.exit(0)

    log('Writing calibs to', opt.out)
    f = open(opt.out, 'w')
    log('Total of', len(allI), 'CCDs')

    batch = []

    def write_batch(f, batch, cmd):
        if cmd is None:
            cmd = ''
        f.write(cmd + ' '.join(batch) + '\n')

    cmd = None
    if opt.command:
        cmd = 'python legacypipe/run-calib.py '
        if opt.opt is not None:
            cmd += opt.opt + ' '

    for j, i in enumerate(allI):

        if opt.delete_sky or opt.delete_pvastrom:
            log(j + 1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if opt.delete_sky and os.path.exists(im.skyfn):
                log('  deleting:', im.skyfn)
                os.unlink(im.skyfn)
            if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
                log('  deleting:', im.pvwcsfn)
                os.unlink(im.pvwcsfn)

        if opt.check:
            log(j + 1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if not im.run_calibs(im, just_check=True):
                log('Calibs for', im.expnum, im.ccdname, im.calname,
                    'already done')
                continue

        if opt.command:
            s = '%i-%s' % (T.expnum[i], T.ccdname[i])
            prefix = 'python legacypipe/run-calib.py ' + opt.opt
            #('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
            #     (T.expnum[i], T.ccdname[i]))
        else:
            s = '%i' % T.index[i]
            prefix = ''

        if j < 10:
            print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname',
                  T.ccdname[i], 'filename', T.image_filename[i])

        if not opt.nper:
            f.write(prefix + s + '\n')
        else:
            batch.append(s)
            if len(batch) >= opt.nper:
                write_batch(f, batch, cmd)
                batch = []

        if opt.check:
            f.flush()

    if len(batch):
        write_batch(f, batch, cmd)

    f.close()
    log('Wrote', opt.out)
    return 0