Exemplo n.º 1
0
def main():

    B = fits_table(
        '/global/cfs/cdirs/cosmo/data/legacysurvey/dr8/survey-bricks.fits.gz')
    B.ll, B.bb = radectolb(B.ra, B.dec)
    I = np.flatnonzero((B.dec > -70) * (np.abs(B.bb) > 10))
    B[I].writeto('bricks-for-gaia.fits')
    BG = B[I]
    BG = BG[np.argsort(-BG.dec)]

    # healpixes = set()
    # nside = 32
    # for r,d in zip(BG.ra,BG.dec):
    #     hpxy = radecdegtohealpix(r, d, nside)
    #     hpring = healpix_xy_to_ring(hpxy, nside)
    #     healpixes.add(hpring)
    # hr,hd = [],[]
    # for hp in healpixes:
    #     hp = healpix_ring_to_xy(hp, nside)
    #     r,d = healpix_to_radecdeg(hp, nside, 0.5, 0.5)
    #     hr.append(r)
    #     hd.append(d)
    # plt.plot(hr, hd, 'b.', alpha=0.1);

    survey = LegacySurveyData('/global/cfs/cdirs/cosmo/work/legacysurvey/dr9')

    #BG = BG[:100]

    # GG = []
    # for i,brick in enumerate(BG):
    #     G = one_brick(brick, survey)
    #     GG.append(G)

    mp = multiproc(32)

    GG = []
    iset = 0
    while len(BG):
        N = 10000
        outfn = '/global/cscratch1/sd/dstn/gaia-mask-dr9-set%i.fits' % iset
        if os.path.exists(outfn):
            Gset = fits_table(outfn)
            print('Read', outfn)
            nb = len(set(Gset.brickname))
            if nb != N:
                print('Warning: file contains', nb, 'bricks, vs', N)
        else:
            Gset = mp.map(bounce_one_brick,
                          [(brick, survey) for brick in BG[:N]])
            Gset = [G for G in Gset if G is not None]
            Gset = merge_tables(Gset, columns='fillzero')
            Gset.writeto(outfn)
        GG.append(Gset)
        iset += 1
        BG = BG[N:]

    G = merge_tables(GG, columns='fillzero')
    G.writeto('/global/cscratch1/sd/dstn/gaia-mask-dr9.fits')
Exemplo n.º 2
0
def main():
    survey = LegacySurveyData()
    ccds = survey.get_ccds_readonly()
    print(len(ccds), 'CCDs')
    ccds = ccds[ccds.ccd_cuts == 0]
    print(len(ccds), 'good CCDs')

    # Find bricks touched by >=1 CCD
    bricks = survey.get_bricks_readonly()
    bricks = bricks[(bricks.dec > -20) * (bricks.dec < 35.)]
    print(len(bricks), 'bricks in Dec range')
    I, J, d = match_radec(bricks.ra,
                          bricks.dec,
                          ccds.ra,
                          ccds.dec,
                          0.5,
                          nearest=True)
    bricks = bricks[I]
    print(len(bricks), 'bricks')

    bands = ['g', 'r', 'z']

    nexps = {}
    for b in bands:
        ne = np.zeros(len(bricks), np.int16)
        nexps[b] = ne
        bricks.set('nexp_' + b, ne)
    npix = {}
    for b in bands:
        n = np.zeros(len(bricks), np.int64)
        npix[b] = n
        bricks.set('npix_' + b, n)

    for b in bands:
        n = np.zeros(len(bricks), np.float32)
        bricks.set('psfdepth_' + b, n)

    args = enumerate(bricks)
    mp = multiproc(8)
    R = mp.map(one_brick, args)

    for ibrick, res in enumerate(R):
        if res is None:
            continue

        (npix, nexps, depths) = res
        for band in bands:
            bricks.get('npix_' + band)[ibrick] = npix[band]
            bricks.get('nexp_' + band)[ibrick] = nexps[band]
            bricks.get('psfdepth_' + band)[ibrick] = depths[band]

    bricks.cut((bricks.nexp_g + bricks.nexp_r + bricks.nexp_z) > 0)
    bricks.writeto('/global/cscratch1/sd/dstn/bricks-nexp.fits')
Exemplo n.º 3
0
def organize_by_brick(sample_fns,
                      sbricks,
                      outdir=None,
                      seed=None,
                      prefix=None):
    '''
    For each sample_fn, split into bricks
    get  brick sample fn for that brick and sample
    write it if does not exist
    sample_fn -- sample_seed.fits file assigned to that mpi task
    sbricks -- survey bricks table cut to radec region
    '''
    dr = get_bybrick_dir(outdir=outdir)
    for sample_fn in sample_fns:
        # Skip if already looped over bricks for this sample
        check_done = os.path.join(dr, get_sample_fn(seed=seed, prefix=prefix))
        check_done = check_done.replace('.fits', '_done.txt')
        if os.path.exists(check_done):
            print('check_done exists: %s' % check_done)
            continue
        #
        sample = fits_table(sample_fn)
        print('sample min,max ra,dec= %f %f %f %f' % (sample.ra.min(),sample.ra.max(),\
                                                      sample.dec.min(),sample.dec.max()))
        # Loop over survey bricks
        survey = LegacySurveyData()
        for sbrick in sbricks:
            # Get output fn for this brick and sample
            fn = os.path.join(
                dr,
                get_brick_sample_fn(brickname=sbrick.brickname,
                                    seed=seed,
                                    prefix=prefix))
            if os.path.exists(fn):
                continue
            # Cut sample by brick's bounds
            brickinfo = survey.get_brick_by_name(sbrick.brickname)
            brickwcs = wcs_for_brick(brickinfo)
            ra1, ra2, dec1, dec2 = brickwcs.radec_bounds()
            keep=  (sample.ra >= ra1)*(sample.ra <= ra2)*\
                   (sample.dec >= dec1)*(sample.dec <= dec2)
            sample2 = sample.copy()
            if np.where(keep)[0].size > 0:
                sample2.cut(keep)
                sample2.writeto(fn)
                print('Wrote %s' % fn)
            else:
                print('WARNING: sample=%s has no ra,dec in brick=%s' %
                      (sample_fn, sbrick.brickname))
        # This sample is done
        with open(check_done, 'w') as foo:
            foo.write('done')
Exemplo n.º 4
0
def get_grid_randoms(truth_fn, bricknames=[], south=True, seed=None):
    
    rng = np.random.RandomState(seed=seed)
    randoms = 0
    survey = LegacySurveyData(survey_dir='/global/cfs/cdirs/cosmo/work/legacysurvey/dr9')
    for iseed,brickname in enumerate(bricknames):
        randoms += get_grid_in_brick(survey,brickname,rng=rng)
    
    randoms.photsys = randoms.full('S' if south else 'N')
    truth = get_truth(truth_fn,south=south)
    randoms.fill(sample_from_truth(randoms,truth,rng=rng),index_self=None,index_other=None)

    return randoms
Exemplo n.º 5
0
 def __init__(self, ls_dir=None, outdir=None, savedir=None, jpeg=False):
     """outdir: required
        ls_dir: not needed if env var LEGACY_SURVEY_DIR already set
        save_dir: where write hdf5 files, outdir if None
     """
     self.outdir = outdir
     self.jpeg = jpeg
     if ls_dir:
         os.environ["LEGACY_SURVEY_DIR"] = ls_dir
     self.savedir = savedir
     if self.savedir is None:
         self.savedir = self.outdir
     self.survey = LegacySurveyData()
Exemplo n.º 6
0
def main(outfn='ccds-annotated.fits', ccds=None, **kwargs):
    survey = LegacySurveyData(ccds=ccds)
    if ccds is None:
        ccds = survey.get_ccds()

    # Set to True if we successfully read the calibration products and computed
    # annotated values
    init_annotations(ccds)

    annotate(ccds, **kwargs)

    print('Writing to', outfn)
    ccds.writeto(outfn)
    print('Wrote', outfn)
    def __init__(self, **kw):
        super(analysis_setup, self).__init__(**kw)
        # raise ValueError('hey')
        self.tol = Tolerances().get(survey=self.survey,
                                    bands=self.bands,
                                    obj=self.obj)
        self.config_dir= os.path.join(os.path.dirname(self.outdir),
                                      'testcase_%s_%s' % \
                                        (kw['survey'],kw['bands']))
        self.rsdir = 'rs0'

        survey = LegacySurveyData()
        brickinfo = get_brickinfo_hack(survey, self.brick)
        self.brickwcs = wcs_for_brick(brickinfo)
Exemplo n.º 8
0
def queue():
    if False:
        survey = LegacySurveyData()
        ccds = survey.get_ccds()
        bricks = survey.get_bricks()
        print(len(bricks), 'bricks')
        print(len(ccds), 'CCDs')

        bricks.cut((bricks.dec >= -30) * (bricks.dec <= 30))
        print(len(bricks), 'in Dec [-30, +30]')

        I = survey.photometric_ccds(ccds)
        ccds.cut(I)
        print(len(ccds), 'pass photometric cut')

        I, J, d = match_radec(bricks.ra,
                              bricks.dec,
                              ccds.ra,
                              ccds.dec,
                              0.5,
                              nearest=True)
        print(len(I), 'bricks with CCDs nearby')
        bricks.cut(I)
        bricknames = bricks.brickname

    else:
        # DR7: use Martin's list of bricks w/ CCD coverage
        f = open('nccds.dat')
        bricknames = []
        for line in f.readlines():
            words = line.strip().split(' ')
            brick = words[0]
            nccd = int(words[1])
            if nccd > 0:
                bricknames.append(brick)

    # qdo
    bb = bricknames
    while len(bb):
        print(' '.join(bb[:100]))
        bb = bb[100:]
    return

    mp = multiproc(16)
    N = len(bricks)
    args = [(brick, i, N, plots, {}) for i, brick in enumerate(bricks)]
    mp.map(run_one_brick, args)
Exemplo n.º 9
0
def getbrickfiles(brickname=None):

    survey = LegacySurveyData()
    brickinfo = survey.get_brick_by_name(brickname)
    brickwcs = wcs_for_brick(brickinfo)
    ccdinfo = survey.ccds_touching_wcs(brickwcs)
    nccd = len(ccdinfo)

    calibdir = survey.get_calib_dir()
    imagedir = survey.survey_dir

    # Construct image file names and the calibration file names.
    expnum = ccdinfo.expnum
    ccdname = ccdinfo.ccdname

    psffiles = list()
    skyfiles = list()
    imagefiles = list()
    for ccd in ccdinfo:
        info = survey.get_image_object(ccd)
        for attr in ['imgfn', 'dqfn', 'wtfn']:
            fn = getattr(info, attr).replace(imagedir + '/', '')
            #if '160108_073601' in fn:
            #    pdb.set_trace()
            imagefiles.append(fn)
        psffiles.append(info.psffn.replace(calibdir, 'calib'))
        skyfiles.append(info.splineskyfn.replace(calibdir, 'calib'))

    #for ii in range(nccd):
    #exp = '{0:08d}'.format(expnum[ii])
    #rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits')
    #psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile))
    #skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile))
    #imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii]))))

    #print(np.array(imagefiles))
    #print(np.array(psffiles))
    #print(np.array(skyfiles))
    return imagefiles, psffiles, skyfiles
Exemplo n.º 10
0
    def __init__(
        self,
        expnum=None,
        ccdname=None,
        ccdwcs=None,
        pattern='/project/projectdirs/cosmo/work/ps1/cats/chunks-qz-star-v3/ps1-%(hp)05d.fits'
    ):
        """Read PS1 or gaia sources for an exposure number + CCD name or CCD WCS

        Args:
            expnum, ccdname: select catalogue with these
            ccdwcs: or select catalogue with this
            pattern: absolute path and wildcard for PS1 or Gaia catalogues
                dr: /project/projectdirs/cosmo/work/
                PS1: ${dr}/ps1/cats/chunks-qz-star-v3/ps1-%(hp)05d.fits
                PS1-Gaia: ${dr}/gaia/chunks-ps1-gaia/chunk-%(hp)05d.fits
        """
        assert ('ps1' in pattern or 'gaia' in pattern)
        #assert(ps1_or_gaia in ['ps1','ps1_gaia'])
        #if ps1_or_gaia == 'ps1':
        #  # PS1 "qz" directory
        #  # e.g. /project/projectdirs/cosmo/work/ps1/cats/chunks-qz-star-v2
        #  self.catdir= os.getenv('PS1CAT_DIR')
        #elif ps1_or_gaia == 'ps1_gaia':
        #  # PS1-Gaia "qz" matches-only directory
        #  # e.g. /project/projectdirs/cosmo/work/gaia/chunks-ps1-gaia
        #  self.catdir= os.getenv('PS1_GAIA_MATCHES')
        #fnpattern = os.path.join(self.catdir, prefix + '-%(hp)05d.fits')
        super(ps1cat, self).__init__(pattern)

        if ccdwcs is None:
            from legacypipe.survey import LegacySurveyData
            survey = LegacySurveyData()
            ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
            im = survey.get_image_object(ccd)
            self.ccdwcs = im.get_wcs()
        else:
            self.ccdwcs = ccdwcs
Exemplo n.º 11
0
    def __init__(self, expnum=None, ccdname=None, ccdwcs=None):
        """Read PS1 or gaia sources for an exposure number + CCD name or CCD WCS

        Args:
            expnum, ccdname: select catalogue with these
            ccdwcs: or select catalogue with this

        """
        self.ps1catdir = os.getenv('PS1CAT_DIR')
        if self.ps1catdir is None:
            raise ValueError(
                'You must have the PS1CAT_DIR environment variable set to point to healpixed PS1 catalogs'
            )
        fnpattern = os.path.join(self.ps1catdir, 'ps1-%(hp)05d.fits')
        super(ps1cat, self).__init__(fnpattern)

        if ccdwcs is None:
            from legacypipe.survey import LegacySurveyData
            survey = LegacySurveyData()
            ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
            im = survey.get_image_object(ccd)
            self.ccdwcs = im.get_wcs()
        else:
            self.ccdwcs = ccdwcs
Exemplo n.º 12
0
    def __init__(self, expnum=None, ccdname=None, ccdwcs=None):
        """Initialize the class with either the exposure number *and* CCD name, or
        directly with the WCS of the CCD of interest.

        """
        # GAIA and PS1 info, gaia for astrometry, ps1 for photometry
        self.gaiadir = os.getenv('GAIACAT_DIR')
        # PS1 only
        self.ps1dir = os.getenv('PS1CAT_DIR')  # PS1 only
        if self.ps1dir is None:
            raise ValueError('Need PS1CAT_DIR environment variable to be set.')
        if self.gaiadir is None:
            print(
                'WARNING: GAIACAT_DIR environment variable not set: using Pan-STARRS1 for astrometry'
            )
        self.nside = 32
        if ccdwcs is None:
            from legacypipe.survey import LegacySurveyData
            survey = LegacySurveyData()
            ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
            im = survey.get_image_object(ccd)
            self.ccdwcs = im.get_wcs()
        else:
            self.ccdwcs = ccdwcs
Exemplo n.º 13
0
def rbmain():
    travis = 'travis' in sys.argv

    extra_args = [
        '--old-calibs-ok',
        #'--verbose',
    ]
    if travis:
        extra_args.extend(
            ['--no-wise-ceres', '--no-gaia', '--no-large-galaxies'])

    if 'ceres' in sys.argv:
        surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
        main(args=[
            '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
            '--no-wise', '--force-all', '--no-write', '--ceres',
            '--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres',
            '--no-depth-cut'
        ])
        sys.exit(0)

    # demo RexGalaxy, with plots
    if False:
        from legacypipe.survey import RexGalaxy
        from tractor import NanoMaggies, PixPos
        from tractor import Image, GaussianMixturePSF, LinearPhotoCal
        from legacypipe.survey import LogRadius
        rex = RexGalaxy(PixPos(1., 2.), NanoMaggies(r=3.), LogRadius(0.))
        print('Rex:', rex)
        print('Rex params:', rex.getParams())
        print('Rex nparams:', rex.numberOfParams())
        H, W = 100, 100
        tim = Image(data=np.zeros((H, W), np.float32),
                    inverr=np.ones((H, W), np.float32),
                    psf=GaussianMixturePSF(1., 0., 0., 4., 4., 0.),
                    photocal=LinearPhotoCal(1., band='r'))
        derivs = rex.getParamDerivatives(tim)
        print('Derivs:', len(derivs))
        print('Rex params:', rex.getParamNames())

        import pylab as plt
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence('rex')

        for d, nm in zip(derivs, rex.getParamNames()):
            plt.clf()
            plt.imshow(d.patch, interpolation='nearest', origin='lower')
            plt.title('Derivative %s' % nm)
            ps.savefig()

        sys.exit(0)

    # Test RexGalaxy

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
    outdir = 'out-testcase6-rex'
    main(args=[
        '--brick',
        '1102p240',
        '--zoom',
        '500',
        '600',
        '650',
        '750',
        '--force-all',
        '--no-write',
        '--no-wise',
        #'--rex', #'--plots',
        '--survey-dir',
        surveydir,
        '--outdir',
        outdir
    ] + extra_args)
    fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 2)
    print('Types:', T.type)
    # Since there is a Tycho-2 star in the blob, forced to be PSF.
    assert (T.type[0] == 'PSF ')
    cmd = (
        '(cd %s && sha256sum -c %s)' %
        (outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum')))
    print(cmd)
    rtn = os.system(cmd)
    assert (rtn == 0)

    # Test with a Tycho-2 star in the blob.

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
    outdir = 'out-testcase6'
    main(args=[
        '--brick', '1102p240', '--zoom', '500', '600', '650', '750',
        '--force-all', '--no-write', '--no-wise', '--survey-dir', surveydir,
        '--outdir', outdir
    ] + extra_args)
    fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 2)
    print('Types:', T.type)
    # Since there is a Tycho-2 star in the blob, forced to be PSF.
    assert (T.type[0] == 'PSF ')

    # Test that we can run splinesky calib if required...

    from legacypipe.decam import DecamImage
    DecamImage.splinesky_boxsize = 128

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
    outdir = 'out-testcase4'

    fn = os.path.join(surveydir, 'calib', 'decam', 'splinesky', '00431',
                      '00431608', 'decam-00431608-N3.fits')
    if os.path.exists(fn):
        os.unlink(fn)

    main(args=[
        '--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
        '--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
        os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
        os.path.join(surveydir, 'images', 'unwise-tr'), '--blob-image',
        '--no-hybrid-psf', '--survey-dir', surveydir, '--outdir', outdir
    ] + extra_args + ['-v'])
    print('Checking for calib file', fn)
    assert (os.path.exists(fn))

    # Wrap-around, hybrid PSF
    surveydir = os.path.join(os.path.dirname(__file__), 'testcase8')
    outdir = 'out-testcase8'

    main(args=[
        '--brick',
        '1209p050',
        '--zoom',
        '720',
        '1095',
        '3220',
        '3500',
        '--force-all',
        '--no-write',
        '--no-wise',  #'--plots',
        '--survey-dir',
        surveydir,
        '--outdir',
        outdir
    ] + extra_args)

    # Test with a Tycho-2 star + another saturated star in the blob.

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase7')
    outdir = 'out-testcase7'
    # remove --no-gaia
    my_extra_args = [a for a in extra_args if a != '--no-gaia']
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia-dr2')
    os.environ['GAIA_CAT_VER'] = '2'
    main(args=[
        '--brick',
        '1102p240',
        '--zoom',
        '250',
        '350',
        '1550',
        '1650',
        '--force-all',
        '--no-write',
        '--no-wise',  #'--plots',
        '--survey-dir',
        surveydir,
        '--outdir',
        outdir
    ] + my_extra_args)
    del os.environ['GAIA_CAT_DIR']
    del os.environ['GAIA_CAT_VER']
    fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 4)

    # Check skipping blobs outside the brick's unique area.
    # (this now doesn't detect any sources at all, reasonably)
    # surveydir = os.path.join(os.path.dirname(__file__), 'testcase5')
    # outdir = 'out-testcase5'
    #
    # fn = os.path.join(outdir, 'tractor', '186', 'tractor-1867p255.fits')
    # if os.path.exists(fn):
    #     os.unlink(fn)
    #
    # main(args=['--brick', '1867p255', '--zoom', '0', '150', '0', '150',
    #            '--force-all', '--no-write', '--coadd-bw',
    #            '--survey-dir', surveydir,
    #            '--early-coadds',
    #            '--outdir', outdir] + extra_args)
    #
    # assert(os.path.exists(fn))
    # T = fits_table(fn)
    # assert(len(T) == 1)

    # Custom RA,Dec; blob ra,dec.
    surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
    outdir = 'out-testcase4b'
    # Catalog written with one entry (--blobradec)
    fn = os.path.join(outdir, 'tractor', 'cus',
                      'tractor-custom-186743p25461.fits')
    if os.path.exists(fn):
        os.unlink(fn)
    main(args=[
        '--radec', '186.743965', '25.461788', '--width', '250', '--height',
        '250', '--force-all', '--no-write', '--no-wise', '--blobradec',
        '186.740369', '25.453855', '--survey-dir', surveydir, '--outdir',
        outdir
    ] + extra_args)

    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 1)

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
    outdir = 'out-testcase3'
    checkpoint_fn = os.path.join(outdir, 'checkpoint.pickle')
    if os.path.exists(checkpoint_fn):
        os.unlink(checkpoint_fn)
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', outdir, '--checkpoint', checkpoint_fn,
        '--checkpoint-period', '1', '--threads', '2'
    ] + extra_args)

    # Read catalog into Tractor sources to test read_fits_catalog
    from legacypipe.catalog import read_fits_catalog
    from legacypipe.survey import LegacySurveyData, GaiaSource
    from tractor.galaxy import DevGalaxy
    from tractor import PointSource

    survey = LegacySurveyData(survey_dir=outdir)
    fn = survey.find_file('tractor', brick='2447p120')
    print('Checking', fn)
    T = fits_table(fn)
    cat = read_fits_catalog(T, fluxPrefix='')
    print('Read catalog:', cat)
    assert (len(cat) == 2)
    src = cat[0]
    assert (type(src) == DevGalaxy)
    assert (np.abs(src.pos.ra - 244.77973) < 0.00001)
    assert (np.abs(src.pos.dec - 12.07233) < 0.00001)
    src = cat[1]
    print('Source', src)
    assert (type(src) in [PointSource, GaiaSource])
    assert (np.abs(src.pos.ra - 244.77830) < 0.00001)
    assert (np.abs(src.pos.dec - 12.07250) < 0.00001)
    # DevGalaxy(pos=RaDecPos[244.77975494973529, 12.072348111713127], brightness=NanoMaggies: g=19.2, r=17.9, z=17.1, shape=re=2.09234, e1=-0.198453, e2=0.023652,
    # PointSource(RaDecPos[244.77833280764278, 12.072521274981987], NanoMaggies: g=25, r=23, z=21.7)

    # Check that we can run again, using that checkpoint file.
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', outdir, '--checkpoint', checkpoint_fn,
        '--checkpoint-period', '1', '--threads', '2'
    ] + extra_args)
    # Assert...... something?

    # Test --checkpoint without --threads
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', outdir, '--checkpoint', checkpoint_fn,
        '--checkpoint-period', '1'
    ] + extra_args)

    # MzLS + BASS data
    # surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass')
    # main(args=['--brick', '3521p002', '--zoom', '2400', '2450', '1200', '1250',
    #            '--no-wise', '--force-all', '--no-write',
    #            '--survey-dir', surveydir2,
    #            '--outdir', 'out-mzlsbass'])

    # From Kaylan's Bootes pre-DR4 run
    # surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass3')
    # main(args=['--brick', '2173p350', '--zoom', '100', '200', '100', '200',
    #            '--no-wise', '--force-all', '--no-write',
    #            '--survey-dir', surveydir2,
    #            '--outdir', 'out-mzlsbass3'] + extra_args)

    # With plots!
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', 'out-testcase3', '--plots', '--nblobs', '1'
    ] + extra_args)

    # Decals Image Simulations
    # Uncomment WHEN galsim build for Travis
    #os.environ["DECALS_SIM_DIR"]= os.path.join(os.path.dirname(__file__),'image_sims')
    #brick= '2447p120'
    #sim_main(args=['--brick', brick, '-n', '2', '-o', 'STAR', \
    #               '-ic', '1', '--rmag-range', '18', '26', '--threads', '1',\
    #               '--zoom', '1020', '1070', '2775', '2815'])
    # Check if correct files written out
    #rt_dir= os.path.join(os.getenv('DECALS_SIM_DIR'),brick,'star','001')
    #assert( os.path.exists(os.path.join(rt_dir,'../','metacat-'+brick+'-star.fits')) )
    #for fn in ['tractor-%s-star-01.fits' % brick,'simcat-%s-star-01.fits' % brick]:
    #    assert( os.path.exists(os.path.join(rt_dir,fn)) )
    #for fn in ['image','model','resid','simscoadd']:
    #    assert( os.path.exists(os.path.join(rt_dir,'qa-'+brick+'-star-'+fn+'-01.jpg')) )

    if not travis:
        # With ceres
        main(args=[
            '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
            '--no-wise', '--force-all', '--no-write', '--ceres',
            '--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres'
        ] + extra_args)
Exemplo n.º 14
0
def main():
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('--mzls',
                        action='store_true',
                        help='Set MzLS (default: DECaLS)')
    parser.add_argument('--ann', help='Set annotated-CCDs file')
    opt = parser.parse_args()

    if opt.mzls:
        from mosaic import MosaicNominalCalibration
        from camera_mosaic import database_filename, camera_name
        nom = MosaicNominalCalibration()

        obstatus_fn = 'obstatus/mosaic-tiles_obstatus.fits'
        out_fn = 'mosaic-obstatus-depth.fits'

        bands = 'z'

        declo, dechi = -5, 90

        bad_expid_fn = 'obstatus/bad_expid.txt'

    else:
        from decam import DecamNominalCalibration
        from camera_decam import database_filename, camera_name
        nom = DecamNominalCalibration()

        # ln -s ~/observing/obstatus/bad_expid.txt obstatus/decam-bad_expid.txt

        obstatus_fn = 'obstatus/decam-tiles_obstatus.fits'
        out_fn = 'decam-obstatus-depth.fits'

        bad_expid_fn = 'obstatus/decam-bad_expid.txt'

        bands = 'grz'

        declo, dechi = -20, 35

    f = open(bad_expid_fn)
    bad_expids = set()
    for line in f:
        line = line.strip()
        if len(line) == 0:
            continue
        if line[0] == '#':
            continue
        words = line.split()
        try:
            expnum = int(words[0])
        except:
            print('Skipping line:', line)
            continue
        bad_expids.add(expnum)
    print('Read', len(bad_expids), 'bad exposure numbers')

    # Convert copilot db to fits.
    import obsdb
    from copilot import db_to_fits
    obsdb.django_setup(database_filename=database_filename)
    ccds = obsdb.MeasuredCCD.objects.all()
    copilot = db_to_fits(ccds)
    all_copilot = copilot.copy()
    fn = 'copilot.fits'
    copilot.writeto(fn)
    print('Wrote', fn)

    print(len(copilot), 'measured CCDs in copilot database')
    copilot.cut(np.array([c.strip() == camera_name for c in copilot.camera]))
    print(len(copilot), 'copilot CCDs with camera = "%s"' % camera_name)
    copilot.cut(copilot.expnum > 0)
    print(len(copilot), 'measured CCDs in copilot database with EXPNUM')

    print('Copilot expfactor extremes:',
          np.percentile(copilot.expfactor[copilot.expfactor != 0], [1, 99]))

    survey = LegacySurveyData()
    if opt.ann:
        ccds = fits_table(opt.ann)
    else:
        print('Reading annotated CCDs files...')
        ccds = survey.get_annotated_ccds()
    print(len(ccds), 'CCDs')

    # Fix parsing of OBJECT field to tileid...
    from obsbot import get_tile_id_from_name
    tileids = []
    for o in ccds.object:
        tid = get_tile_id_from_name(o.strip())
        if tid is None:
            tid = 0
        tileids.append(tid)
    tileids = np.array(tileids)
    print(len(np.unique(tileids)),
          'unique tile ids in annotated file, from OBJECT')
    print(len(np.unique(ccds.tileid)),
          'unique tile ids in ann file from TILEID')
    D = np.flatnonzero(tileids != ccds.tileid)
    print(len(D), 'different tileids')
    print('From OBJECT:', tileids[D])
    print('From TILEID:', ccds.tileid[D])
    ccds.tileid = tileids

    O = fits_table(obstatus_fn)
    print(len(O), 'tiles')

    if opt.mzls:
        from camera_mosaic import fix_expnums
        # Fix MzLS exposure numbers with wrong leading "3".
        fix_expnums(ccds.expnum)
        # Also fix leading "3" in expnums in OBSTATUS file
        fix_expnums(O.z_expnum)
        # And copilot database
        fix_expnums(copilot.expnum)

        print('Z_EXPNUM range:', O.z_expnum.min(), 'min >0:',
              O.z_expnum[O.z_expnum > 0].min(), O.z_expnum.max())

    print('Pass numbers:', np.unique(O.get('pass')))

    if opt.mzls:
        goodtiles = (O.in_desi * (O.dec > 30) * (O.get('pass') <= 3))
        print(sum(goodtiles), 'tiles of interest')
    else:
        goodtiles = (O.in_desi * (O.get('pass') <= 3))
        print(sum(goodtiles), 'tiles in the footprint')

    #O.cut(goodtiles)
    #print('Cut to', len(O), 'tiles of interest')

    # *after* fixing tileids
    allccds = ccds.copy()

    # Map tile IDs back to index in the obstatus file.
    tileid_to_index = np.empty(max(O.tileid) + 1, int)
    tileid_to_index[:] = -1
    tileid_to_index[O.tileid] = np.arange(len(O))

    assert (len(np.unique(O.tileid)) == len(O))

    I = tileid_to_index[O.tileid]
    assert (np.all(I == np.arange(len(O))))

    # Look at whether exposures from other programs are near our tile centers.
    # Basically nope.
    # plt.clf()
    # e,K = np.unique(ccds.expnum, return_index=True)
    # I,J,d = match_radec(O.ra, O.dec, ccds.ra_bore[K], ccds.dec_bore[K],
    #                     1./60., nearest=True)
    # KK = K[np.flatnonzero(ccds.tileid[K] > 0)]
    # I,J,d2 = match_radec(O.ra, O.dec, ccds.ra_bore[KK], ccds.dec_bore[KK],
    #                      1./60., nearest=True)
    # ha = dict(range=(0., 60.), bins=60, histtype='step')
    # plt.hist(d * 3600., color='b', **ha)
    # plt.hist(d2 * 3600., color='r', **ha)
    # plt.xlabel('Distance from tile to nearest DECam boresight (arcsec)')
    # plt.savefig('dists.png')

    notileids = ccds[ccds.tileid <= 0]
    print(len(notileids), 'CCDs have no tileid')
    I, J, d = match_radec(notileids.ra_bore,
                          notileids.dec_bore,
                          O.ra,
                          O.dec,
                          0.5,
                          nearest=True)

    plt.clf()
    plt.hist(d, bins=50)
    plt.xlabel('Distance to nearest tile center (deg)')
    plt.savefig('tiledist.png')

    plt.clf()
    plt.hist(d * 3600, bins=50, range=(0, 30))
    plt.xlabel('Distance to nearest tile center (arcsec)')
    plt.savefig('tiledist2.png')

    ccds.cut(ccds.tileid > 0)
    print(len(ccds), 'CCDs with tileid')

    expnums, I = np.unique(ccds.expnum, return_index=True)
    print(len(expnums), 'unique exposures (with tileids)')

    ccds.photometric = (ccds.ccd_cuts == 0)

    # Compute the mean depth per exposure
    E = ccds[I]
    for expnum in expnums:
        I = np.flatnonzero(ccds.expnum == expnum)
        j = np.flatnonzero(E.expnum == expnum)
        assert (len(j) == 1)
        j = j[0]
        E.photometric[j] = np.all(ccds.photometric[I])
        #E.photometric[j] = np.all(ccds.ccd_cuts[I] == 0)
        if len(np.unique(ccds.photometric[I])) == 2:
            print('Exposure', expnum,
                  'has photometric and non-photometric CCDs')
            non = I[ccds.photometric[I] == False]
            phot = I[ccds.photometric[I]]

            if opt.mzls and len(phot) == 3:
                print('Accepting an exposure with 3 good CCDs')
                E.photometric[j] = True
                # And remove this exposure from the bad_expid list.
                if expnum in bad_expids:
                    bad_expids.remove(expnum)
                    print('Removing exposure', expnum, 'from bad_expid file')

                continue

            for ii in non:
                print(
                    '    http://legacysurvey.org/viewer-dev/?ra=%.3f&dec=%.3f&zoom=11&ccds3&bad=%i-%s'
                    % (ccds.ra_center[ii], ccds.dec_center[ii], expnum,
                       ccds.ccdname[ii]))
                print(
                    '    http://legacysurvey.org/viewer-dev/ccd/decals-dr5/decam-%s-%s-%s/'
                    % (ccds.expnum[ii], ccds.ccdname[ii], ccds.filter[ii]))
            print('  image:', ccds.image_filename[I][0])
            print('  boresight:', ccds.ra_bore[I][0], ccds.dec_bore[I][0])
            #print('  ccdnames:', ccds.ccdname[I])
            print('  photometric:', len(phot), ', non-photometric:', len(non))
            print('  median phot depth:', np.median(ccds.galdepth[phot]))
            #print('  depth:', ccds.galdepth[I])
            print('  non-photometric CCDs:', ccds.ccdname[non])
            print('    depths:', ccds.galdepth[non])
            print('    ccdnmatch', ccds.ccdnmatch[non], 'vs',
                  ccds.ccdnmatch[phot])
            print('    ccdtransp:', ccds.ccdtransp[non], 'vs',
                  ccds.ccdtransp[phot])
            print('    ccd zpt vs frame zpt:',
                  ccds.ccdzpt[non] - ccds.zpt[non])
            dp = ccds.ccdzpt[phot] - ccds.zpt[phot]
            print('      phot ccds zpt vs frame: range', dp.min(), dp.max(),
                  'mean', dp.mean())

            whitelist = [
                346662,
                346664,
                346665,  # S3/S29 striping
                346754,  # one bad chip, wispy
                346967,
                347304,  # M5 globular
                347664,  # zpt scatter
                347744,  # weird eye-shaped ghost; but lots of cov.
                347755,
                347768,
                347769,
                347782,  # shallow, zpt scatter -- wispy pattern on focal plane
                347918,
                347920,  # straddling transparency cut
                347934,
                347936,
                347941,
                347945,
                347947,  # zpt scatter
                392377,
                392380,
                393173,  # bright star
                393671,  # bright star
                393672,
                393673,  # scatter
                425339,
                425340,  # strangely low ccdnmatch
                426225,
                430808,  # globular cluster
                431640,  # bright star
                431644,  # globular
                432154,  # one amp high bias
                432159,  # transp. on boundary
                432179,  # one amp high bias, +
                432747,
                432748,
                432751,  # scatter
                433305,
                433306,  # bright star
                497062,
                497064,
                497065,  # low ccdnmatch
                509516,
                509517,  # bright star
                511247,
                511263,  # low ccdnmatch
                511513,
                511514,  # bright star
                512303,  # bright star
                520560,  # bright star
                521782,  # scatter
                522212,  # bright star
                535138,  # bright stars, satellite hits?
                535141,
                535142,
                535143,
                535149,  # low ccdnmatch
                535210,  # bright star
                535695,  # globular
                536065,  # globular
                536385,  # strangely zero ccdnmatch
                547761,  # nice galaxy
                548257,  # bright star
                553779,  # scatter 
                553795,  # shallow
                554284,  # marginal zpt
                563659,  # zpt scatter
                563850,  # mild striping
                563852,  # ??
                583118,  # bright stars?
                592621,  # marginal zpt, scatter
                592859,  # some pattern noise
                605068,  # ??
                625710,  # strangely low ccdnmatch
                631005,  # bright star
                634493,  # globular
                634786,  # strangely low ccdnmatch
                634877,  # globular
                635535,  # bright star, glob
                635962,  # low ccdnmatch
                635973,  # bias level?
                636018,  # bias level?
                637688,  # bright star
            ]
            blacklist = [
                425328,  # 2.7" seeing
                488244,
                488256,
                488260,
                488261,
                488263,
                488268,  # weird striping
                488270,  # weird striping
                496913,
                496917,
                496918,
                496919,
                496920,
                496921,
                496922,  # 3" seeing
                496923,
                496925,
                496926,
                496927,
                496928,
                496930,  # 3" seeing
                509162,
                509163,
                509166,
                509172,
                509176,
                509182,
                509202,  # 3" seeing
                535471,  # 4" seeing!
                535498,  # 3" seeing
                548218,  # double PSF -- telescope moved?
                563835,  # striping
                563842,  # striping
            ]

            if expnum in whitelist:
                print('** Exposure', expnum,
                      'in whitelist -- marking as photometric')
                E.photometric[j] = True

        # Don't include zeros in computing average depths!
        Igood = I[(ccds.galdepth[I] > 0) * (ccds.ccdzpt[I] < 30)]
        if len(Igood) > 0:
            E.galdepth[j] = np.mean(ccds.galdepth[Igood])
        else:
            E.galdepth[j] = 0.
    del expnums

    keep = np.array([not (expnum in bad_expids) for expnum in ccds.expnum])
    ccds.cut(keep)
    print(len(ccds), 'CCDs NOT in the bad_expids file')

    keep = np.array([not (expnum in bad_expids) for expnum in copilot.expnum])
    copilot.cut(keep)
    print(len(copilot), 'copilot exposures NOT in the bad_expids file')

    keep = np.array([not (expnum in bad_expids) for expnum in E.expnum])
    E.cut(keep)
    print(len(E), 'CCD Exposures NOT in the bad_expids file')

    # plt.clf()
    # plt.plot(O.ra, O.dec, 'k.')
    # plt.axis([360,0,-25,35])
    # plt.title('All tiles')
    # plt.savefig('tiles-all.png')
    #
    # print('in_desi:', np.unique(O.in_desi))
    # plt.clf()
    # J = np.flatnonzero(O.in_desi == 1)
    # plt.plot(O.ra[J], O.dec[J], 'k.')
    # plt.axis([360,0,-25,35])
    # plt.title('In DESI')
    # plt.savefig('tiles-desi.png')
    #
    # print('in_des:', np.unique(O.in_des))
    # plt.clf()
    # J = np.flatnonzero(O.in_des == 1)
    # plt.plot(O.ra[J], O.dec[J], 'k.')
    # plt.axis([360,0,-25,35])
    # plt.title('IN DES')
    # plt.savefig('tiles-des.png')

    #print('Number of exposures of each tile:')
    #print(Counter(E.tileid).most_common())
    print()
    print()
    print('Number of exposures of tiles:')
    for band in bands:
        I = np.flatnonzero(E.filter == band)
        c = Counter(E.tileid[I])
        c2 = Counter([v for k, v in c.most_common()])
        print('  ', band, 'band:', c2.most_common())

    # Detection inverse-variance is the quantity that adds when there are
    # multiple exposures.
    #   detsig1 = ccds.sig1 / ccds.galnorm_mean
    #   depth = 5. * detsig1
    #   # that's flux in nanomaggies -- convert to mag
    #   ccds.galdepth = -2.5 * (np.log10(depth) - 9)

    with np.errstate(divide='ignore', over='ignore'):
        # actually 5*detsig1...
        detsig = 10.**((E.galdepth - 22.5) / -2.5)
        E.detiv = 1. / detsig**2
        E.detiv[E.galdepth == 0] = 0.
        print('Smallest detivs:', E.detiv[np.argsort(E.detiv)[:10]])
        print('w/ galdepths:', E.galdepth[np.argsort(E.detiv)[:10]])
        print('Smallest positive detivs:',
              E.detiv[np.argsort(E.detiv + 1e12 * (E.detiv == 0))[:10]])
        print('w/ galdepths:',
              E.galdepth[np.argsort(E.detiv + 1e12 * (E.detiv == 0))[:10]])

    for band in bands:
        print()
        print('------------------')
        print(band, 'band.')

        # "I" indexes into exposures E.
        I = np.flatnonzero(
            (E.filter == band) * E.photometric * np.isfinite(E.detiv))
        print(len(I), 'photometric exposures in', band)

        # "iv" is parallel to O; will be converted to "galdepth".
        iv = np.zeros(len(O), np.float32)
        # "J" indexes into obstatus tiles O.
        J = tileid_to_index[E.tileid[I]]
        assert (np.all((J >= 0) * (J < len(O))))
        assert (np.all(O.tileid[J] == E.tileid[I]))

        #print('tileid range', E.tileid[I].min(), E.tileid[I].max())
        # d = np.array([degrees_between(*a) for a in
        #               zip(E.ra_bore[I], E.dec_bore[I], O.ra[J], O.dec[J])])
        # print('Degrees between tiles & exposures:', d)

        np.add.at(iv, J, E.detiv[I])
        print('galdepth range:', E.galdepth[I].min(), E.galdepth[I].max())
        print('detiv range:', E.detiv[I].min(), E.detiv[I].max())
        #print('index range:', J.min(), J.max())
        nexp = np.zeros(len(O), int)
        np.add.at(nexp, J, 1)
        print('tile exposure counts:', Counter(nexp))

        # convert iv back to galdepth in mags
        with np.errstate(divide='ignore'):
            galdepth = -2.5 * (np.log10(np.sqrt(1. / iv)) - 9)
            galdepth[iv == 0] = 0.

        # Shallowest before extinction correction
        #I = np.argsort(iv + 1e6*(iv == 0))
        I = np.argsort(galdepth + 50. * (galdepth == 0))
        print(
            'Shallowest depth estimates from annotated CCDs file, before extinction:'
        )
        for i in I[:10]:
            print('  ', galdepth[i], 'iv', iv[i], 'tile', O.tileid[i],
                  'expnum',
                  O.get('%s_expnum' % band)[i])
            e = O.get('%s_expnum' % band)[i]
            j = np.flatnonzero(E.expnum == e)
            print('    galdepth', E.galdepth[j])

        fid = nom.fiducial_exptime(band)
        extinction = O.ebv_med * fid.A_co
        #print('Extinction range:', extinction.min(), extinction.max())
        galdepth -= extinction

        galdepth[iv == 0] = 0.

        # Shallowest galdepth > 0
        I = np.argsort(galdepth + 50. * (galdepth == 0))
        print('Shallowest depth estimates from annotated CCDs file:')
        for i in I[:10]:
            print('  ', galdepth[i], 'tile', O.tileid[i], 'expnum',
                  O.get('%s_expnum' % band)[i])

        #print('galdepth deciles:', np.percentile(galdepth, [0,10,20,30,40,50,60,70,80,90,100]))

        # Z_DONE, Z_EXPNUM but no Z_DEPTH
        missing_depth = np.flatnonzero(
            (O.get('%s_expnum' % band) > 0) * (O.get('%s_done' % band) == 1) *
            (galdepth == 0))
        print('Found', len(missing_depth), 'tiles with', band,
              'DONE and EXPNUM but no DEPTH; setting to DEPTH=30')
        print('  eg, EXPNUMs',
              O.get('%s_expnum' % band)[missing_depth[:10]], 'DATE',
              O.get('%s_date' % band)[missing_depth[:10]])
        # Don't actually update 'galdepth[missing_depth]' until after this next check...

        # Flag tiles that have *only* non-photometric exposures with depth = 1.
        I = np.flatnonzero((E.filter == band) * np.logical_not(E.photometric))
        print(len(I), 'exposures are non-photometric in', band, 'band')
        J = tileid_to_index[E.tileid[I]]
        only_nonphot = J[galdepth[J] == 0.]
        print(len(only_nonphot), 'tiles have only non-photometric exposures')
        print('Marking', len(only_nonphot), 'non-photometric tiles in', band,
              'with depth=1')

        orig_galdepth = galdepth.copy()

        galdepth[missing_depth] = 30.
        galdepth[only_nonphot] = 1.

        J = tileid_to_index[E.tileid[I]]
        nonphot = (galdepth[J] == 1.)
        print('Non-photometric galdepths:', E.galdepth[I])
        print('Non-photometric galdepths:', E.galdepth[I[nonphot]])
        plt.clf()
        phot = np.flatnonzero((E.filter == band) * E.photometric)
        plt.hist(E.galdepth[phot],
                 range=(18, 26),
                 bins=50,
                 histtype='step',
                 color='b',
                 label='Photometric')
        plt.hist(E.galdepth[I[nonphot]],
                 range=(18, 26),
                 bins=50,
                 histtype='step',
                 color='r',
                 label='Non-phot')
        plt.legend()
        plt.savefig('nonphot-%s.png' % band)

        # expnum_to_copilot = np.empty(expnums.max()+1, int)
        # expnum_to_copilot[:] = -1
        # expnum_to_copilot[copilot.expnum] = np.arange(len(copilot))
        expnum_to_copilot = dict([(e, i)
                                  for i, e in enumerate(copilot.expnum)])

        if False:
            # Let's check the accuracy of the copilot's depth estimates...
            target_exptime = copilot.expfactor * fid.exptime
            # What fraction of the target exposure time did we take?
            depth_factor = copilot.exptime / target_exptime
            nomdepth = fid.single_exposure_depth
            depth = nomdepth + 2.5 * np.log10(np.sqrt(depth_factor))
            #print('Copilot predicted depths:', depth)
            IC = np.array(
                [expnum_to_copilot.get(e, -1) for e in allccds.expnum])
            K = np.flatnonzero(IC >= 0)
            ext = np.array([
                e['ugrizY'.index(f)]
                for e, f in zip(allccds.decam_extinction, allccds.filter)
            ])
            dd = allccds.galdepth - ext
            print('Making scatterplot...', len(K), 'points')
            plt.clf()
            #plt.plot(dd[K], depth[IC[K]], 'b.', alpha=0.2, mec='none')
            plt.scatter(dd[K],
                        depth[IC[K]],
                        c=np.clip(copilot.expfactor[IC[K]], 0, 2),
                        s=10,
                        alpha=0.2,
                        edgecolors='none')
            plt.colorbar()
            plt.xlabel('Pipeline depth')
            plt.ylabel('Copilot depth proxy')
            plt.plot([20, 25], [20, 25], 'k-', alpha=0.25)
            plt.plot([20, 25], [20 + 0.1, 25 + 0.1], 'k--', alpha=0.25)
            plt.plot([20, 25], [20 - 0.1, 25 - 0.1], 'k--', alpha=0.25)
            plt.axis([20.5, 23, 21, 23.5])
            plt.title(
                'Copilot vs Pipeline depth estimates.  (color = exp.factor)')
            plt.savefig('depth-copilot-%s.png' % band)
            print('Made scatterplot')

        plt.clf()
        ha = dict(bins=60, range=(0, 30), log=True, histtype='step')
        plt.hist(O.get('%s_depth' % band), color='k', label='Before', **ha)
        plt.hist(orig_galdepth, color='b', label='Annotated CCDs', **ha)

        # Do we have measurements for any of these missing tiles in the copilot db?
        for code in [30, 0]:
            Igal = np.flatnonzero(
                (O.get('%s_expnum' % band) > 0) *
                (O.get('%s_done' % band) == 1) * (galdepth == code))
            expnums = O.get('%s_expnum' % band)[Igal]
            print(
                len(expnums),
                'still marked DONE, with EXPNUM, but missing DEPTH, with code =',
                code)

            Ihuh = np.flatnonzero(
                (O.get('%s_done' % band) == 1) * (galdepth == code))
            print(len(Ihuh), 'tiles marked DONE, without EXPNUM, and DEPTH =',
                  code)

            if len(Ihuh):
                print('Tile ids:', O.tileid[Ihuh])
                for t in O.tileid[Ihuh]:
                    I = np.flatnonzero(E.tileid == t)
                    print('  tile', t, ': exposure numbers:', E.expnum[I])
                    print('  with depths', E.galdepth[I])
                    i = tileid_to_index[t]
                    if i >= 0:
                        print('    depth', galdepth[i])
                    else:
                        print('    no depth')

                # Within an arcmin?
                I, J, d = match_radec(O.ra[Ihuh],
                                      O.dec[Ihuh],
                                      all_copilot.rabore,
                                      all_copilot.decbore,
                                      1. / 60.,
                                      nearest=True)
                print('For', len(Ihuh), 'weird tiles,')
                print(len(I), 'matches within an arcmin in the copilot db')
                print('Smallest distances:', d[np.argsort(d)[:10]])

                I, J, d = match_radec(O.ra[Ihuh],
                                      O.dec[Ihuh],
                                      allccds.ra_bore,
                                      allccds.dec_bore,
                                      1. / 60.,
                                      nearest=True)
                print(len(I), 'matches within an arcmin in the CCDs table')
                print('Smallest distances:', d[np.argsort(d)[:10]])

                O[Ihuh].writeto('weird-%s-%i.fits' % (band, code))

            if len(expnums) == 0:
                continue
            IC = np.array([expnum_to_copilot.get(e, -1) for e in expnums])
            K = np.flatnonzero(IC >= 0)
            expnums = expnums[K]
            # these are the indices into O / galdepth
            Igal = Igal[K]
            co = copilot[IC[K]]
            print(len(expnums), 'matched to copilot database')

            target_exptime = co.expfactor * fid.exptime
            # What fraction of the target exposure time did we take?
            depth_factor = co.exptime / target_exptime
            nomdepth = fid.single_exposure_depth
            print('Nominal single-exposure depth:', nomdepth)
            co.depth = nomdepth + 2.5 * np.log10(np.sqrt(depth_factor))
            print('Copilot predicted depths:', co.depth)
            J = np.flatnonzero(np.isfinite(co.depth))
            co = co[J]
            # indices into O
            Igal = Igal[J]
            print(len(Igal), 'good copilot depth estimates')

            pcts = [0, 1, 5, 25, 50, 75, 95, 99, 100]
            print('Copilot depth percentiles:', np.percentile(co.depth, pcts))

            print('Shallowest exposures:')
            I = np.argsort(co.depth)
            for i in I[:10]:
                print('  Expnum', co.expnum[i], 'depth', co.depth[i],
                      'exptime', co.exptime[i])
            co[I].writeto('depths.fits')

            from astrometry.util.starutil_numpy import mjdtodate
            print('Copilot-matched entries:')
            I = np.argsort(co.expnum)
            for i in I:
                print('  EXPNUM', co.expnum[i], 'date',
                      mjdtodate(co.mjd_obs[i]), '  copilot name',
                      co.filename[i])
                # e = co.expnum[i]
                # I = np.flatnonzero(allccds.expnum == e-1)
                # fn1 = None
                # fn2 = None
                # if len(I):
                #     print('    CCDs file contains', len(I), 'entries for expnum', e-1)
                #     print('      filename', allccds.image_filename[I[0]])
                #     fn1 = allccds.image_filename[I[0]]
                # else:
                #     print('    No CCDs file entries for expnum', e-1)
                # I = np.flatnonzero(allccds.expnum == e+1)
                # if len(I):
                #     print('    CCDs file contains', len(I), 'entries for expnum', e+1)
                #     print('      filename', allccds.image_filename[I[0]])
                #     fn2 = allccds.image_filename[I[0]]
                # else:
                #     print('    No CCDs file entries for expnum', e+1)
                #
                # if fn1 is not None and fn2 is not None:
                #     full1 = os.path.join(survey.get_image_dir(), fn1)
                #     #print('Full path 1:', full1)
                #     if os.path.exists(full1):
                #         print('exists')
                #     full2 = os.path.join(survey.get_image_dir(), fn2)
                #     #print('Full path 2:', full2)
                #     if os.path.exists(full2):
                #         print('exists')
                #     if os.path.exists(full1) and os.path.exists(full2):
                #         dir1 = os.path.dirname(full1)
                #         dir2 = os.path.dirname(full2)
                #         if dir1 == dir2:
                #             #print('dir:', dir1)
                #             fns = os.listdir(dir1)
                #             fns.sort()
                #             fns = [fn for fn in fns if ('oki' in fn or 'ooi' in fn)]
                #             base1 = os.path.basename(full1)
                #             base2 = os.path.basename(full2)
                #             i1 = fns.index(base1)
                #             i2 = fns.index(base2)
                #             print('Files found at list elements', i1, i2)
                #             #print(fns[i1:i2+1])
                #             for fn in fns[i1:i2+1]:
                #                 print('EXPNUM', e, 'range', os.path.join(os.path.dirname(fn1), fn))
                #             if i1 + 4 == i2:
                #                 print('EXPNUM', e, 'expected', os.path.join(os.path.dirname(fn1), fns[i1+2]))
                #             if i1 + 2 == i2:
                #                 print('EXPNUM', e, 'expected', os.path.join(os.path.dirname(fn1), fns[i1+1]))

            #print('Before:', galdepth[Igal])
            galdepth[Igal] = co.depth
            #print('After:', galdepth[Igal])

            Igal = np.flatnonzero(
                (O.get('%s_expnum' % band) > 0) *
                (O.get('%s_done' % band) == 1) * (galdepth == code))
            expnums = O.get('%s_expnum' % band)[Igal]
            print(
                len(expnums),
                'still marked DONE, with EXPNUM, but missing DEPTH with code =',
                code, 'after copilot patching')
            print('Exposure numbers:', expnums)
            print('Exposure dates:', O.get('%s_date' % band)[Igal])

            print('Date counter:',
                  Counter(O.get('%s_date' % band)[Igal]).most_common())

        O.set('%s_depth' % band, galdepth)

        plt.hist(O.get('%s_depth' % band), color='r', label='After', **ha)
        plt.savefig('depth-hist-%s.png' % band)

        #print('Depth deciles: [', ', '.join(['%.3f' % f for f in np.percentile(O.get('%s_depth' % band), [0,10,20,30,40,50,60,70,80,90,100])]) + ']')

        rlo, rhi = 0, 360
        dlo, dhi = declo, dechi
        J = np.flatnonzero(
            (O.in_desi == 1) * (O.in_des == 0) * (O.dec > dlo) * (O.dec < dhi))
        print('Median E(B-V) in DECaLS area:', np.median(O.ebv_med[J]))
        print('Median extinction in DECaLS area, %s band:' % band,
              np.median(extinction[J]))

        I2 = np.flatnonzero((O.get('%s_expnum' % band) > 0) *
                            (O.get('%s_depth' % band) == 30) *
                            (O.get('%s_done' % band) == 1) * (O.in_desi == 1))
        print(len(I2), 'with EXPNUM and DONE and IN_DESI, but no DEPTH')
        # Sort by expnum
        I2 = I2[np.argsort(O.get('%s_expnum' % band)[I2])]
        print('Exposure numbers:', sorted(O.get('%s_expnum' % band)[I2]))
        print('Dates:', sorted(O.get('%s_date' % band)[I2]))
        print('Dates:', np.unique(O.get('%s_date' % band)[I2]))

        for i in I2:
            print('  date',
                  O.get('%s_date' % band)[i], 'expnum',
                  O.get('%s_expnum' % band)[i])

        # for i2,o in zip(I2, O[I2]):
        #     print()
        #     e = o.get('%s_expnum' % band)
        #     print('  Expnum', e, 'orig galdepth', orig_galdepth[i2])
        #     date = o.get('%s_date' % band)
        #     print('  Date', date)
        #     print('  Pass', o.get('pass'), 'Tile', o.tileid)
        #     jj = np.flatnonzero(allccds.expnum == e)
        #     print('  In DESI:', o.in_desi, 'In DES:', o.in_des)
        #     print('  ', len(jj), 'matching CCDs')
        #     if len(jj) == 0:
        #         continue
        #     print('  CCDs OBJECT', [ob.strip() for ob in allccds.object[jj]])
        #     print('  CCDs Tileid', allccds.tileid[jj])
        #     print('  CCDs galdepth', allccds.galdepth[jj])
        #     print('  CCDs photometric', allccds.photometric[jj])
        #
        #     ii = np.flatnonzero(E.expnum == e)
        #     print('  ', len(ii), 'Exposures matching')
        #     if len(ii):
        #         ee = E[ii[0]]
        #         print('  exposure tileid', ee.tileid)
        #         print('  index', tileid_to_index[ee.tileid])
        #         print('  vs i2=', i2)
        #         print('  only_nonphot', only_nonphot[i2], 'missing_depth', missing_depth[i2])
        #
        #     kk = np.flatnonzero(allccds.tileid == o.tileid)
        #     kk = np.array(sorted(set(kk) - set(jj)))
        #     print('  ', len(kk), 'other CCDs of this tile')
        # #print('Dates:', O.get('%s_date' % band)[I])

        from astrometry.util.plotutils import antigray
        rr, dd = np.meshgrid(np.linspace(rlo, rhi, 720),
                             np.linspace(dlo, dhi, 360))
        JJ, II, d = match_radec(rr.ravel(),
                                dd.ravel(),
                                O.ra,
                                O.dec,
                                1.5,
                                nearest=True)
        indesi = np.zeros(rr.shape, bool)
        indesi.flat[JJ] = ((O.in_desi[II] == 1) * (O.in_des[II] == 0))

        plt.figure(figsize=(14, 6))
        plt.subplots_adjust(left=0.1, right=0.99)

        for passnum in [1, 2, 3]:
            print('Pass', passnum)
            plt.clf()

            J = np.flatnonzero(
                (O.in_desi == 1) * (O.in_des == 0) * (O.dec > dlo) *
                (O.dec < dhi) * (O.get('pass') == passnum))
            #plt.plot(O.ra[J], O.dec[J], 'k.', alpha=0.5)

            # Plot the gray background showing the in_desi footprint
            plt.imshow(indesi,
                       extent=[rlo, rhi, dlo, dhi],
                       vmin=0,
                       vmax=4,
                       cmap=antigray,
                       aspect='auto',
                       interpolation='nearest',
                       origin='lower')
            depth = O.get('%s_depth' % band)
            #J = np.flatnonzero((O.get('pass') == passnum) * (depth > 0))

            J = np.flatnonzero(
                (O.get('pass') == passnum) * (depth > 1) * (depth < 30))
            # print('Depths:', depth[J])
            pct = np.percentile(depth[J],
                                [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100])
            #print('Depth deciles:', np.percentile(depth[J], [0,10,20,30,40,50,60,70,80,90,100]))
            print('Depth deciles: [',
                  ', '.join(['%.3f' % f for f in pct]) + ']')

            if len(J) == 0:
                sys.exit(0)

            target = fid.single_exposure_depth
            print('Target depth:', target)

            cmap = cmap_discretize('RdBu', 11)
            dm = 0.275

            plt.scatter(O.ra[J],
                        O.dec[J],
                        c=depth[J] - target,
                        linewidths=0,
                        cmap=cmap,
                        vmin=-dm,
                        vmax=+dm,
                        zorder=-10,
                        s=1)
            plt.colorbar(ticks=np.arange(-0.25, 0.251, 0.05))

            hh, ww = rr.shape
            rgba = np.zeros((hh, ww, 4), np.float32)
            JJ, II, d = match_radec(rr.ravel(),
                                    dd.ravel(),
                                    O.ra[J],
                                    O.dec[J],
                                    1.,
                                    nearest=True)
            Jy, Jx = np.unravel_index(JJ, rr.shape)
            rgba[Jy, Jx, :] = cmap((np.clip(depth[J[II]] - target, -dm, dm) -
                                    (-dm)) / (dm - (-dm)))
            plt.imshow(rgba,
                       extent=[rlo, rhi, dlo, dhi],
                       aspect='auto',
                       interpolation='nearest',
                       origin='lower')

            I = np.flatnonzero((depth == 0) * (O.get('%s_done' % band) == 1) *
                               (O.get('pass') == passnum))
            plt.plot(O.ra[I], O.dec[I], 'g.')

            plt.title('Band %s, Pass %i' % (band, passnum))
            plt.xlabel('RA (deg)')
            plt.ylabel('Dec (deg)')
            plt.axis([rhi, rlo, dlo, dhi])
            plt.savefig('depth-%s-%i.png' % (band, passnum))

        plt.clf()

        print('Fiducial single-exposure-depth:', fid.single_exposure_depth)

        for passnum in [1, 2, 3]:
            depth = O.get('%s_depth' % band)
            J = np.flatnonzero(
                (O.get('pass') == passnum) * (depth > 1) * (depth < 30))
            depth = depth[J]

            print('Pass', passnum)
            print(sum(depth < fid.single_exposure_depth - 0.25), 'of',
                  len(depth), 'tiles are more than 0.25 mag shallow')

            odepth = O.get('%s_depth' % band)
            K = np.flatnonzero(
                (O.get('%s_done' % band) == 0) * (O.get('pass') == passnum) *
                (odepth > 1) * (odepth < 30))
            print(sum(odepth[K] < fid.single_exposure_depth - 0.25), 'of',
                  len(odepth[K]),
                  'DONE=0 tiles are more than 0.25 mag shallow')

            for k in K:
                print('  EXPNUM',
                      O.get('%s_expnum' % band)[k], 'DATE',
                      O.get('%s_date' % band)[k], 'DEPTH',
                      O.get('%s_depth' % band)[k])

            K = np.flatnonzero(
                (O.get('%s_done' % band) == 1) * (O.get('pass') == passnum) *
                (odepth > 1) * (odepth < 30))
            print(sum(odepth[K] < fid.single_exposure_depth - 0.25), 'of',
                  len(odepth[K]),
                  'DONE=1 tiles are more than 0.25 mag shallow')

            K = np.flatnonzero((O.get('%s_done' % band) == 1) *
                               (O.get('pass') == passnum) * (odepth == 1))
            print(len(K), 'DONE=1 tiles have DEPTH=1 (non-photometric)')

            K = np.flatnonzero((O.get('%s_done' % band) == 1) *
                               (O.get('pass') == passnum) * (odepth == 30))
            print(len(K), 'DONE=1 tiles have DEPTH=30 (unknown depth)')

            K = np.flatnonzero((O.get('%s_done' % band) == 1) *
                               (O.get('pass') == passnum) * (odepth == 0))
            print(len(K), 'DONE=1 tiles have DEPTH=0')

            K = np.flatnonzero((O.get('%s_done' % band) == 0) *
                               (O.get('pass') == passnum) * (odepth != 0))
            print(len(K), 'tiles have DONE=0 but DEPTH != 0')

            mlo, mhi = 21, 24
            plt.hist(np.clip(depth, mlo, mhi),
                     bins=100,
                     range=(mlo, mhi),
                     histtype='step',
                     color=' bgr'[passnum],
                     label='Pass %i' % passnum)
        plt.axvline(fid.single_exposure_depth, color='k')
        plt.axvline(fid.single_exposure_depth - 0.25,
                    color='k',
                    linestyle='--')
        plt.xlabel('Depth (mag)')
        plt.legend(loc='upper left')
        plt.title('Depth: %s' % band)
        plt.savefig('depth-%s.png' % band)

        for passnum in [1, 2, 3]:
            depth = O.get('%s_depth' % band)

            roi = ((O.in_desi == 1) * (O.in_des == 0) * (O.dec > dlo) *
                   (O.dec < dhi) * (O.get('pass') == passnum))

            J = np.flatnonzero(roi)

            done = np.flatnonzero(roi * (O.get('%s_done' % band) == 1))

            redo = np.flatnonzero(roi * np.logical_or(
                (depth > 1) * (depth < 30) *
                (depth < fid.single_exposure_depth - 0.25), depth == 1))

            print(
                'Band %s, pass %i: total tiles %i, done %i, redo %i, keep %i' %
                (band, passnum, len(J), len(done), len(redo),
                 len(done) - len(redo)))

            A = np.flatnonzero(roi * (depth > 1) * (depth < 30) *
                               (depth > fid.single_exposure_depth - 0.25))
            B = np.flatnonzero(roi * (depth > 1) * (depth < 30) *
                               (depth <= fid.single_exposure_depth - 0.25))

            C = np.flatnonzero(roi * (depth == 1))

            D = np.flatnonzero(roi * (depth == 30))

            print(
                'Band %s, pass %i: total tiles: %i, A: %i, B: %i, C: %i, D: %i'
                % (band, passnum, len(J), len(A), len(B), len(C), len(D)))

            plt.clf()
            plt.plot(O.ra[J], O.dec[J], 'ko', alpha=0.1)
            plt.plot(O.ra[done], O.dec[done], 'k.')
            plt.plot(O.ra[redo], O.dec[redo], 'r.')
            plt.axis([360, 0, -20, 38])
            plt.title('Tiles to redo: %s band, pass %i: %i of %i' %
                      (band, passnum, len(redo), len(done)))
            plt.savefig('redo-%s-%i.png' % (band, passnum))

            if band == 'z':

                redo = np.flatnonzero(
                    (O.get('pass') == passnum) * np.logical_or(
                        (depth > 1) * (depth < 30) *
                        (depth < fid.single_exposure_depth - 0.5), depth == 1))

                A = np.flatnonzero(roi * (depth > 1) * (depth < 30) *
                                   (depth > fid.single_exposure_depth - 0.5))
                B = np.flatnonzero(roi * (depth > 1) * (depth < 30) *
                                   (depth <= fid.single_exposure_depth - 0.5))

                print(
                    'Band %s, pass %i: total tiles: %i, A: %i, B: %i, C: %i, D: %i (shallow = 0.5 mag less than target)'
                    % (band, passnum, len(J), len(A), len(B), len(C), len(D)))

                plt.clf()
                plt.plot(O.ra[J], O.dec[J], 'ko', alpha=0.1)
                plt.plot(O.ra[done], O.dec[done], 'k.')
                plt.plot(O.ra[redo], O.dec[redo], 'r.')
                plt.axis([360, 0, -20, 38])
                plt.title(
                    'Tiles to redo (> 0.5 mag shallow): %s band, pass %i: %i of %i'
                    % (band, passnum, len(redo), len(done)))
                plt.savefig('redo2-%s-%i.png' % (band, passnum))

        print('Passes 1-3 combined:')
        depth = O.get('%s_depth' % band)
        J = np.flatnonzero((depth > 1) * (depth < 30))
        depth = depth[J]
        print(sum(depth < fid.single_exposure_depth - 0.25), 'of', len(depth),
              'tiles are more than 0.25 mag shallow')

        odepth = O.get('%s_depth' % band)
        K = np.flatnonzero(
            (O.get('%s_done' % band) == 0) * (odepth > 1) * (odepth < 30))
        print(sum(odepth[K] < fid.single_exposure_depth - 0.25), 'of',
              len(odepth[K]), 'DONE=0 tiles are more than 0.25 mag shallow')

        for k in K:
            print('  EXPNUM',
                  O.get('%s_expnum' % band)[k], 'DATE',
                  O.get('%s_date' % band)[k], 'DEPTH',
                  O.get('%s_depth' % band)[k])

        K = np.flatnonzero(
            (O.get('%s_done' % band) == 1) * (odepth > 1) * (odepth < 30))
        print(sum(odepth[K] < fid.single_exposure_depth - 0.25), 'of',
              len(odepth[K]), 'DONE=1 tiles are more than 0.25 mag shallow')

        K = np.flatnonzero((O.get('%s_done' % band) == 1) * (odepth > 1) *
                           (odepth < 30) * goodtiles)
        print(sum(odepth[K] < fid.single_exposure_depth - 0.25), 'of',
              len(odepth[K]),
              'interesting DONE=1 tiles are more than 0.25 mag shallow')

        K = np.flatnonzero((O.get('%s_done' % band) == 1) * (odepth == 1))
        print(len(K), 'DONE=1 tiles have DEPTH=1 (non-photometric)')

        K = np.flatnonzero(
            (O.get('%s_done' % band) == 1) * (odepth == 1) * goodtiles)
        print(len(K),
              'interesting DONE=1 tiles have DEPTH=1 (non-photometric)')

        K = np.flatnonzero((O.get('%s_done' % band) == 1) * (odepth == 30))
        print(len(K), 'DONE=1 tiles have DEPTH=30 (unknown depth)')

        K = np.flatnonzero(
            (O.get('%s_done' % band) == 1) * (odepth == 30) * goodtiles)
        print(len(K), 'interesting DONE=1 tiles have DEPTH=30 (unknown depth)')

        K = np.flatnonzero((O.get('%s_done' % band) == 1) * (odepth == 0))
        print(len(K), 'DONE=1 tiles have DEPTH=0')

        K = np.flatnonzero(
            (O.get('%s_done' % band) == 1) * (odepth == 0) * goodtiles)
        print(len(K), 'interesting DONE=1 tiles have DEPTH=0')

        K = np.flatnonzero((O.get('%s_done' % band) == 0) * (odepth != 0))
        print(len(K), 'tiles have DONE=0 but DEPTH != 0')

    O.writeto(out_fn)
Exemplo n.º 15
0
def main(survey=None, opt=None, args=None):
    '''Driver function for forced photometry of individual Legacy
    Survey images.
    '''
    if args is None:
        args = sys.argv[1:]
    print('forced_photom.py', ' '.join(args))

    if opt is None:
        parser = get_parser()
        opt = parser.parse_args(args)

    import logging
    if opt.verbose == 0:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG
    logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
    # tractor logging is *soooo* chatty
    logging.getLogger('tractor.engine').setLevel(lvl + 10)

    t0 = Time()
    if survey is None:
        survey = LegacySurveyData(survey_dir=opt.survey_dir,
                                  cache_dir=opt.cache_dir,
                                  output_dir=opt.out_dir)
    if opt.skip:
        if opt.out is not None:
            outfn = opt.out
        else:
            outfn = survey.find_file('forced',
                                     output=True,
                                     camera=opt.camera,
                                     expnum=opt.expnum)
        if os.path.exists(outfn):
            print('Ouput file exists:', outfn)
            return 0

    if opt.derivs and opt.agn:
        print('Sorry, can\'t do --derivs AND --agn')
        return -1

    if opt.out is None and opt.out_dir is None:
        print('Must supply either --out or --out-dir')
        return -1

    if opt.expnum is None and opt.out is None:
        print('If no --expnum is given, must supply --out filename')
        return -1

    if not opt.forced:
        opt.apphot = True

    zoomslice = None
    if opt.zoom is not None:
        (x0, x1, y0, y1) = opt.zoom
        zoomslice = (slice(y0, y1), slice(x0, x1))

    ps = None
    if opt.plots is not None:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence(opt.plots)

    # Cache CCDs files before the find_ccds call...
    # Copy required files into the cache?
    if opt.pre_cache:

        def copy_files_to_cache(fns):
            for fn in fns:
                cachefn = fn.replace(survey.survey_dir, survey.cache_dir)
                if not cachefn.startswith(survey.cache_dir):
                    print('Skipping', fn)
                    continue
                outdir = os.path.dirname(cachefn)
                trymakedirs(outdir)
                print('Copy', fn)
                print('  to', cachefn)
                shutil.copyfile(fn, cachefn)

        assert (survey.cache_dir is not None)
        fnset = set()
        fn = survey.find_file('bricks')
        fnset.add(fn)
        fns = survey.find_file('ccd-kds')
        fnset.update(fns)
        copy_files_to_cache(fnset)

    # Read metadata from survey-ccds.fits table
    ccds = survey.find_ccds(camera=opt.camera,
                            expnum=opt.expnum,
                            ccdname=opt.ccdname)
    print(len(ccds), 'with camera', opt.camera, 'and expnum', opt.expnum,
          'and ccdname', opt.ccdname)
    # sort CCDs
    ccds.cut(np.lexsort((ccds.ccdname, ccds.expnum, ccds.camera)))

    # If there is only one catalog survey_dir, we pass it to get_catalog_in_wcs
    # as the northern survey.
    catsurvey_north = survey
    catsurvey_south = None

    if opt.catalog_dir_north is not None:
        assert (opt.catalog_dir_south is not None)
        assert (opt.catalog_resolve_dec_ngc is not None)
        catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
        catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)
    elif opt.catalog_dir is not None:
        catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)

    # Copy required CCD & calib files into the cache?
    if opt.pre_cache:
        assert (survey.cache_dir is not None)
        fnset = set()
        for ccd in ccds:
            im = survey.get_image_object(ccd)
            for key in im.get_cacheable_filename_variables():
                fn = getattr(im, key)
                if fn is None or not (os.path.exists(fn)):
                    continue
                fnset.add(fn)
        copy_files_to_cache(fnset)

    args = []
    for ccd in ccds:
        args.append((survey, catsurvey_north, catsurvey_south,
                     opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))

    if opt.threads:
        from astrometry.util.multiproc import multiproc
        from astrometry.util.timingpool import TimingPool, TimingPoolMeas
        pool = TimingPool(opt.threads)
        poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
        Time.add_measurement(poolmeas)
        mp = multiproc(None, pool=pool)
        tm = Time()
        FF = mp.map(bounce_one_ccd, args)
        print('Multi-processing forced-phot:', Time() - tm)
        del mp
        Time.measurements.remove(poolmeas)
        del poolmeas
        pool.close()
        pool.join()
        del pool
    else:
        FF = map(bounce_one_ccd, args)

    FF = [F for F in FF if F is not None]
    if len(FF) == 0:
        print('No photometry results to write.')
        return 0
    # Keep only the first header
    _, version_hdr, _, _ = FF[0]
    # unpack results
    outlier_masks = [m for _, _, m, _ in FF]
    outlier_hdrs = [h for _, _, _, h in FF]
    FF = [F for F, _, _, _ in FF]
    F = merge_tables(FF)

    if len(ccds):
        version_hdr.delete('CPHDU')
        version_hdr.delete('CCDNAME')

    from legacypipe.utils import add_bits
    from legacypipe.bits import DQ_BITS
    add_bits(version_hdr, DQ_BITS, 'DQMASK', 'DQ', 'D')
    from legacyzpts.psfzpt_cuts import CCD_CUT_BITS
    add_bits(version_hdr, CCD_CUT_BITS, 'CCD_CUTS', 'CC', 'C')
    for i, ap in enumerate(apertures_arcsec):
        version_hdr.add_record(
            dict(name='APRAD%i' % i,
                 value=ap,
                 comment='(optical) Aperture radius, in arcsec'))

    unitmap = {
        'exptime': 'sec',
        'flux': 'nanomaggy',
        'flux_ivar': '1/nanomaggy^2',
        'apflux': 'nanomaggy',
        'apflux_ivar': '1/nanomaggy^2',
        'psfdepth': '1/nanomaggy^2',
        'galdepth': '1/nanomaggy^2',
        'sky': 'nanomaggy/arcsec^2',
        'psfsize': 'arcsec',
        'fwhm': 'pixels',
        'ccdrarms': 'arcsec',
        'ccddecrms': 'arcsec',
        'ra': 'deg',
        'dec': 'deg',
        'skyrms': 'counts/sec',
        'dra': 'arcsec',
        'ddec': 'arcsec',
        'dra_ivar': '1/arcsec^2',
        'ddec_ivar': '1/arcsec^2'
    }

    columns = F.get_columns()
    order = [
        'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
        'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'fwhm', 'ccd_cuts',
        'airmass', 'sky', 'skyrms', 'psfdepth', 'galdepth', 'ccdzpt',
        'ccdrarms', 'ccddecrms', 'ccdphrms', 'ra', 'dec', 'flux', 'flux_ivar',
        'fracflux', 'rchisq', 'fracmasked', 'fracin', 'apflux', 'apflux_ivar',
        'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar', 'ddec_ivar'
    ]
    columns = [c for c in order if c in columns]
    units = [unitmap.get(c, '') for c in columns]

    if opt.out is not None:
        outdir = os.path.dirname(opt.out)
        if len(outdir):
            trymakedirs(outdir)
        tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.out))
        fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
        F.writeto(tmpfn, units=units, append=True, columns=columns)
        os.rename(tmpfn, opt.out)
        print('Wrote', opt.out)
    else:
        with survey.write_output('forced',
                                 camera=opt.camera,
                                 expnum=opt.expnum) as out:
            F.writeto(None,
                      fits_object=out.fits,
                      primheader=version_hdr,
                      units=units,
                      columns=columns)
            print('Wrote', out.real_fn)

    if opt.outlier_mask is not None:
        # Add outlier bit meanings to the primary header
        version_hdr.add_record(
            dict(name='COMMENT', value='Outlier mask bit meanings'))
        version_hdr.add_record(
            dict(name='OUTL_POS',
                 value=1,
                 comment='Outlier mask bit for Positive outlier'))
        version_hdr.add_record(
            dict(name='OUTL_NEG',
                 value=2,
                 comment='Outlier mask bit for Negative outlier'))

    if opt.outlier_mask == 'default':
        outdir = os.path.join(opt.out_dir, 'outlier-masks')
        camexp = set(zip(ccds.camera, ccds.expnum))
        for c, e in camexp:
            I = np.flatnonzero((ccds.camera == c) * (ccds.expnum == e))
            ccd = ccds[I[0]]
            imfn = ccd.image_filename.strip()
            outfn = os.path.join(outdir, imfn.replace('.fits',
                                                      '-outlier.fits'))
            trymakedirs(outfn, dir=True)
            tempfn = outfn.replace('.fits', '-tmp.fits')
            with fitsio.FITS(tempfn, 'rw', clobber=True) as fits:
                fits.write(None, header=version_hdr)
                for i in I:
                    mask = outlier_masks[i]
                    _, _, _, meth, tile = survey.get_compression_args(
                        'outliers_mask', shape=mask.shape)
                    fits.write(mask,
                               header=outlier_hdrs[i],
                               extname=ccds.ccdname[i],
                               compress=meth,
                               tile_dims=tile)
            os.rename(tempfn, outfn)
            print('Wrote', outfn)
    elif opt.outlier_mask is not None:
        with fitsio.FITS(opt.outlier_mask, 'rw', clobber=True) as F:
            F.write(None, header=version_hdr)
            for i, (hdr, mask) in enumerate(zip(outlier_hdrs, outlier_masks)):
                _, _, _, meth, tile = survey.get_compression_args(
                    'outliers_mask', shape=mask.shape)
                F.write(mask,
                        header=hdr,
                        extname=ccds.ccdname[i],
                        compress=meth,
                        tile_dims=tile)
        print('Wrote', opt.outlier_mask)

    tnow = Time()
    print('Total:', tnow - t0)
    return 0
Exemplo n.º 16
0
def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--plots', action='store_true')
    parser.add_argument('--brick', help='Brick name to run')
    parser.add_argument(
        '--input-dir',
        default='/global/projecta/projectdirs/cosmo/work/legacysurvey/dr7')
    #/global/cscratch1/sd/desiproc/dr7out')
    parser.add_argument('--survey-dir',
                        default='/global/cscratch1/sd/dstn/dr7-depthcut')
    parser.add_argument('--output-dir',
                        default='/global/cscratch1/sd/dstn/bright')
    opt = parser.parse_args()

    plots = opt.plots
    ps = PlotSequence('bright')
    brickname = opt.brick

    insurvey = LegacySurveyData(opt.input_dir, cache_dir=opt.survey_dir)
    outsurvey = LegacySurveyData(opt.output_dir, output_dir=opt.output_dir)

    bfn = insurvey.find_file('blobmap', brick=brickname)
    print('Found blob map', bfn)
    blobs = fitsio.read(bfn)
    h, w = blobs.shape

    brick = insurvey.get_brick_by_name(brickname)
    brickwcs = wcs_for_brick(brick)
    radius = np.sqrt(2.) * 0.25 * 1.01
    neighbors = insurvey.get_bricks_near(brick.ra, brick.dec, radius)
    print(len(neighbors), 'bricks nearby')

    def showbool(X):
        d = downsample_max(X, 8)
        h, w = X.shape
        plt.imshow(d,
                   interpolation='nearest',
                   origin='lower',
                   vmin=0,
                   vmax=1,
                   extent=[0, w, 0, h],
                   cmap='gray')

    brightblobs = set()

    for nb in neighbors:
        if nb.brickname == brickname:
            # ignore myself!
            continue
        print('Neighbor:', nb.brickname)

        mfn = insurvey.find_file('maskbits', brick=nb.brickname)
        if not os.path.exists(mfn):
            print('No maskbits file:', mfn)
            continue
        maskbits = fitsio.read(mfn)
        bright = ((maskbits & MASKBITS['BRIGHT']) > 0)
        print(np.sum(bright > 0), 'BRIGHT pixels set')
        primary = (maskbits & MASKBITS['NPRIMARY'] == 0)
        print(np.sum(primary), 'PRIMARY pixels set')
        edge = binary_dilation(primary, structure=np.ones((3, 3), bool))
        edge = edge * np.logical_not(primary)
        brightedge = edge & bright

        if plots:
            plt.clf()
            showbool(bright)
            plt.title('bright: brick %s' % nb.brickname)
            ps.savefig()

            # plt.clf()
            # showbool(primary)
            # plt.title('PRIMARY, brick %s' % nb.brickname)
            # ps.savefig()
            #
            # plt.clf()
            # showbool(edge)
            # plt.title('boundary, brick %s' % nb.brickname)
            # ps.savefig()

            plt.clf()
            showbool(brightedge)
            plt.title('bright at edge, brick %s' % nb.brickname)
            ps.savefig()

        nwcs = wcs_for_brick(nb)

        yy, xx = np.nonzero(brightedge)
        print(len(yy), 'bright edge pixels')
        if len(yy) == 0:
            continue
        rr, dd = nwcs.pixelxy2radec(xx + 1, yy + 1)
        print('RA range', rr.min(), rr.max(), 'vs brick', brick.ra1, brick.ra2)
        print('Dec range', dd.min(), dd.max(), 'vs brick', brick.dec1,
              brick.dec2)
        # Find pixels that are within this brick's unique area
        I, = np.nonzero((rr >= brick.ra1) * (rr <= brick.ra2) *
                        (dd >= brick.dec1) * (dd <= brick.dec2))

        if plots:
            plt.clf()
            plt.plot(
                [brick.ra1, brick.ra1, brick.ra2, brick.ra2, brick.ra1],
                [brick.dec1, brick.dec2, brick.dec2, brick.dec1, brick.dec1],
                'b-')
            plt.plot(rr, dd, 'k.')
            plt.plot(rr[I], dd[I], 'r.')
            plt.title('Bright pixels from %s' % nb.brickname)
            ps.savefig()

        if len(I) == 0:
            print('No edge pixels touch')
            #plt.plot(br,bd, 'b-')
            continue
        #print('Edge pixels touch!')
        #plt.plot(br,bd, 'r-', zorder=20)

        ok, x, y = brickwcs.radec2pixelxy(rr[I], dd[I])
        x = np.round(x).astype(int) - 1
        y = np.round(y).astype(int) - 1
        print('Pixel ranges X', x.min(), x.max(), 'Y', y.min(), y.max())
        assert (np.all((x >= 0) * (x < w) * (y >= 0) * (y < h)))
        print('Adding blobs:', np.unique(blobs[y, x]))
        brightblobs.update(blobs[y, x])
        print('Blobs touching bright pixels:', brightblobs)

    print()
    brightblobs.discard(-1)
    if len(brightblobs) == 0:
        print('No neighboring bright blobs to update!')
        return
    print('Updating', len(brightblobs), 'blobs:', brightblobs)

    tmap = np.zeros(blobs.max() + 2, bool)
    for b in brightblobs:
        tmap[b + 1] = True
    touching = tmap[blobs + 1]

    if plots:
        plt.clf()
        showbool(touching)
        plt.title('Blobs touching bright, brick %s' % brickname)
        ps.savefig()

    mfn = insurvey.find_file('maskbits', brick=brickname)
    maskbits, hdr = fitsio.read(mfn, header=True)
    updated = maskbits | (MASKBITS['BRIGHT'] * touching)
    if np.all(maskbits == updated):
        print('No bits updated!  (Bright stars were already masked)')
        return
    maskbits = updated

    if plots:
        plt.clf()
        showbool((maskbits & MASKBITS['BRIGHT']) > 0)
        plt.title('New maskbits map for BRIGHT, brick %s' % brickname)
        ps.savefig()

    with outsurvey.write_output('maskbits', brick=brickname) as out:
        out.fits.write(maskbits, hdr=hdr)

    tfn = insurvey.find_file('tractor', brick=brickname)
    phdr = fitsio.read_header(tfn, ext=0)
    hdr = fitsio.read_header(tfn, ext=1)
    T = fits_table(tfn)
    print('Read', len(T), 'sources')
    print('Bright:', Counter(T.brightstarinblob))
    iby = np.clip(np.round(T.by), 0, h - 1).astype(int)
    ibx = np.clip(np.round(T.bx), 0, w - 1).astype(int)
    if plots:
        before = np.flatnonzero(T.brightstarinblob)
    T.brightstarinblob |= touching[iby, ibx]
    print('Bright:', Counter(T.brightstarinblob))

    # yuck -- copy the TUNIT headers from input to output.
    units = [
        hdr.get('TUNIT%i' % (i + 1), '') for i in range(len(T.get_columns()))
    ]

    if plots:
        plt.clf()
        showbool((maskbits & MASKBITS['BRIGHT']) > 0)
        ax = plt.axis()
        after = np.flatnonzero(T.brightstarinblob)
        plt.plot(T.bx[before], T.by[before], 'gx')
        plt.plot(T.bx[after], T.by[after], 'r.')
        plt.axis(ax)
        plt.title('sources with brightstarinblob, brick %s' % brickname)
        ps.savefig()

    with outsurvey.write_output('tractor', brick=brickname) as out:
        T.writeto(None,
                  fits_object=out.fits,
                  primheader=phdr,
                  header=hdr,
                  units=units)
Exemplo n.º 17
0
def main():

    # indir = '/global/cscratch1/sd/dstn/dr8test-1'
    # name = 'dr8-test1'
    # pretty = 'DR8 test1'

    # indir = '/scratch1/scratchdirs/desiproc/dr8test002/'
    # name = 'dr8-test2'
    # pretty = 'DR8 test2 (outliers)'

    # indir = '/scratch1/scratchdirs/desiproc/dr8test003/'
    # name = 'dr8-test3'
    # pretty = 'DR8 test3 (outliers)'
    #
    # indir = '/scratch1/scratchdirs/desiproc/dr8test004/'
    # name = 'dr8-test4'
    # pretty = 'DR8 test4 (large-galaxies)'

    # indir = '/global/cscratch1/sd/dstn/dr8test005/'
    # name = 'dr8-test5'
    # pretty = 'DR8 test5 (trident)'

    # indir = '/global/cscratch1/sd/dstn/dr8test006/'
    # name = 'dr8-test6'
    # pretty = 'DR8 test6 (sky)'

    # indir = '/global/cscratch1/sd/dstn/dr8test007/'
    # name = 'dr8-test7'
    # pretty = 'DR8 test7 (outliers)'

    #indir = '/global/cscratch1/sd/dstn/dr8test14/'
    #name = 'dr8-test14'
    #pretty = 'DR8 test14 (rc)'

    #indir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8a/'
    #name = 'dr8a'
    #pretty = 'DR8a (rc)'

    if False:
        indir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-decam/'
        name = 'dr8b-decam'
        pretty = 'DR8b DECam'
        survey_dir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-decam'

    if True:
        indir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-90prime-mosaic/'
        name = 'dr8b-90p-mos'
        pretty = 'DR8b BASS+MzLS'
        survey_dir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-90prime-mosaic'

    # ln -s /global/project/projectdirs/cosmo/work/legacysurvey/dr8b/runbrick-decam/coadds-only/coadd/ .

    sublayers = ['', '-model', '-resid']
    subpretty = {'': ' images', '-model': ' models', '-resid': ' residuals'}
    # survey_dir = '/global/cscratch1/sd/desiproc/dr7'

    # sublayers = ['']
    # subpretty = {'':' images'}

    #survey_dir = '/global/cscratch1/sd/dstn/dr8-depthcut'
    #survey_dir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8a/'

    rsync = False

    datadir = 'data'

    survey = LegacySurveyData(survey_dir=survey_dir)

    fn = 'map/test_layers.py'
    txt = open(fn).read()
    for x in sublayers:
        txt = txt + '\n' + 'test_layers.append(("%s%s", "%s%s"))\n' % (
            name, x, pretty, subpretty[x])
    open(fn, 'wb').write(txt.encode())
    print('Wrote', fn)

    basedir = os.path.join(datadir, name)

    if rsync:
        cmd = 'rsync -LRarv %s/./{coadd/*/*/*-{image-,model-,ccds}*.fits*,tractor} %s/%s' % (
            indir, datadir, name)
        print(cmd)
        os.system(cmd)

        # ...?
        cmd = 'rsync -Rarv %s/./{images,survey-ccds*.fits} %s/%s' % (
            survey_dir, datadir, name)
        print(cmd)
        os.system(cmd)
    else:
        # symlink
        if os.path.exists(basedir):
            print('Not symlinking', indir, 'to', basedir, ': already exists!')
        else:
            os.makedirs(basedir)
            for subdir in ['coadd', 'tractor']:
                os.symlink(os.path.join(indir, subdir),
                           os.path.join(basedir, subdir),
                           target_is_directory=True)
            for fn in ['images', 'calib']:
                os.symlink(os.path.join(indir, subdir),
                           os.path.join(basedir, subdir),
                           target_is_directory=False)
            for pat in ['survey-ccds-*']:
                for fn in [
                        os.path.basename(f)
                        for f in glob(os.path.join(indir, pat))
                ]:
                    os.symlink(os.path.join(indir, subdir),
                               os.path.join(basedir, subdir),
                               target_is_directory=False)

    allbricks = survey.get_bricks_readonly()

    imagefns = glob(os.path.join(basedir, 'coadd', '*', '*',
                                 '*-image-*.fits*'))

    extraimagefns = glob(
        os.path.join(basedir, 'extra-images', 'coadd', '*', '*',
                     '*-image-*.fits*'))

    print('Image filenames:', len(imagefns), 'plus', len(extraimagefns),
          'extras')
    imagefns += extraimagefns

    brickset = set()
    for fn in imagefns:
        dirs = fn.split('/')
        brickname = dirs[-2]
        brickset.add(brickname)
    print(len(brickset), 'bricks found')

    I, = np.nonzero([b in brickset for b in allbricks.brickname])
    bricks = allbricks[I]

    brickfn = os.path.join(basedir, 'survey-bricks.fits.gz')
    bricks.writeto(brickfn)
    print('Wrote', brickfn)

    threads = 8
    tharg = '--threads %i ' % threads
    #tharg = ''

    for x in sublayers:
        cmd = 'python -u render-tiles.py --kind %s%s --bricks' % (name, x)
        print(cmd)
        os.system(cmd)

    # images
    for scale in range(1, 8):
        cmd = 'python -u render-tiles.py --kind %s --scale --zoom %i %s' % (
            name, scale, tharg)
        print(cmd)
        os.system(cmd)

    # models
    for scale in range(1, 8):
        cmd = 'python -u render-tiles.py --kind %s-model --scale --zoom %i %s' % (
            name, scale, tharg)
        print(cmd)
        os.system(cmd)

    # resids
    for scale in range(1, 8):
        cmd = 'python -u render-tiles.py --kind %s-resid --scale --zoom %i %s' % (
            name, scale, tharg)
        print(cmd)
        os.system(cmd)

    for x in sublayers:
        cmd = 'python -u render-tiles.py --kind %s%s --top' % (name, x)
        print(cmd)
        os.system(cmd)
Exemplo n.º 18
0
def main():

    parser = argparse.ArgumentParser()
    parser.add_argument('--build-sample',
                        action='store_true',
                        help='Build the sample.')
    parser.add_argument('--jpg-cutouts',
                        action='store_true',
                        help='Get jpg cutouts from the viewer.')
    parser.add_argument('--ccd-cutouts',
                        action='store_true',
                        help='Get CCD cutouts of each galaxy.')
    parser.add_argument('--runbrick',
                        action='store_true',
                        help='Run the pipeline.')
    parser.add_argument('--build-webpage',
                        action='store_true',
                        help='(Re)build the web content.')
    args = parser.parse_args()

    # Top-level directory
    key = 'LEGACY_SURVEY_LARGE_GALAXIES'
    if key not in os.environ:
        print('Required ${} environment variable not set'.format(key))
        return 0
    largedir = os.getenv(key)
    samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')

    # --------------------------------------------------
    # Build the sample of large galaxies based on the available imaging.
    if args.build_sample:

        # Read the parent catalog.
        cat = read_rc3()

        # Create a simple WCS object for each object and find all the CCDs
        # touching that WCS footprint.
        survey = LegacySurveyData(version='dr2')  # hack!
        allccds = survey.get_ccds()
        keep = np.concatenate((survey.apply_blacklist(allccds),
                               survey.photometric_ccds(allccds)))
        allccds.cut(keep)

        ccdlist = []
        outcat = []
        for gal in cat:
            galwcs = _simplewcs(gal)

            ccds1 = allccds[ccds_touching_wcs(galwcs, allccds)]
            ccds1 = ccds1[_uniqccds(ccds1)]

            if len(
                    ccds1
            ) > 0 and 'g' in ccds1.filter and 'r' in ccds1.filter and 'z' in ccds1.filter:
                print('Found {} CCDs for {}, D(25)={:.4f}'.format(
                    len(ccds1), gal['GALAXY'], gal['RADIUS']))

                ccdsfile = os.path.join(
                    largedir, 'ccds',
                    '{}-ccds.fits'.format(gal['GALAXY'].strip().lower()))
                print('  Writing {}'.format(ccdsfile))
                if os.path.isfile(ccdsfile):
                    os.remove(ccdsfile)
                ccds1.writeto(ccdsfile)

                ccdlist.append(ccds1)
                if len(outcat) == 0:
                    outcat = gal
                else:
                    outcat = vstack((outcat, gal))
                #if gal['GALAXY'] == 'MCG5-19-36':
                #    pdb.set_trace()

        # Write out the final catalog.
        samplefile = os.path.join(largedir, 'large-galaxies-sample.fits')
        if os.path.isfile(samplefile):
            os.remove(samplefile)
        print('Writing {}'.format(samplefile))
        outcat.write(samplefile)
        print(outcat)

        # Do we need to transfer any of the data to nyx?
        _getfiles(merge_tables(ccdlist))

    # --------------------------------------------------
    # Get data, model, and residual cutouts from the legacysurvey viewer.  Also
    # get thumbnails that are lower resolution.
    if args.jpg_cutouts:
        thumbsize = 100
        sample = fits.getdata(samplefile, 1)
        for gal in sample:
            size = np.ceil(10 * gal['RADIUS'] / PIXSCALE)
            thumbpixscale = PIXSCALE * size / thumbsize

            #imageurl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
            #  '&pixscale={:.3f}&size={:g}'.format(PIXSCALE, size)
            #imagejpg = os.path.join(largedir, 'cutouts', gal['GALAXY'].strip().lower()+'-image.jpg')
            #if os.path.isfile(imagejpg):
            #    os.remove(imagejpg)
            #os.system('wget --continue -O {:s} "{:s}"' .format(imagejpg, imageurl))

            thumburl = 'http://legacysurvey.org/viewer/jpeg-cutout-decals-dr2?ra={:.6f}&dec={:.6f}'.format(gal['RA'], gal['DEC'])+\
              '&pixscale={:.3f}&size={:g}'.format(thumbpixscale, thumbsize)
            thumbjpg = os.path.join(
                largedir, 'cutouts',
                gal['GALAXY'].strip().lower() + '-image-thumb.jpg')
            if os.path.isfile(thumbjpg):
                os.remove(thumbjpg)
            os.system('wget --continue -O {:s} "{:s}"'.format(
                thumbjpg, thumburl))

    # --------------------------------------------------
    # (Re)build the webpage.
    if args.build_webpage:

        # index.html
        html = open(os.path.join(largedir, 'index.html'), 'w')
        html.write('<html><body>\n')
        html.write('<h1>Sample of Large Galaxies</h1>\n')
        html.write('<table border="2" width="30%">\n')
        html.write('<tbody>\n')
        sample = fits.getdata(samplefile, 1)
        for gal in sample:
            # Add coordinates and sizes here.
            galaxy = gal['GALAXY'].strip().lower()
            html.write('<tr>\n')
            html.write('<td><a href="html/{}.html">{}</a></td>\n'.format(
                galaxy, galaxy.upper()))
            html.write(
                '<td><a href="http://legacysurvey.org/viewer/?ra={:.6f}&dec={:.6f}" target="_blank"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'
                .format(gal['RA'], gal['DEC'], galaxy, galaxy.upper()))
            #           html.write('<td><a href="html/{}.html"><img src=cutouts/{}-image-thumb.jpg alt={} /></a></td>\n'.format(galaxy, galaxy, galaxy.upper()))
            html.write('</tr>\n')
        html.write('</tbody>\n')
        html.write('</table>\n')
        html.write('</body></html>\n')
        html.close()

        sys.exit(1)

        # individual galaxy pages
        for gal in sample[:3]:
            galaxy = gal['GALAXY'].strip().lower()
            html = open(os.path.join(largedir, 'html/{}.html'.format(galaxy)),
                        'w')
            html.write('<html><body>\n')
            html.write(
                '<a href=../cutouts/{}.jpg><img src=../cutouts/{}-image.jpg alt={} /></a>\n'
                .format(galaxy, galaxy, galaxy, galaxy.upper()))
            html.write('</body></html>\n')
            html.close()

    # --------------------------------------------------
    # Get cutouts of all the CCDs for each galaxy.
    if args.ccd_cutouts:
        sample = fits.getdata(samplefile, 1)

        for gal in sample[1:2]:
            galaxy = gal['GALAXY'].strip().lower()
            ccdsfile = os.path.join(largedir, 'ccds',
                                    '{}-ccds.fits'.format(galaxy))
            ccds = fits.getdata(ccdsfile)

            pdb.set_trace()

    # --------------------------------------------------
    # Run the pipeline.
    if args.runbrick:
        sample = fits.getdata(samplefile, 1)

        for gal in sample[1:2]:
            galaxy = gal['GALAXY'].strip().lower()
            diam = 10 * np.ceil(gal['RADIUS'] / PIXSCALE).astype(
                'int16')  # [pixels]

            # Note: zoom is relative to the center of an imaginary brick with
            # dimensions (0, 3600, 0, 3600).
            survey = LegacySurveyData(version='dr2', output_dir=largedir)
            run_brick(None,
                      survey,
                      radec=(gal['RA'], gal['DEC']),
                      blobxy=zip([diam / 2], [diam / 2]),
                      threads=1,
                      zoom=(1800 - diam / 2, 1800 + diam / 2, 1800 - diam / 2,
                            1800 + diam / 2),
                      wise=False,
                      forceAll=True,
                      writePickles=False,
                      do_calibs=False,
                      write_metrics=False,
                      pixPsf=True,
                      splinesky=True,
                      early_coadds=True,
                      stages=['writecat'],
                      ceres=False)

            pdb.set_trace()
Exemplo n.º 19
0
def main(survey=None, opt=None):

    print(' '.join(sys.argv))
    '''Driver function for forced photometry of individual Legacy
    Survey images.
    '''
    if opt is None:
        parser = get_parser()
        opt = parser.parse_args()

    Time.add_measurement(MemMeas)
    t0 = tlast = Time()

    if opt.skip and os.path.exists(opt.outfn):
        print('Ouput file exists:', opt.outfn)
        sys.exit(0)

    if opt.derivs and opt.agn:
        print('Sorry, can\'t do --derivs AND --agn')
        sys.exit(0)

    if not opt.forced:
        opt.apphot = True

    zoomslice = None
    if opt.zoom is not None:
        (x0, x1, y0, y1) = opt.zoom
        zoomslice = (slice(y0, y1), slice(x0, x1))

    ps = None
    if opt.plots is not None:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence(opt.plots)

    # Try parsing first arg as exposure number (otherwise, it's a filename)
    try:
        expnum = int(opt.expnum)
        filename = None
    except:
        # make this 'None' for survey.find_ccds()
        expnum = None
        filename = opt.expnum

    # Try parsing HDU: "all" or HDU name or HDU number.
    all_hdus = (opt.ccdname == 'all')
    hdu = -1
    ccdname = None
    if not all_hdus:
        try:
            hdu = int(opt.ccdname)
        except:
            ccdname = opt.ccdname

    if survey is None:
        survey = LegacySurveyData(survey_dir=opt.survey_dir)

    catsurvey_north = survey
    catsurvey_south = None

    if opt.catalog_dir_north is not None:
        assert (opt.catalog_dir_south is not None)
        assert (opt.catalog_resolve_dec_ngc is not None)
        catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
        catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)

    if opt.catalog_dir is not None:
        catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)

    if filename is not None and hdu >= 0:
        # FIXME -- try looking up in CCDs file?
        # Read metadata from file
        print('Warning: faking metadata from file contents')
        T = exposure_metadata([filename], hdus=[hdu])
        print('Metadata:')
        T.about()

        if not 'ccdzpt' in T.columns():
            phdr = fitsio.read_header(filename)
            T.ccdzpt = np.array([phdr['MAGZERO']])
            print('WARNING: using header MAGZERO')
            T.ccdraoff = np.array([0.])
            T.ccddecoff = np.array([0.])
            print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.')

    else:
        # Read metadata from survey-ccds.fits table
        T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
        print(len(T), 'with expnum', expnum, 'and ccdname', ccdname)
        if hdu >= 0:
            T.cut(T.image_hdu == hdu)
            print(len(T), 'with HDU', hdu)
        if filename is not None:
            T.cut(np.array([f.strip() == filename for f in T.image_filename]))
            print(len(T), 'with filename', filename)
        if opt.camera is not None:
            T.cut(T.camera == opt.camera)
            print(len(T), 'with camera', opt.camera)
        if not all_hdus:
            assert (len(T) == 1)

    args = []
    for ccd in T:
        args.append((survey, catsurvey_north, catsurvey_south,
                     opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))

    if opt.threads:
        from astrometry.util.multiproc import multiproc
        from astrometry.util.timingpool import TimingPool, TimingPoolMeas
        pool = TimingPool(opt.threads)
        poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
        Time.add_measurement(poolmeas)
        mp = multiproc(None, pool=pool)
        tm = Time()
        FF = mp.map(bounce_one_ccd, args)
        print('Multi-processing forced-phot:', Time() - tm)
    else:
        FF = map(bounce_one_ccd, args)

    FF = [F for F in FF if F is not None]
    if len(FF) == 0:
        print('No photometry results to write.')
        return 0
    # Keep only the first header
    _, version_hdr = FF[0]
    FF = [F for F, hdr in FF]
    F = merge_tables(FF)

    if all_hdus:
        version_hdr.delete('CPHDU')
        version_hdr.delete('CCDNAME')

    units = {
        'exptime': 'sec',
        'flux': 'nanomaggy',
        'flux_ivar': '1/nanomaggy^2',
        'apflux': 'nanomaggy',
        'apflux_ivar': '1/nanomaggy^2',
        'psfdepth': '1/nanomaggy^2',
        'galdepth': '1/nanomaggy^2',
        'sky': 'nanomaggy/arcsec^2',
        'psfsize': 'arcsec'
    }
    if opt.derivs:
        units.update({
            'dra': 'arcsec',
            'ddec': 'arcsec',
            'dra_ivar': '1/arcsec^2',
            'ddec_ivar': '1/arcsec^2'
        })

    columns = F.get_columns()
    order = [
        'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
        'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'ccd_cuts',
        'airmass', 'sky', 'psfdepth', 'galdepth', 'ra', 'dec', 'flux',
        'flux_ivar', 'fracflux', 'rchisq', 'fracmasked', 'apflux',
        'apflux_ivar', 'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar',
        'ddec_ivar'
    ]
    columns = [c for c in order if c in columns]

    # Set units headers (must happen after column ordering is set!)
    hdr = fitsio.FITSHDR()
    for i, col in enumerate(columns):
        if col in units:
            hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))

    outdir = os.path.dirname(opt.outfn)
    if len(outdir):
        trymakedirs(outdir)
    tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.outfn))
    fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
    F.writeto(tmpfn, header=hdr, append=True, columns=columns)
    os.rename(tmpfn, opt.outfn)
    print('Wrote', opt.outfn)

    tnow = Time()
    print('Total:', tnow - t0)
    return 0
Exemplo n.º 20
0
def main():
    """Main program.
    """
    import argparse

    parser = argparse.ArgumentParser(
        description=
        "This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)."
    )
    parser.add_argument('--calibs',
                        action='store_true',
                        help='Output CCDs that need to be calibrated.')

    parser.add_argument('--nper',
                        type=int,
                        default=None,
                        help='Batch N calibs per line')
    parser.add_argument(
        '--byexp',
        action='store_true',
        default=False,
        help='Run one whole exposure per job (not one CCD per job)')

    parser.add_argument('--forced',
                        action='store_true',
                        help='Output forced-photometry commands')

    parser.add_argument('--lsb',
                        action='store_true',
                        help='Output Low-Surface-Brightness commands')

    parser.add_argument('--stage', help='Stage image files to given directory')

    parser.add_argument('--touching',
                        action='store_true',
                        help='Cut to only CCDs touching selected bricks')
    parser.add_argument('--near',
                        action='store_true',
                        help='Quick cut to only CCDs near selected bricks')

    parser.add_argument('--check-coadd',
                        action='store_true',
                        help='Check which coadds actually need to run.')
    parser.add_argument('--out',
                        help='Output filename for calibs, default %(default)s',
                        default='jobs')
    parser.add_argument('--command',
                        action='store_true',
                        help='Write out full command-line to run calib')
    parser.add_argument('--opt', help='With --command, extra options to add')

    parser.add_argument('--maxra', type=float, help='Maximum RA to run')
    parser.add_argument('--minra', type=float, help='Minimum RA to run')
    parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
    parser.add_argument('--mindec', type=float, help='Minimum Dec to run')

    parser.add_argument('--region', help='Region to select')

    parser.add_argument('--bricks', help='Set bricks.fits file to load')
    parser.add_argument('--ccds', help='Set ccds.fits file to load')
    parser.add_argument('--ignore_cuts',
                        action='store_true',
                        default=False,
                        help='no photometric cuts')
    parser.add_argument('--save_to_fits',
                        action='store_true',
                        default=False,
                        help='save cut brick,ccd to fits table')
    parser.add_argument(
        '--name',
        action='store',
        default='dr3',
        help='save with this suffix, e.g. refers to ccds table')

    parser.add_argument('--delete-sky',
                        action='store_true',
                        help='Delete any existing sky calibration files')

    parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')

    parser.add_argument('--nccds',
                        action='store_true',
                        default=False,
                        help='Prints number of CCDs per brick')

    parser.add_argument('--bands',
                        default='g,r,z',
                        help='Set bands to keep: comma-separated list.')

    opt = parser.parse_args()

    want_ccds = (opt.calibs or opt.forced or opt.lsb)
    want_bricks = not want_ccds

    survey = LegacySurveyData()
    if opt.bricks is not None:
        B = fits_table(opt.bricks)
        log('Read', len(B), 'from', opt.bricks)
    else:
        B = survey.get_bricks()

    log('Bricks Dec range:', B.dec.min(), B.dec.max())

    if opt.ccds is not None:
        T = fits_table(opt.ccds)
        log('Read', len(T), 'from', opt.ccds)
    else:
        T = survey.get_ccds()
        log(len(T), 'CCDs')
    T.index = np.arange(len(T))

    if opt.ignore_cuts == False:
        log('Applying CCD cuts...')
        if 'ccd_cuts' in T.columns():
            T.cut(T.ccd_cuts == 0)
            log(len(T), 'CCDs survive cuts')

    bands = opt.bands.split(',')
    log('Filters:', np.unique(T.filter))
    T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
    log('Cut to', len(T), 'CCDs in filters', bands)

    log('CCDs Dec range:', T.dec.min(), T.dec.max())

    # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
    # plt.clf()
    # plt.hist(counts, counts.max()+1)
    # plt.savefig('bricks.png')
    # B.cut(I[counts >= 9])
    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # #plt.scatter(B.ra[I], B.dec[I], c=counts)
    # plt.savefig('bricks2.png')

    # DES Stripe82
    #rlo,rhi = 350.,360.
    # rlo,rhi = 300., 10.
    # dlo,dhi = -6., 4.
    # TINY bit
    #rlo,rhi = 350.,351.1
    #dlo,dhi = 0., 1.1

    # EDR+
    # 860 bricks
    # ~10,000 CCDs
    #rlo,rhi = 239,246
    #dlo,dhi =   5, 13

    # DR1
    #rlo,rhi = 0, 360
    # part 1
    #dlo,dhi = 25, 40
    # part 2
    #dlo,dhi = 20,25
    # part 3
    #dlo,dhi = 15,20
    # part 4
    #dlo,dhi = 10,15
    # part 5
    #dlo,dhi = 5,10
    # the rest
    #dlo,dhi = -11, 5
    #dlo,dhi = 15,25.5

    dlo, dhi = -25, 40
    rlo, rhi = 0, 360

    # Arjun says 3x3 coverage area is roughly
    # RA=240-252 DEC=6-12 (but not completely rectangular)

    # COSMOS
    #rlo,rhi = 148.9, 151.2
    #dlo,dhi = 0.9, 3.5

    # A nice well-behaved region (EDR2/3)
    # rlo,rhi = 243.6, 244.6
    # dlo,dhi = 8.1, 8.6

    # 56 bricks, ~725 CCDs
    #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
    # 240 bricks, ~3000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
    # 535 bricks, ~7000 CCDs
    #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))

    if opt.region in ['test1', 'test2', 'test3', 'test4']:
        nm = dict(
            test1='2446p115',  # weird stuff around bright star
            test2='1183p292',  # faint sources around bright galaxy
            test3='3503p005',  # DES
            test4='1163p277',  # Pollux
        )[opt.region]

        B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
        log('Cut to', len(B), 'bricks')
        log(B.ra, B.dec)
        dlo, dhi = -90, 90
        rlo, rhi = 0, 360

    elif opt.region == 'edr':
        # EDR:
        # 535 bricks, ~7000 CCDs
        rlo, rhi = 240, 245
        dlo, dhi = 5, 12

    elif opt.region == 'dr8-decam':
        rlo, rhi = 0, 360
        dlo, dhi = -70, 40
        log('DR8-DECam region')

    elif opt.region == 'edrplus':
        rlo, rhi = 235, 248
        dlo, dhi = 5, 15

    elif opt.region == 'edr-south':
        rlo, rhi = 240, 245
        dlo, dhi = 5, 10

    elif opt.region == 'cosmos1':
        # 16 bricks in the core of the COSMOS field.
        rlo, rhi = 149.75, 150.75
        dlo, dhi = 1.6, 2.6

    elif opt.region == 'pristine':
        # Stream?
        rlo, rhi = 240, 250
        dlo, dhi = 10, 15

    elif opt.region == 'des':
        dlo, dhi = -6., 4.
        rlo, rhi = 317., 7.

        T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
        log('Cut to', len(T), 'CCDs with "CPDES82" in filename')

    elif opt.region == 'subdes':
        rlo, rhi = 320., 360.
        dlo, dhi = -1.25, 1.25

    elif opt.region == 'northwest':
        rlo, rhi = 240, 360
        dlo, dhi = 20, 40
    elif opt.region == 'north':
        rlo, rhi = 120, 240
        dlo, dhi = 20, 40
    elif opt.region == 'northeast':
        rlo, rhi = 0, 120
        dlo, dhi = 20, 40
    elif opt.region == 'southwest':
        rlo, rhi = 240, 360
        dlo, dhi = -20, 0
    elif opt.region == 'south':
        rlo, rhi = 120, 240
        dlo, dhi = -20, 0
    elif opt.region == 'southeast':
        rlo, rhi = 0, 120
        dlo, dhi = -20, 0
    elif opt.region == 'southsoutheast':
        rlo, rhi = 0, 120
        dlo, dhi = -20, -10
    elif opt.region == 'midwest':
        rlo, rhi = 240, 360
        dlo, dhi = 0, 20
    elif opt.region == 'middle':
        rlo, rhi = 120, 240
        dlo, dhi = 0, 20
    elif opt.region == 'mideast':
        rlo, rhi = 0, 120
        dlo, dhi = 0, 20

    elif opt.region == 'grz':
        # Bricks with grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
        log('Cut to', len(B), 'bricks with grz coverage')

    elif opt.region == 'nogrz':
        # Bricks without grz coverage.
        # Be sure to use  --bricks survey-bricks-in-dr1.fits
        # which has_[grz] columns.
        B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
        log('Cut to', len(B), 'bricks withOUT grz coverage')

    elif opt.region == 'deep2':
        rlo, rhi = 250, 260
        dlo, dhi = 30, 35

    elif opt.region == 'deep2f2':
        rlo, rhi = 251.4, 254.4
        dlo, dhi = 34.6, 35.3

    elif opt.region == 'deep2f3':
        rlo, rhi = 351.25, 353.75
        dlo, dhi = 0, 0.5

    elif opt.region == 'deep3':
        rlo, rhi = 214, 216
        dlo, dhi = 52.25, 53.25

    elif opt.region == 'virgo':
        rlo, rhi = 185, 190
        dlo, dhi = 10, 15

    elif opt.region == 'virgo2':
        rlo, rhi = 182, 192
        dlo, dhi = 8, 18

    elif opt.region == 'coma':
        # van Dokkum et al Coma cluster ultra-diffuse galaxies: 3x3 field centered on Coma cluster
        rc, dc = 195., 28.
        dd = 1.5
        cosdec = np.cos(np.deg2rad(dc))
        rlo, rhi = rc - dd / cosdec, rc + dd / cosdec
        dlo, dhi = dc - dd, dc + dd

    elif opt.region == 'lsb':
        rlo, rhi = 147.2, 147.8
        dlo, dhi = -0.4, 0.4

    elif opt.region == 'eboss-sgc':
        # generous boundaries to make sure get all relevant images
        # RA -45 to +45
        # Dec -5 to +7
        rlo, rhi = 310., 50.
        dlo, dhi = -6., 6.

    elif opt.region == 'eboss-ngc':
        # generous boundaries to make sure get all relevant images
        # NGC ELGs
        # RA 115 to 175
        # Dec 15 to  30
        # rlo,rhi = 122., 177.
        # dlo,dhi =  12.,  32.
        rlo, rhi = 126., 168.
        dlo, dhi = 18., 33.

    elif opt.region == 'mzls':
        dlo, dhi = -10., 90.  # -10: pull in Stripe 82 data too

    elif opt.region == 'dr4-bootes':
        # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched
        #dlo,dhi = 34., 35.
        #rlo,rhi = 209.5, 210.5
        dlo, dhi = 33., 36.
        rlo, rhi = 216.5, 219.5

    elif opt.region == 'des-sn-x3':
        #rlo,rhi = 36., 37.
        #dlo,dhi = -5., -4.
        rlo, rhi = 36., 36.5
        dlo, dhi = -4.5, -4.

    elif opt.region == 'ngc2632':
        # open cluster
        rlo, rhi = 129.0, 131.0
        dlo, dhi = 19.0, 20.5

    elif opt.region == 'dr8sky':
        rlo, rhi = 35.0, 37.0
        dlo, dhi = -3.0, -1.0

    # ADM DR8 test regions, see, e.g.:
    # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR8#Testregions
    elif opt.region == 'dr8-test-s82':
        rlo, rhi = 0, 45
        dlo, dhi = -1.25, 1.25
    elif opt.region == 'dr8-test-hsc-sgc':
        rlo, rhi = 30, 40
        dlo, dhi = -6.5, -1.25
    elif opt.region == 'dr8-test-hsc-ngc':
        rlo, rhi = 177.5, 182.5
        dlo, dhi = -1, 1
    elif opt.region == 'dr8-test-edr':
        rlo, rhi = 240, 245
        dlo, dhi = 5, 12
    elif opt.region == 'dr8-test-hsc-north':
        rlo, rhi = 240, 250
        dlo, dhi = 42, 45
    elif opt.region == 'dr8-test-deep2-egs':
        rlo, rhi = 213, 216.5
        dlo, dhi = 52, 54
    elif opt.region == 'dr8-test-overlap':
        rlo, rhi = 132, 140.5
        dlo, dhi = 31.5, 35

    if opt.mindec is not None:
        dlo = opt.mindec
    if opt.maxdec is not None:
        dhi = opt.maxdec
    if opt.minra is not None:
        rlo = opt.minra
    if opt.maxra is not None:
        rhi = opt.maxra

    if rlo < rhi:
        B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi))
    else:  # RA wrap
        B.cut(
            np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) *
            (B.dec <= dhi))
    log(len(B), 'bricks in range; cut Dec range', B.dec.min(), B.dec.max())
    #for name in B.get('brickname'):
    #    print(name)
    #B.writeto('bricks-cut.fits')

    bricksize = 0.25
    # A bit more than 0.25-degree brick radius + Bok image radius ~ 0.57
    search_radius = 1.05 * np.sqrt(2.) * (bricksize +
                                          (0.455 * 4096 / 3600.)) / 2.

    log(len(T), 'CCDs')
    log(len(B), 'Bricks')
    I, J, d = match_radec(B.ra,
                          B.dec,
                          T.ra,
                          T.dec,
                          search_radius,
                          nearest=True)
    B.cut(I)
    log('Cut to', len(B), 'bricks near CCDs')
    log('Bricks Dec range:', B.dec.min(), B.dec.max())

    # plt.clf()
    # plt.plot(B.ra, B.dec, 'b.')
    # plt.title('DR3 bricks')
    # plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1])
    # plt.savefig('bricks.png')

    if opt.touching:
        I, J, d = match_radec(T.ra,
                              T.dec,
                              B.ra,
                              B.dec,
                              search_radius,
                              nearest=True)
        # list the ones that will be cut
        # drop = np.ones(len(T))
        # drop[I] = False
        # for i in np.flatnonzero(drop):
        #     from astrometry.util.starutil_numpy import degrees_between
        #     dists = degrees_between(B.ra, B.dec, T.ra[i], T.dec[i])
        #     mindist = min(dists)
        #     print('Dropping:', T.ra[i], T.dec[i], 'min dist', mindist, 'search_radius', search_radius)

        T.cut(I)
        log('Cut to', len(T), 'CCDs near bricks')

    # sort by RA increasing
    B.cut(np.argsort(B.ra))

    if opt.save_to_fits:
        assert (opt.touching)
        # Write cut tables to file
        for tab, typ in zip([B, T], ['bricks', 'ccds']):
            fn = '%s-%s-cut.fits' % (typ, opt.region)
            if os.path.exists(fn):
                os.remove(fn)
            tab.writeto(fn)
            log('Wrote %s' % fn)
        # Write text files listing ccd and filename names
        # nm1,nm2= 'ccds-%s.txt'% opt.region,'filenames-%s.txt' % opt.region
        # if os.path.exists(nm1):
        #     os.remove(nm1)
        # if os.path.exists(nm2):
        #     os.remove(nm2)
        # f1,f2=open(nm1,'w'),open(nm2,'w')
        # fns= list(set(T.get('image_filename')))
        # for fn in fns:
        #     f2.write('%s\n' % fn.strip())
        # for ti in T:
        #     f1.write('%s\n' % ti.get('image_filename').strip())
        # f1.close()
        # f2.close()
        # log('Wrote *-names.txt')

    if opt.touching:

        if want_bricks:
            # Shortcut the list of bricks that are definitely touching CCDs --
            # a brick-ccd pair within this radius must be touching.
            closest_radius = 0.95 * (bricksize + 0.262 * 2048 / 3600.) / 2.

            J1, nil, nil = match_radec(B.ra,
                                       B.dec,
                                       T.ra,
                                       T.dec,
                                       closest_radius,
                                       nearest=True)
            log(len(J1), 'of', len(B), 'bricks definitely touch CCDs')
            tocheck = np.ones(len(B), bool)
            tocheck[J1] = False
            J2 = []
            for j in np.flatnonzero(tocheck):
                b = B[j]
                wcs = wcs_for_brick(b)
                I = ccds_touching_wcs(wcs, T)
                log(len(I), 'CCDs for brick', b.brickname)
                if len(I) == 0:
                    continue
                J2.append(j)
            J = np.hstack((J1, J2))
            J = np.sort(J).astype(int)
            B.cut(J)
            log('Cut to', len(B), 'bricks touching CCDs')

        else:
            J = []
            allI = set()
            for j, b in enumerate(B):
                wcs = wcs_for_brick(b)
                I = ccds_touching_wcs(wcs, T)
                log(len(I), 'CCDs for brick', b.brickname)
                if len(I) == 0:
                    continue
                allI.update(I)
                J.append(j)
            allI = list(allI)
            allI.sort()
            B.cut(np.array(J))
            log('Cut to', len(B), 'bricks touching CCDs')

    elif opt.near:
        # Find CCDs near bricks
        allI, nil, nil = match_radec(T.ra,
                                     T.dec,
                                     B.ra,
                                     B.dec,
                                     search_radius,
                                     nearest=True)
        # Find bricks near CCDs
        J, nil, nil = match_radec(B.ra,
                                  B.dec,
                                  T.ra,
                                  T.dec,
                                  search_radius,
                                  nearest=True)
        B.cut(J)
        log('Cut to', len(B), 'bricks near CCDs')

    else:
        allI = np.arange(len(T))

    if opt.byexp:
        nil, eI = np.unique(T.expnum[allI], return_index=True)
        allI = allI[eI]
        print('Cut to', len(allI), 'expnums')

    if opt.nccds:
        from queue import Queue
        from threading import Thread

        log('Checking number of CCDs per brick')

        def worker():
            while True:
                i = q.get()
                if i is None:
                    break
                b = B[i]
                wcs = wcs_for_brick(b)
                I = ccds_touching_wcs(wcs, T)
                log(b.brickname, len(I))
                q.task_done()

        q = Queue()
        num_threads = 24
        threads = []

        for i in range(num_threads):
            t = Thread(target=worker)
            t.start()
            threads.append(t)

        for i in range(len(B)):
            q.put(i)

        q.join()
        for i in range(num_threads):
            q.put(None)
        for t in threads:
            t.join()

    if opt.write_ccds:
        T[allI].writeto(opt.write_ccds)
        log('Wrote', opt.write_ccds)

    if want_bricks:
        # Print the list of bricks and exit.
        for b in B:
            print(b.brickname)
        if opt.save_to_fits:
            B.writeto('bricks-%s-touching.fits' % opt.region)
        if not want_ccds:
            sys.exit(0)

    ## Be careful here -- T has been cut; we want to write out T.index.
    ## 'allI' contains indices into T.

    if opt.stage is not None:
        cmd_pat = 'rsync -LRarv %s %s'
        fns = set()
        for iccd in allI:
            im = survey.get_image_object(T[iccd])
            fns.update([
                im.imgfn, im.wtfn, im.dqfn, im.psffn, im.merged_psffn,
                im.merged_splineskyfn, im.splineskyfn
            ])
        for i, fn in enumerate(fns):
            print('File', i + 1, 'of', len(fns), ':', fn)
            if not os.path.exists(fn):
                print('No such file:', fn)
                continue
            base = survey.get_survey_dir()
            if base.endswith('/'):
                base = base[:-1]

            rel = os.path.relpath(fn, base)

            dest = os.path.join(opt.stage, rel)
            print('Dest:', dest)
            if os.path.exists(dest):
                print('Exists:', dest)
                continue

            cmd = cmd_pat % ('%s/./%s' % (base, rel), opt.stage)
            print(cmd)
            rtn = os.system(cmd)
            assert (rtn == 0)
        sys.exit(0)

    if opt.forced:
        log('Writing forced-photometry commands to', opt.out)
        f = open(opt.out, 'w')
        log('Total of', len(allI), 'CCDs')
        for j, i in enumerate(allI):
            expstr = '%08i' % T.expnum[i]
            imgfn = os.path.join(survey.survey_dir, 'images',
                                 T.image_filename[i].strip())
            if (not os.path.exists(imgfn) and imgfn.endswith('.fz')
                    and os.path.exists(imgfn[:-3])):
                imgfn = imgfn[:-3]

            outfn = os.path.join(
                expstr[:5], expstr, 'forced-%s-%s-%s.fits' %
                (T.camera[i].strip(), expstr, T.ccdname[i]))

            f.write(
                'python legacypipe/forced_photom.py --apphot --derivs --catalog-dir /project/projectdirs/cosmo/data/legacysurvey/dr7/ %i %s forced/%s\n'
                % (T.expnum[i], T.ccdname[i], outfn))

        f.close()
        log('Wrote', opt.out)

        fn = 'forced-ccds.fits'
        T[allI].writeto(fn)
        print('Wrote', fn)

        sys.exit(0)

    if opt.lsb:
        log('Writing LSB commands to', opt.out)
        f = open(opt.out, 'w')
        log('Total of', len(allI), 'CCDs')
        for j, i in enumerate(allI):
            exp = T.expnum[i]
            ext = T.ccdname[i].strip()
            outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
            f.write(
                'python legacyanalysis/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n'
                % (exp, ext, outfn, exp, ext))
        f.close()
        log('Wrote', opt.out)
        sys.exit(0)

    log('Writing calibs to', opt.out)
    f = open(opt.out, 'w')
    log('Total of', len(allI), 'CCDs')

    batch = []

    def write_batch(f, batch, cmd):
        if cmd is None:
            cmd = ''
        f.write(cmd + ' '.join(batch) + '\n')

    cmd = None
    if opt.command:
        cmd = 'python legacypipe/run-calib.py '
        if opt.opt is not None:
            cmd += opt.opt + ' '

    for j, i in enumerate(allI):

        if opt.delete_sky:
            log(j + 1, 'of', len(allI))
            im = survey.get_image_object(T[i])
            if opt.delete_sky and os.path.exists(im.skyfn):
                log('  deleting:', im.skyfn)
                os.unlink(im.skyfn)

        if opt.command:
            if opt.byexp:
                s = '--expnum %i' % (T.expnum[i])
            else:
                s = '%i-%s' % (T.expnum[i], T.ccdname[i])
            prefix = 'python legacypipe/run-calib.py '
            if opt.opt is not None:
                prefix = prefix + opt.opt
            #('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
            #     (T.expnum[i], T.ccdname[i]))
        else:
            s = '%i' % T.index[i]
            prefix = ''

        if j < 10:
            print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname',
                  T.ccdname[i], 'filename', T.image_filename[i])

        if not opt.nper:
            f.write(prefix + s + '\n')
        else:
            batch.append(s)
            if len(batch) >= opt.nper:
                write_batch(f, batch, cmd)
                batch = []

    if len(batch):
        write_batch(f, batch, cmd)

    f.close()
    log('Wrote', opt.out)
    return 0
Exemplo n.º 21
0
def run_one_brick(X):
    brick, ibrick, nbricks, plots, kwargs = X

    survey = LegacySurveyData()

    print()
    print()
    print('Brick', (ibrick + 1), 'of', nbricks, ':', brick.brickname)

    dirnm = os.path.join('depthcuts', brick.brickname[:3])
    outfn = os.path.join(dirnm, 'ccds-%s.fits' % brick.brickname)
    if os.path.exists(outfn):
        print('Exists:', outfn)
        return 0

    H, W = 3600, 3600
    pixscale = 0.262
    bands = ['g', 'r', 'z']

    # Get WCS object describing brick
    targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale)
    targetrd = np.array([
        targetwcs.pixelxy2radec(x, y)
        for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
    ])
    gitver = get_git_version()

    ccds = survey.ccds_touching_wcs(targetwcs)
    if ccds is None:
        print('No CCDs actually touching brick')
        return 0
    print(len(ccds), 'CCDs actually touching brick')

    ccds.cut(np.in1d(ccds.filter, bands))
    print('Cut on filter:', len(ccds), 'CCDs remain.')

    if 'ccd_cuts' in ccds.get_columns():
        norig = len(ccds)
        ccds.cut(ccds.ccd_cuts == 0)
        print(len(ccds), 'of', norig, 'CCDs pass cuts')
    else:
        print('No CCD cuts')

    if len(ccds) == 0:
        print('No CCDs left')
        return 0

    ps = None
    if plots:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence('depth-%s' % brick.brickname)

    splinesky = True
    gaussPsf = False
    pixPsf = True
    do_calibs = False
    normalizePsf = True

    get_depth_maps = kwargs.pop('get_depth_maps', False)

    try:
        D = make_depth_cut(survey,
                           ccds,
                           bands,
                           targetrd,
                           brick,
                           W,
                           H,
                           pixscale,
                           plots,
                           ps,
                           splinesky,
                           gaussPsf,
                           pixPsf,
                           normalizePsf,
                           do_calibs,
                           gitver,
                           targetwcs,
                           get_depth_maps=get_depth_maps,
                           **kwargs)
        if get_depth_maps:
            keep, overlapping, depthmaps = D
        else:
            keep, overlapping = D
    except:
        print('Failed to make_depth_cut():')
        import traceback
        traceback.print_exc()
        return -1

    print(np.sum(overlapping), 'CCDs overlap the brick')
    print(np.sum(keep), 'CCDs passed depth cut')
    ccds.overlapping = overlapping
    ccds.passed_depth_cut = keep

    if not os.path.exists(dirnm):
        try:
            os.makedirs(dirnm)
        except:
            pass

    if get_depth_maps:
        for band, depthmap in depthmaps:
            doutfn = os.path.join(dirnm,
                                  'depth-%s-%s.fits' % (brick.brickname, band))
            hdr = fitsio.FITSHDR()
            # Plug the WCS header cards into these images
            targetwcs.add_to_header(hdr)
            hdr.delete('IMAGEW')
            hdr.delete('IMAGEH')
            hdr.add_record(dict(name='EQUINOX', value=2000.))
            hdr.add_record(dict(name='FILTER', value=band))
            fitsio.write(doutfn, depthmap, header=hdr)
            print('Wrote', doutfn)

    tmpfn = os.path.join(os.path.dirname(outfn),
                         'tmp-' + os.path.basename(outfn))
    ccds.writeto(tmpfn)
    os.rename(tmpfn, outfn)
    print('Wrote', outfn)

    return 0
Exemplo n.º 22
0
'''
After running forced_photom.py on a set of CCDs, this script merges
the results back into a catalog.
'''
from astrometry.util.fits import *
import numpy as np
from glob import glob
from collections import Counter

from legacypipe.survey import LegacySurveyData

fns = glob('forced/*/*/forced-*.fits')
F = merge_tables([fits_table(fn) for fn in fns])

dr6 = LegacySurveyData('/project/projectdirs/cosmo/data/legacysurvey/dr6')
B = dr6.get_bricks_readonly()

I = np.flatnonzero((B.ra1 < F.ra.max()) * (B.ra2 > F.ra.min()) * (B.dec1 < F.dec.max()) * (B.dec2 > F.dec.min()))
print(len(I), 'bricks')
T = merge_tables([fits_table(dr6.find_file('tractor', brick=B.brickname[i])) for i in I])
print(len(T), 'sources')
T.cut(T.brick_primary)
print(len(T), 'primary')

# map from F to T index
imap = dict([((b,o),i) for i,(b,o) in enumerate(zip(T.brickid, T.objid))])
F.tindex = np.array([imap[(b,o)] for b,o in zip(F.brickid, F.objid)])
assert(np.all(T.brickid[F.tindex] == F.brickid))
assert(np.all(T.objid[F.tindex] == F.objid))

fcols = 'apflux apflux_ivar camera expnum ccdname exptime flux flux_ivar fracflux mask mjd rchi2 x y brickid objid'.split()
Exemplo n.º 23
0
def main(args=None):
    """Main routine which parses the optional inputs."""
    t0= Time()
    # Command line options
    if args is None:
        # Read from cmd line
        parser= get_parser()
        args = parser.parse_args(args=args)
    else:
        # args is already a argparse.Namespace obj
        pass
    # Print calling sequence
    print('Args:', args)
    if args.do_more == 'yes':
      assert(not args.minid is None)
    # Setup loggers
    if args.verbose:
        lvl = logging.DEBUG
    else:
        lvl = logging.INFO
    logging.basicConfig(level=lvl, stream=sys.stdout) #,format='%(message)s')
    log = logging.getLogger('decals_sim')
    # Sort through args
    #log.info('decals_sim.py args={}'.format(args))
    #max_nobj=500
    #max_nchunk=1000
    #if args.ith_chunk is not None: assert(args.ith_chunk <= max_nchunk-1)
    #assert(args.nchunk <= max_nchunk)
    #assert(args.nobj <= max_nobj)
    #if args.ith_chunk is not None:
    #    assert(args.nchunk == 1) #if choose a chunk, only doing 1 chunk
    if args.nobj is None:
        parser.print_help()
        sys.exit(1)

    # Exit if expected output already exists
    rsdir= get_outdir_runbrick(args.outdir,
                        args.brick,args.rowstart,
                        do_skipids=args.do_skipids,
                        do_more=args.do_more)
    rsdir= os.path.basename(rsdir)
    tractor_fn= os.path.join(args.outdir,
                    'tractor',args.brick[:3],args.brick,
                    rsdir,
                    'tractor-%s.fits' % args.brick)
    if (os.path.exists(tractor_fn) &
        (not args.overwrite_if_exists)):
       print('Exiting, already finished %s' % tractor_fn)
       return 0 #sys.exit(0)

    brickname = args.brick
    objtype = args.objtype

    # Output dir
    decals_sim_dir = args.outdir

    #nchunk = args.nchunk
    #rand = np.random.RandomState(args.seed) # determines seed for all chunks
    #seeds = rand.random_integers(0,2**18, max_nchunk)

    log.info('Object type = {}'.format(objtype))
    #log.info('Number of objects = {}'.format(nobj))
    #log.info('Number of chunks = {}'.format(nchunk))
    # Optionally zoom into a portion of the brick
    survey = LegacySurveyData(survey_dir=args.survey_dir)
    brickinfo= get_brickinfo_hack(survey,brickname)
    #brickinfo = survey.get_brick_by_name(brickname)
    #print(brickname)
    brickwcs = wcs_for_brick(brickinfo)
    W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale()

    log.info('Brick = {}'.format(brickname))
    if args.zoom is not None: # See also runbrick.stage_tims()
        (x0, x1, y0, y1) = args.zoom
        W = x1 - x0
        H = y1 - y0
        brickwcs = brickwcs.get_subimage(x0, y0, W, H)
        log.info('Zoom (pixel boundaries) = {}'.format(args.zoom))
    targetrd = np.array([brickwcs.pixelxy2radec(x, y) for x, y in
                         [(1,1), (W,1), (W,H), (1,H), (1,1)]])

    radec_center = brickwcs.radec_center()
    log.info('RA, Dec center = {}'.format(radec_center))
    log.info('Brick = {}'.format(brickname))
    t0= ptime('First part of Main()',t0)

    # SAMPLE table
    sample_kwargs= {"objtype":args.objtype,
                    "brick":args.brick,
                    "outdir":args.outdir,
                    "randoms_db":args.randoms_db,
                    "minid":args.minid,
                    "do_skipids":args.do_skipids,
                    "randoms_from_fits":args.randoms_from_fits,
                    "dont_sort_sampleid":args.dont_sort_sampleid}
    Samp,seed= get_sample(**sample_kwargs)

    Samp= Samp[args.rowstart:args.rowstart + args.nobj]
    # Performance
    #if objtype in ['elg','lrg']:
    #    Samp=Samp[np.argsort( Samp.get('%s_n' % objtype) )]
    print('Max sample size=%d, actual sample size=%d' % (args.nobj,len(Samp)))
    assert(len(Samp) <= args.nobj)
    t0= ptime('Got randoms sample',t0)

    # Store args in dict for easy func passing
    kwargs=dict(Samp=Samp,\
                brickname=brickname, \
                checkpoint=args.checkpoint, \
                seed= seed,
                decals_sim_dir= decals_sim_dir,\
                brickwcs= brickwcs, \
                objtype=objtype,\
                nobj=len(Samp),\
                maxobjs=args.nobj,\
                rowst=args.rowstart,\
                do_skipids=args.do_skipids,\
                do_more=args.do_more,\
                minid=args.minid,\
                survey_dir=args.survey_dir,\
                args=args)

    # Stop if starting row exceeds length of radec,color table
    if len(Samp) == 0:
        fn= get_outdir_runbrick(kwargs['decals_sim_dir'],
                        kwargs['brickname'],kwargs['rowst'],
                        do_skipids=kwargs['do_skipids'],do_more=kwargs['do_more'])
        fn+= '_exceeded.txt'
        junk= os.system('touch %s' % fn)
        print('Wrote %s' % fn)
        #we want not to add any sample -- obiwan
        #raise ValueError('starting row=%d exceeds number of artificial sources, quit' % args.rowstart)

    # Create simulated catalogues and run Tractor
    create_metadata(kwargs=kwargs)
    t0= ptime('create_metadata',t0)
    # do chunks
    #for ith_chunk in chunk_list:
    #log.info('Working on chunk {:02d}/{:02d}'.format(ith_chunk,kwargs['nchunk']-1))
    # Random ra,dec and source properties
    create_ith_simcat(d=kwargs)
    #log.info('HUI-TEST:::out of create_ith_simcat')
    t0= ptime('create_ith_simcat',t0)
    # Run tractor
    #log.info('HUI-TEST:::running tractor')
    do_one_chunk(d=kwargs)
    #log.info('HUI-TEST::: checkpoint3i')
    t0= ptime('do_one_chunk',t0)
    # Clean up output
    if args.no_cleanup == False:
        do_ith_cleanup(d=kwargs)
    #log.info('HUI-TEST::: checkpoint3j')
    t0= ptime('do_ith_cleanup',t0)
    log.info('All done!')
    #log.info('HUI-TEST::: checkpoint3k')
    return 0
Exemplo n.º 24
0
def main():
    survey = LegacySurveyData()

    brickname = '2351p137'
    # RA,Dec = 235.0442, 13.7125
    bx, by = 3300, 1285
    sz = 50
    bbox = [bx - sz, bx + sz, by - sz, by + sz]
    objid = 1394
    bands = ['g', 'r', 'z']

    from legacypipe.runbrick import stage_tims, _get_mod, rgbkwargs, rgbkwargs_resid
    from legacypipe.survey import get_rgb, imsave_jpeg
    from legacypipe.coadds import make_coadds
    from astrometry.util.multiproc import multiproc
    from astrometry.util.fits import fits_table
    from legacypipe.catalog import read_fits_catalog

    # brick = survey.get_brick_by_name(brickname)
    # # Get WCS object describing brick
    # targetwcs = wcs_for_brick(brick)
    # (x0,x1,y0,y1) = bbox
    # W = x1-x0
    # H = y1-y0
    # targetwcs = targetwcs.get_subimage(x0, y0, W, H)
    # H,W = targetwcs.shape

    mp = multiproc()
    P = stage_tims(brickname=brickname,
                   survey=survey,
                   target_extent=bbox,
                   pixPsf=True,
                   hybridPsf=True,
                   depth_cut=False,
                   mp=mp)
    print('Got', P.keys())

    tims = P['tims']
    targetwcs = P['targetwcs']
    H, W = targetwcs.shape

    # Read Tractor catalog
    fn = survey.find_file('tractor', brick=brickname)
    print('Trying to read catalog', fn)
    cat = fits_table(fn)
    print('Read', len(cat), 'sources')
    ok, xx, yy = targetwcs.radec2pixelxy(cat.ra, cat.dec)
    I = np.flatnonzero((xx > 0) * (xx < W) * (yy > 0) * (yy < H))
    cat.cut(I)
    print('Cut to', len(cat), 'sources within box')

    I = np.flatnonzero(cat.objid != objid)
    cat.cut(I)
    print('Cut to', len(cat), 'sources with objid !=', objid)

    #cat.about()
    # Convert FITS catalog into tractor source objects
    print('Creating tractor sources...')

    srcs = read_fits_catalog(cat, fluxPrefix='')
    print('Sources:')
    for src in srcs:
        print(' ', src)

    print('Rendering model images...')
    mods = [_get_mod((tim, srcs)) for tim in tims]

    print('Producing coadds...')
    C = make_coadds(tims, bands, targetwcs, mods=mods, mp=mp)
    print('Coadds:', dir(C))

    coadd_list = [('image', C.coimgs, rgbkwargs),
                  ('model', C.comods, rgbkwargs),
                  ('resid', C.coresids, rgbkwargs_resid)]
    #C.coimgs, C.comods, C.coresids
    for name, ims, rgbkw in coadd_list:
        rgb = get_rgb(ims, bands, **rgbkw)
        kwa = {}
        #with survey.write_output(name + '-jpeg', brick=brickname) as out:
        #    imsave_jpeg(out.fn, rgb, origin='lower', **kwa)
        #    print('Wrote', out.fn)
        outfn = name + '.jpg'
        imsave_jpeg(outfn, rgb, origin='lower', **kwa)
        del rgb
Exemplo n.º 25
0
def main():
    ps = PlotSequence('cov')

    survey = LegacySurveyData()

    ra, dec = 242.0, 10.2

    fn = 'coverage-ccds.fits'
    if not os.path.exists(fn):
        ccds = survey.get_ccds()
        ccds.cut(ccds.filter == 'r')
        ccds.cut(ccds.propid == '2014B-0404')
        ccds.cut(np.hypot(ccds.ra_bore - ra, ccds.dec_bore - dec) < 2.5)
        print(np.unique(ccds.expnum), 'unique exposures')
        print('propids', np.unique(ccds.propid))
        ccds.writeto(fn)
    else:
        ccds = fits_table(fn)

    plt.clf()
    for e in np.unique(ccds.expnum):
        I = np.flatnonzero(ccds.expnum == e)
        plt.plot(ccds.ra[I], ccds.dec[I], '.')
    ps.savefig()

    degw = 3.0
    pixscale = 10.

    W = degw * 3600 / 10.
    H = W

    hi = 6
    cmap = cmap_discretize('jet', hi + 1)

    wcs = Tan(ra, dec, W / 2. + 0.5, H / 2. + 0.5, -pixscale / 3600., 0., 0.,
              pixscale / 3600., float(W), float(H))

    r0, d0 = wcs.pixelxy2radec(1, 1)
    r1, d1 = wcs.pixelxy2radec(W, H)
    extent = [min(r0, r1), max(r0, r1), min(d0, d1), max(d0, d1)]

    for expnums in [
        [348666],
        [348666, 348710, 348686],
        [348659, 348667, 348658, 348666, 348665, 348669, 348668],
            None,
        [
            348683, 348687, 347333, 348686, 348685, 348692, 348694, 348659,
            348667, 348658, 348666, 348665, 348669, 348668, 348707, 348709,
            348708, 348710, 348711, 348716, 348717
        ],
    ]:

        nexp = np.zeros((H, W), np.uint8)

        for ccd in ccds:
            if expnums is not None and not ccd.expnum in expnums:
                continue

            ccdwcs = survey.get_approx_wcs(ccd)
            r, d = ccdwcs.pixelxy2radec(1, 1)
            ok, x0, y0 = wcs.radec2pixelxy(r, d)
            r, d = ccdwcs.pixelxy2radec(ccd.width, ccd.height)
            ok, x1, y1 = wcs.radec2pixelxy(r, d)
            xlo = np.clip(int(np.round(min(x0, x1))) - 1, 0, W - 1)
            xhi = np.clip(int(np.round(max(x0, x1))) - 1, 0, W - 1)
            ylo = np.clip(int(np.round(min(y0, y1))) - 1, 0, H - 1)
            yhi = np.clip(int(np.round(max(y0, y1))) - 1, 0, H - 1)
            nexp[ylo:yhi + 1, xlo:xhi + 1] += 1

        plt.clf()
        plt.imshow(nexp,
                   interpolation='nearest',
                   origin='lower',
                   vmin=-0.5,
                   vmax=hi + 0.5,
                   cmap=cmap,
                   extent=extent)
        plt.colorbar(ticks=np.arange(hi + 1))
        ps.savefig()

    O = fits_table('obstatus/decam-tiles_obstatus.fits')
    O.cut(np.hypot(O.ra - ra, O.dec - dec) < 2.5)

    for p in [1, 2, 3]:
        print('Pass', p, 'exposures:', O.r_expnum[O.get('pass') == p])

    O.cut(O.get('pass') == 2)
    print(len(O), 'pass 2 nearby')

    d = np.hypot(O.ra - ra, O.dec - dec)
    print('Dists:', d)

    I = np.flatnonzero(d < 0.5)
    assert (len(I) == 1)
    ocenter = O[I[0]]
    print('Center expnum', ocenter.r_expnum)

    I = np.flatnonzero(d >= 0.5)
    O.cut(I)

    #center = ccds[ccds.expnum == ocenter.r_expnum]
    #p2 = ccds[ccds.

    ok, xc, yc = wcs.radec2pixelxy(ocenter.ra, ocenter.dec)

    xx, yy = np.meshgrid(np.arange(W) + 1, np.arange(H) + 1)
    c_d2 = (xc - xx)**2 + (yc - yy)**2

    best = np.ones((H, W), bool)

    for o in O:
        ok, x, y = wcs.radec2pixelxy(o.ra, o.dec)
        d2 = (x - xx)**2 + (y - yy)**2
        best[d2 < c_d2] = False
        del d2

    del c_d2, xx, yy

    # plt.clf()
    # plt.imshow(best, interpolation='nearest', origin='lower', cmap='gray',
    #            vmin=0, vmax=1)
    # ps.savefig()

    plt.clf()
    plt.imshow(nexp * best,
               interpolation='nearest',
               origin='lower',
               vmin=-0.5,
               vmax=hi + 0.5,
               cmap=cmap,
               extent=extent)
    plt.colorbar(ticks=np.arange(hi + 1))
    ps.savefig()

    plt.clf()
    n, b, p = plt.hist(np.clip(nexp[best], 0, hi),
                       range=(-0.5, hi + 0.5),
                       bins=hi + 1)
    plt.xlim(-0.5, hi + 0.5)
    ps.savefig()

    print('b', b)
    print('n', n)
    print('fracs', np.array(n) / np.sum(n))

    print('pcts',
          ', '.join(['%.1f' % f for f in 100. * np.array(n) / np.sum(n)]))
Exemplo n.º 26
0
def main():

    indir = '/global/cscratch1/sd/dstn/dr8test-1'
    name = 'dr8-test1'
    pretty = 'DR8 test1'
    sublayers = ['', '-model', '-resid']
    subpretty = {'': ' images', '-model': ' models', '-resid': ' residuals'}
    survey_dir = '/global/cscratch1/sd/desiproc/dr7'
    datadir = 'data'

    survey = LegacySurveyData(survey_dir=survey_dir)

    fn = 'map/test_layers.py'
    txt = open(fn).read()
    for x in sublayers:
        txt = txt + '\n' + 'test_layers.append(("%s%s", "%s%s"))\n' % (
            name, x, pretty, subpretty[x])
    open(fn, 'wb').write(txt.encode())
    print('Wrote', fn)

    cmd = 'rsync -LRarv %s/./{coadd/*/*/*-{image,model}-*.fits*,tractor} %s/%s' % (
        indir, datadir, name)
    print(cmd)
    os.system(cmd)

    basedir = os.path.join(datadir, name)

    allbricks = survey.get_bricks_readonly()

    imagefns = glob(os.path.join(basedir, 'coadd', '*', '*',
                                 '*-image-*.fits*'))
    print('Image filenames:', len(imagefns))
    brickset = set()
    for fn in imagefns:
        dirs = fn.split('/')
        brickname = dirs[-2]
        brickset.add(brickname)
    print(len(brickset), 'bricks found')

    I, = np.nonzero([b in brickset for b in allbricks.brickname])
    bricks = allbricks[I]

    brickfn = os.path.join(basedir, 'survey-bricks.fits.gz')
    bricks.writeto(brickfn)
    print('Wrote', brickfn)

    threads = 8
    tharg = '--threads %i ' % threads

    # images
    for scale in range(1, 8):
        cmd = 'python -u render-tiles.py --kind %s --scale --zoom %i %s' % (
            name, scale, tharg)
        print(cmd)
        os.system(cmd)

    # models
    for scale in range(1, 8):
        cmd = 'python -u render-tiles.py --kind %s-model --scale --zoom %i %s' % (
            name, scale, tharg)
        print(cmd)
        os.system(cmd)

    for x in sublayers:
        cmd = 'python -u render-tiles.py --kind %s%s --top' % (name, x)
        print(cmd)
        os.system(cmd)
Exemplo n.º 27
0
alldec = []
allstate = []
alltasks = []

#     allra.append(ra)
#     alldec.append(dec)
#     allstate.append([state] * len(ra))
#     alltasks.append(tasks)

ra = np.hstack(allra)
dec = np.hstack(alldec)
state = np.hstack(allstate)
tasks = np.hstack(alltasks)

# Match to actual table of bricks to get brickq.
survey = LegacySurveyData()
bricks = survey.get_bricks_readonly()
I,J,d = match_radec(ra, dec, bricks.ra, bricks.dec, 0.2, nearest=True)
print(len(ra), 'jobs')
print(len(I), 'matches')
ra = ra[I]
dec = dec[I]
state = state[I]
tasks = tasks[I]
brickq = bricks.brickq[J]

for q in [0,1,2,3]:

    print()
    print('Brickq', q)
    plt.clf()
Exemplo n.º 28
0
def psf_residuals(expnum,
                  ccdname,
                  stampsize=35,
                  nstar=30,
                  magrange=(13, 17),
                  verbose=0,
                  splinesky=False):

    # Set the debugging level.
    if verbose == 0:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG
    logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)

    pngprefix = 'qapsf-{}-{}'.format(expnum, ccdname)

    # Gather all the info we need about this CCD.
    survey = LegacySurveyData()
    ccd = survey.find_ccds(expnum=expnum, ccdname=ccdname)[0]
    band = ccd.filter
    ps1band = dict(g=0, r=1, i=2, z=3, Y=4)
    print('Band {}'.format(band))

    #scales = dict(g=0.0066, r=0.01, z=0.025)
    #vmin, vmax = np.arcsinh(-1), np.arcsinh(100)
    #print(scales[band])

    im = survey.get_image_object(ccd)
    iminfo = im.get_image_info()
    H, W = iminfo['dims']

    wcs = im.get_wcs()

    # Choose a uniformly selected subset of PS1 stars on this CCD.
    ps1 = ps1cat(ccdwcs=wcs)
    cat = ps1.get_stars(band=band, magrange=magrange)

    rand = np.random.RandomState(seed=expnum * ccd.ccdnum)
    these = rand.choice(len(cat) - 1, nstar, replace=False)
    #these = rand.random_integers(0,len(cat)-1,nstar)
    cat = cat[these]
    cat = cat[np.argsort(cat.median[:, ps1band[band]])]  # sort by magnitude
    #print(cat.nmag_ok)

    get_tim_kwargs = dict(pixPsf=True, splinesky=splinesky)

    # Make a QAplot of the positions of all the stars.
    tim = im.get_tractor_image(**get_tim_kwargs)
    img = tim.getImage()
    #img = tim.getImage()/scales[band]

    fig = plt.figure(figsize=(5, 10))
    ax = fig.gca()
    ax.get_xaxis().get_major_formatter().set_useOffset(False)
    #ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest',
    #          origin='lower',vmin=vmax,vmax=vmax)

    ax.imshow(img, **tim.ima)
    ax.axis('off')
    ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.format(
        band, expnum, ccdname, ccd.airmass, ccd.seeing))

    for istar, ps1star in enumerate(cat):
        ra, dec = (ps1star.ra, ps1star.dec)
        ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
        ax.text(xpos,
                ypos,
                '{:2d}'.format(istar + 1),
                color='red',
                horizontalalignment='left')
        circ = plt.Circle((xpos, ypos), radius=30, color='g', fill=False, lw=1)
        ax.add_patch(circ)

    #radec = wcs.radec_bounds()
    #ax.scatter(cat.ra,cat.dec)
    #ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998])
    #ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015])
    #ax.set_xlabel('$RA\ (deg)$',fontsize=18)
    #ax.set_ylabel('$Dec\ (deg)$',fontsize=18)
    fig.savefig(pngprefix + '-ccd.png', bbox_inches='tight')

    # Initialize the many-stamp QAplot
    ncols = 3
    nrows = np.ceil(nstar / ncols).astype('int')

    inchperstamp = 2.0
    fig = plt.figure(figsize=(inchperstamp * 3 * ncols, inchperstamp * nrows))
    irow = 0
    icol = 0

    for istar, ps1star in enumerate(cat):
        ra, dec = (ps1star.ra, ps1star.dec)
        mag = ps1star.median[ps1band[band]]  # r-band

        ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
        ix, iy = int(xpos), int(ypos)

        # create a little tractor Image object around the star
        slc = (slice(max(iy - stampsize, 0), min(iy + stampsize + 1, H)),
               slice(max(ix - stampsize, 0), min(ix + stampsize + 1, W)))

        # The PSF model 'const2Psf' is the one used in DR1: a 2-component
        # Gaussian fit to PsfEx instantiated in the image center.
        tim = im.get_tractor_image(slc=slc, **get_tim_kwargs)
        stamp = tim.getImage()
        ivarstamp = tim.getInvvar()

        # Initialize a tractor PointSource from PS1 measurements
        flux = NanoMaggies.magToNanomaggies(mag)
        star = PointSource(RaDecPos(ra, dec), NanoMaggies(**{band: flux}))

        # Fit just the source RA,Dec,flux.
        tractor = Tractor([tim], [star])
        tractor.freezeParam('images')

        print('2-component MOG:', tim.psf)
        tractor.printThawedParams()

        for step in range(50):
            dlnp, X, alpha = tractor.optimize()
            if dlnp < 0.1:
                break
        print('Fit:', star)
        model_mog = tractor.getModelImage(0)
        chi2_mog = -2.0 * tractor.getLogLikelihood()
        mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0]

        # Now change the PSF model to a pixelized PSF model from PsfEx instantiated
        # at this place in the image.
        psf = PixelizedPsfEx(im.psffn)
        tim.psf = psf.constantPsfAt(xpos, ypos)

        #print('PSF model:', tim.psf)
        #tractor.printThawedParams()
        for step in range(50):
            dlnp, X, alpha = tractor.optimize()
            if dlnp < 0.1:
                break

        print('Fit:', star)
        model_psfex = tractor.getModelImage(0)
        chi2_psfex = -2.0 * tractor.getLogLikelihood()
        mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0]

        #mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95])
        sig = np.std((stamp - model_psfex)[ivarstamp > 0])
        mn, mx = [-2.0 * sig, 5 * sig]

        # Generate a QAplot.
        if (istar > 0) and (istar % (ncols) == 0):
            irow = irow + 1
        icol = 3 * istar - 3 * ncols * irow
        #print(istar, irow, icol, icol+1, icol+2)

        ax1 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol),
                               aspect='equal')
        ax1.axis('off')
        #ax1.imshow(stamp, **tim.ima)
        ax1.imshow(stamp,
                   cmap='gray',
                   interpolation='nearest',
                   origin='lower',
                   vmin=mn,
                   vmax=mx)
        ax1.text(0.1,
                 0.9,
                 '{:2d}'.format(istar + 1),
                 color='white',
                 horizontalalignment='left',
                 verticalalignment='top',
                 transform=ax1.transAxes)

        ax2 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 1),
                               aspect='equal')
        ax2.axis('off')
        #ax2.imshow(stamp-model_mog, **tim.ima)
        ax2.imshow(stamp - model_mog,
                   cmap='gray',
                   interpolation='nearest',
                   origin='lower',
                   vmin=mn,
                   vmax=mx)
        ax2.text(0.1,
                 0.9,
                 'MoG',
                 color='white',
                 horizontalalignment='left',
                 verticalalignment='top',
                 transform=ax2.transAxes)
        ax2.text(0.08,
                 0.08,
                 '{:.3f}'.format(mag_mog),
                 color='white',
                 horizontalalignment='left',
                 verticalalignment='bottom',
                 transform=ax2.transAxes)

        #ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14)
        #ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex))

        ax3 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 2),
                               aspect='equal')
        ax3.axis('off')
        #ax3.imshow(stamp-model_psfex, **tim.ima)
        ax3.imshow(stamp - model_psfex,
                   cmap='gray',
                   interpolation='nearest',
                   origin='lower',
                   vmin=mn,
                   vmax=mx)
        ax3.text(0.1,
                 0.9,
                 'PSFEx',
                 color='white',
                 horizontalalignment='left',
                 verticalalignment='top',
                 transform=ax3.transAxes)
        ax3.text(0.08,
                 0.08,
                 '{:.3f}'.format(mag_psfex),
                 color='white',
                 horizontalalignment='left',
                 verticalalignment='bottom',
                 transform=ax3.transAxes)

        if istar == (nstar - 1):
            break
    fig.savefig(pngprefix + '-stargrid.png', bbox_inches='tight')
Exemplo n.º 29
0
def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--expnum',
        type=str,
        help='Run specified exposure numbers (can be comma-separated list')
    parser.add_argument(
        '--all-found',
        action='store_true',
        default=False,
        help='Only write output if all required input files are found')
    parser.add_argument('--ccds',
                        help='Set ccds.fits file to load, default is all')
    parser.add_argument('--continue',
                        dest='con',
                        help='Continue even if one exposure is bad',
                        action='store_true',
                        default=False)
    parser.add_argument('--outdir',
                        help='Output directory, default %(default)s',
                        default='calib')

    opt = parser.parse_args()

    survey = LegacySurveyData()
    if opt.ccds:
        ccds = fits_table(opt.ccds)
        ccds = survey.cleanup_ccds_table(ccds)
        survey.ccds = ccds

    if opt.expnum is not None:
        expnums = [(None, int(x, 10)) for x in opt.expnum.split(',')]
    else:
        ccds = survey.get_ccds()
        expnums = set(zip(ccds.camera, ccds.expnum))
        print(len(expnums), 'unique camera+expnums')

    for i, (camera, expnum) in enumerate(expnums):
        print()
        print('Exposure', i + 1, 'of', len(expnums), ':', camera, 'expnum',
              expnum)
        if camera is None:
            C = survey.find_ccds(expnum=expnum)
            print(len(C), 'CCDs with expnum', expnum)
            camera = C.camera[0]
            print('Set camera to', camera)

        C = survey.find_ccds(expnum=expnum, camera=camera)
        print(len(C), 'CCDs with expnum', expnum, 'and camera', camera)

        im0 = survey.get_image_object(C[0])

        skyoutfn = im0.merged_skyfn
        psfoutfn = im0.merged_psffn

        print('Checking for', skyoutfn)
        print('Checking for', psfoutfn)
        if os.path.exists(skyoutfn) and os.path.exists(psfoutfn):
            print('Exposure', expnum, 'is done already')
            continue

        if not os.path.exists(skyoutfn):
            try:
                merge_splinesky(survey, expnum, C, skyoutfn, opt)
            except:
                if not opt.con:
                    raise
                import traceback
                traceback.print_exc()
                print('Exposure failed:', expnum, '.  Continuing...')

        if not os.path.exists(psfoutfn):
            try:
                merge_psfex(survey, expnum, C, psfoutfn, opt)
            except:
                if not opt.con:
                    raise
                import traceback
                traceback.print_exc()
                print('Exposure failed:', expnum, '.  Continuing...')
Exemplo n.º 30
0
        type=str,
        default=None,
        help='Override the $LEGACY_SURVEY_DIR environment variable')
    parser.add_argument('--out',
                        '-o',
                        default='skyfibers.fits',
                        help='Output filename')
    parser.add_argument('--plots',
                        '-p',
                        default=None,
                        help='Plots base filename')
    parser.add_argument('--brick', default=None, help='Brick name')

    opt = parser.parse_args()
    if not opt.brick:
        parser.print_help()
        sys.exit(-1)

    from legacypipe.survey import LegacySurveyData

    survey = LegacySurveyData(survey_dir=opt.survey_dir)

    skyfibers = sky_fibers_for_brick(survey, opt.brick)
    skyfibers.writeto(opt.out, header=skyfibers._header)
    print('Wrote', opt.out)

    if opt.plots:
        import matplotlib
        matplotlib.use('Agg')
        sky_fiber_plots(survey, opt.brick, skyfibers, opt.plots)