Exemple #1
0
            ps.savefig()

        imx = np.argmax(m22)
        diff = np.diff(m22[:imx + 1])
        # Assert monotonic up to the peak (from the left)
        # (low-level jitter/wrap-around is allowed)
        self.assertTrue(np.all(np.logical_or(np.abs(diff) < 1e-9, diff > 0)))

        diff = np.diff(m22[imx:])
        # Assert monotonic decreasing after to the peak
        # (low-level jitter/wrap-around is allowed)
        self.assertTrue(np.all(np.logical_or(np.abs(diff) < 1e-9, diff < 0)))

        # Assert that wrap-around exists for PixelizedPsf model

        diff = np.diff(m21[:imx + 1])
        self.assertFalse(np.all(np.logical_or(np.abs(diff) < 1e-9, diff > 0)))

        diff = np.diff(m21[imx:])
        self.assertFalse(np.all(np.logical_or(np.abs(diff) < 1e-9, diff < 0)))


if __name__ == '__main__':
    import sys
    if '--plots' in sys.argv:
        sys.argv.remove('--plots')
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence('gal')

    unittest.main()
def showSipSolutions(srcs, wcs0, andDir, x0, y0, W, H, filterName, plotPrefix):
    '''
    srcs: afw Catalog of sources
    wcs0: original WCS
    andDir: astrometry_net_data directory
    '''
    imargs = dict(imageSize=(W, H), filterName=filterName, x0=x0, y0=y0)

    # Set up astrometry_net_data
    os.environ['ASTROMETRY_NET_DATA_DIR'] = andDir
    andConfig = measAstrom.AstrometryNetDataConfig()
    fn = os.path.join(andDir, 'andConfig.py')
    andConfig.load(fn)

    # Set up meas_astrom
    conf = measAstrom.ANetBasicAstrometryConfig(sipOrder=4)
    ast = measAstrom.ANetBasicAstrometryTask(conf,
                                             andConfig,
                                             logLevel=Log.DEBUG)

    # What reference sources are in the original WCS
    refs = ast.getReferenceSourcesForWcs(wcs0, **imargs)
    print('Got', len(refs), 'reference objects for initial WCS')

    # How does a straight TAN solution look?
    conf2 = measAstrom.ANetBasicAstrometryConfig(sipOrder=4,
                                                 calculateSip=False)
    ast2 = measAstrom.ANetBasicAstrometryTask(conf2,
                                              andConfig,
                                              logLevel=Log.DEBUG)
    solve = ast2.determineWcs2(srcs, **imargs)
    tanwcs = solve.tanWcs

    # How about if we fit a SIP WCS using the *original* WCS?
    wcs2 = ast.getSipWcsFromWcs(wcs0, (W, H), x0=x0, y0=y0)

    # (We determineWcs() for a SIP solution below...)

    # Make some plots in pixel space by pushing ref sources through WCSes
    rx0, ry0 = [], []
    rx2, ry2 = [], []
    rx3, ry3 = [], []
    for src in refs:
        xy = wcs0.skyToPixel(src.getCoord())
        rx0.append(xy[0])
        ry0.append(xy[1])

        xy = tanwcs.skyToPixel(src.getCoord())
        rx2.append(xy[0])
        ry2.append(xy[1])

        xy = wcs2.skyToPixel(src.getCoord())
        rx3.append(xy[0])
        ry3.append(xy[1])

    rx0 = np.array(rx0)
    ry0 = np.array(ry0)
    rx2 = np.array(rx2)
    ry2 = np.array(ry2)
    rx3 = np.array(rx3)
    ry3 = np.array(ry3)

    x = np.array([src.getX() for src in srcs])
    y = np.array([src.getY() for src in srcs])

    from astrometry.libkd.spherematch import match
    from astrometry.util.plotutils import plothist, PlotSequence

    ps = PlotSequence(plotPrefix)

    # Match up various sources...
    R = 2.

    II, d = match(np.vstack((x, y)).T, np.vstack((rx0, ry0)).T, R)
    I = II[:, 0]
    J = II[:, 1]

    pa = dict(range=((-R, R), (-R, R)))

    plt.clf()
    plothist(x[I] - rx0[J], y[I] - ry0[J], 200, **pa)
    plt.title('Source positions - Reference positions (initial WCS)')
    plt.xlabel('delta-X (pixels)')
    plt.ylabel('delta-Y (pixels)')
    ps.savefig()

    II, d = match(np.vstack((x, y)).T, np.vstack((rx2, ry2)).T, R)
    I = II[:, 0]
    J = II[:, 1]

    plt.clf()
    plothist(x[I] - rx2[J], y[I] - ry2[J], 200, **pa)
    plt.title('Source positions - Reference positions (TAN WCS)')
    plt.xlabel('delta-X (pixels)')
    plt.ylabel('delta-Y (pixels)')
    ps.savefig()

    II, d = match(np.vstack((x, y)).T, np.vstack((rx3, ry3)).T, R)
    I = II[:, 0]
    J = II[:, 1]
    plt.clf()
    plothist(x[I] - rx3[J], y[I] - ry3[J], 200, **pa)
    plt.title('Source positions - Reference positions (SIP WCS #2)')
    plt.xlabel('delta-X (pixels)')
    plt.ylabel('delta-Y (pixels)')
    ps.savefig()

    II, d = match(np.vstack((rx0, ry0)).T, np.vstack((rx3, ry3)).T, R)
    I = II[:, 0]
    J = II[:, 1]
    plt.clf()
    plothist(rx0[I] - rx3[J], ry0[I] - ry3[J], 200, **pa)
    plt.title(
        'Reference positions (Original WCS) - Reference positions (SIP WCS #2)'
    )
    plt.xlabel('delta-X (pixels)')
    plt.ylabel('delta-Y (pixels)')
    ps.savefig()

    matches = solve.tanMatches
    msx, msy = [], []
    mrx, mry = [], []
    for m in matches:
        ref, src = m.first, m.second
        xy = tanwcs.skyToPixel(ref.getCoord())
        mrx.append(xy[0])
        mry.append(xy[1])
        msx.append(src.getX())
        msy.append(src.getY())

    plt.clf()
    plt.plot(x, y, 'r.')
    plt.plot(msx, msy, 'o', mec='r')
    plt.plot(rx0, ry0, 'g.')
    plt.plot(mrx, mry, 'gx')
    plt.title('TAN matches')
    ps.savefig()

    # Get SIP solution (4th order)

    solve = ast.determineWcs2(srcs, **imargs)
    wcs1 = solve.sipWcs

    matches = solve.sipMatches
    msx, msy = [], []
    mrx, mry = [], []
    for m in matches:
        ref, src = m.first, m.second
        xy = tanwcs.skyToPixel(ref.getCoord())
        mrx.append(xy[0])
        mry.append(xy[1])
        msx.append(src.getX())
        msy.append(src.getY())

    plt.clf()
    plt.plot(x, y, 'r.')
    plt.plot(msx, msy, 'o', mec='r')
    plt.plot(rx0, ry0, 'g.')
    plt.plot(mrx, mry, 'gx')
    plt.title('SIP matches')
    ps.savefig()

    rx1, ry1 = [], []
    for src in refs:
        xy = wcs1.skyToPixel(src.getCoord())
        rx1.append(xy[0])
        ry1.append(xy[1])
    rx1 = np.array(rx1)
    ry1 = np.array(ry1)

    plt.clf()
    plt.plot(x, y, 'o', mec='r', mfc='none')
    plt.plot(rx0, ry0, 'bx')
    plt.plot(rx1, ry1, 'g+')
    plt.plot(rx2, ry2, 'mx')
    plt.plot(rx3, ry3, 'r+')
    ps.savefig()

    plt.axis([x0, x0 + 500, y0, y0 + 500])
    ps.savefig()

    II, d = match(np.vstack((x, y)).T, np.vstack((rx1, ry1)).T, R)
    I = II[:, 0]
    J = II[:, 1]

    plt.clf()
    plothist(x[I] - rx1[J], y[I] - ry1[J], 200, **pa)
    plt.title('Source positions - Reference positions (SIP WCS)')
    plt.xlabel('delta-X (pixels)')
    plt.ylabel('delta-Y (pixels)')
    ps.savefig()
Exemple #3
0
    print s
    print s.getProfile()

    s.sersicindex.setValue(4.0)
    print s.getProfile()

    d = DevGalaxy(s.pos, s.brightness, s.shape)
    print d
    print d.getProfile()
    
    # Extrapolation!
    # s.sersicindex.setValue(0.5)
    # print s.getProfile()
    

    ps = PlotSequence('ser')
    
    # example PSF (from WISE W1 fit)
    w = np.array([ 0.77953706,  0.16022146,  0.06024237])
    mu = np.array([[-0.01826623, -0.01823262],
                   [-0.21878855, -0.0432496 ],
                   [-0.83365747, -0.13039277]])
    sigma = np.array([[[  7.72925584e-01,   5.23305564e-02],
                       [  5.23305564e-02,   8.89078473e-01]],
                       [[  9.84585869e+00,   7.79378820e-01],
                       [  7.79378820e-01,   8.84764455e+00]],
                       [[  2.02664489e+02,  -8.16667434e-01],
                        [ -8.16667434e-01,   1.87881670e+02]]])
    
    psf = GaussianMixturePSF(w, mu, sigma)
    
def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--name1', help='Name for first data set')
    parser.add_argument('--name2', help='Name for second data set')
    parser.add_argument('--plot-prefix',
                        default='compare',
                        help='Prefix for plot filenames; default "%default"')
    parser.add_argument('--match',
                        default=1.0,
                        help='Astrometric cross-match distance in arcsec')
    parser.add_argument('dir1', help='First directory to compare')
    parser.add_argument('dir2', help='Second directory to compare')

    opt = parser.parse_args()

    ps = PlotSequence(opt.plot_prefix)

    name1 = opt.name1
    if name1 is None:
        name1 = os.path.basename(opt.dir1)
        if not len(name1):
            name1 = os.path.basename(os.path.dirname(opt.dir1))
    name2 = opt.name2
    if name2 is None:
        name2 = os.path.basename(opt.dir2)
        if not len(name2):
            name2 = os.path.basename(os.path.dirname(opt.dir2))
    tt = 'Comparing %s to %s' % (name1, name2)

    # regex for tractor-*.fits catalog filename
    catre = re.compile('tractor-.*.fits')

    cat1, cat2 = [], []
    for basedir, cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
        for dirpath, dirnames, filenames in os.walk(basedir, followlinks=True):
            for fn in filenames:
                if not catre.match(fn):
                    print('Skipping', fn, 'due to filename')
                    continue
                fn = os.path.join(dirpath, fn)
                t = fits_table(fn)
                print(len(t), 'from', fn)
                cat.append(t)
    cat1 = merge_tables(cat1, columns='fillzero')
    cat2 = merge_tables(cat2, columns='fillzero')
    print('Total of', len(cat1), 'from', name1)
    print('Total of', len(cat2), 'from', name2)
    cat1.cut(cat1.brick_primary)
    cat2.cut(cat2.brick_primary)
    print('Total of', len(cat1), 'BRICK_PRIMARY from', name1)
    print('Total of', len(cat2), 'BRICK_PRIMARY from', name2)

    cat1.cut((cat1.decam_anymask[:, 1] == 0) *
             (cat1.decam_anymask[:, 2] == 0) * (cat1.decam_anymask[:, 4] == 0))
    cat2.cut((cat2.decam_anymask[:, 1] == 0) *
             (cat2.decam_anymask[:, 2] == 0) * (cat2.decam_anymask[:, 4] == 0))
    print('Total of', len(cat1), 'unmasked from', name1)
    print('Total of', len(cat2), 'unmasked from', name2)

    I, J, d = match_radec(cat1.ra,
                          cat1.dec,
                          cat2.ra,
                          cat2.dec,
                          opt.match / 3600.,
                          nearest=True)
    print(len(I), 'matched')

    plt.clf()
    plt.hist(d * 3600., 100)
    plt.xlabel('Match distance (arcsec)')
    plt.title(tt)
    ps.savefig()

    matched1 = cat1[I]
    matched2 = cat2[J]

    for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
        K = np.flatnonzero((matched1.decam_flux_ivar[:, iband] > 0) *
                           (matched2.decam_flux_ivar[:, iband] > 0))

        print('Median mw_trans', band, 'is',
              np.median(matched1.decam_mw_transmission[:, iband]))

        plt.clf()
        plt.errorbar(
            matched1.decam_flux[K, iband],
            matched2.decam_flux[K, iband],
            fmt='.',
            color=cc,
            xerr=1. / np.sqrt(matched1.decam_flux_ivar[K, iband]),
            yerr=1. / np.sqrt(matched2.decam_flux_ivar[K, iband]),
            alpha=0.1,
        )
        plt.xlabel('%s flux: %s' % (name1, band))
        plt.ylabel('%s flux: %s' % (name2, band))
        plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
        plt.axis([-100, 1000, -100, 1000])
        plt.title(tt)
        ps.savefig()

    for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
        good = ((matched1.decam_flux_ivar[:, iband] > 0) *
                (matched2.decam_flux_ivar[:, iband] > 0))
        K = np.flatnonzero(good)
        psf1 = (matched1.type == 'PSF ')
        psf2 = (matched2.type == 'PSF ')
        P = np.flatnonzero(good * psf1 * psf2)

        mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
            matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])

        iv1 = matched1.decam_flux_ivar[:, iband]
        iv2 = matched2.decam_flux_ivar[:, iband]
        std = np.sqrt(1. / iv1 + 1. / iv2)

        plt.clf()
        plt.plot(
            mag1[K],
            (matched2.decam_flux[K, iband] - matched1.decam_flux[K, iband]) /
            std[K],
            '.',
            alpha=0.1,
            color=cc)
        plt.plot(
            mag1[P],
            (matched2.decam_flux[P, iband] - matched1.decam_flux[P, iband]) /
            std[P],
            '.',
            alpha=0.1,
            color='k')
        plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' %
                   (name2, name1, band))
        plt.xlabel('%s mag: %s' % (name1, band))
        plt.axhline(0, color='k', alpha=0.5)
        plt.axis([24, 16, -10, 10])
        plt.title(tt)
        ps.savefig()

    plt.clf()
    lp, lt = [], []
    for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
        good = ((matched1.decam_flux_ivar[:, iband] > 0) *
                (matched2.decam_flux_ivar[:, iband] > 0))
        #good = True
        psf1 = (matched1.type == 'PSF ')
        psf2 = (matched2.type == 'PSF ')
        mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
            matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
        iv1 = matched1.decam_flux_ivar[:, iband]
        iv2 = matched2.decam_flux_ivar[:, iband]
        std = np.sqrt(1. / iv1 + 1. / iv2)
        #std = np.hypot(std, 0.01)
        G = np.flatnonzero(good * psf1 * psf2 * np.isfinite(mag1) *
                           (mag1 >= 20) *
                           (mag1 < dict(g=24, r=23.5, z=22.5)[band]))

        n, b, p = plt.hist(
            (matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) /
            std[G],
            range=(-4, 4),
            bins=50,
            histtype='step',
            color=cc,
            normed=True)

        sig = (matched2.decam_flux[G, iband] -
               matched1.decam_flux[G, iband]) / std[G]
        print('Raw mean and std of points:', np.mean(sig), np.std(sig))
        med = np.median(sig)
        rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.
        print('Median and percentile-based sigma:', med, rsigma)
        lp.append(p[0])
        lt.append('%s: %.2f +- %.2f' % (band, med, rsigma))

    bins = []
    gaussint = []
    for blo, bhi in zip(b, b[1:]):
        c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
        c /= (bhi - blo)
        #bins.extend([blo,bhi])
        #gaussint.extend([c,c])
        bins.append((blo + bhi) / 2.)
        gaussint.append(c)
    plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)

    plt.title(tt)
    plt.xlabel('Flux difference / error (sigma)')
    plt.axvline(0, color='k', alpha=0.1)
    plt.ylim(0, 0.45)
    plt.legend(lp, lt, loc='upper right')
    ps.savefig()

    for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
        plt.clf()
        mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
            matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
        mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
            matched2.decam_flux[:, iband], matched2.decam_flux_ivar[:, iband])

        meanmag = NanoMaggies.nanomaggiesToMag(
            (matched1.decam_flux[:, iband] + matched2.decam_flux[:, iband]) /
            2.)

        psf1 = (matched1.type == 'PSF ')
        psf2 = (matched2.type == 'PSF ')
        good = ((matched1.decam_flux_ivar[:, iband] > 0) *
                (matched2.decam_flux_ivar[:, iband] > 0) * np.isfinite(mag1) *
                np.isfinite(mag2))
        K = np.flatnonzero(good)
        P = np.flatnonzero(good * psf1 * psf2)

        plt.errorbar(mag1[K],
                     mag2[K],
                     fmt='.',
                     color=cc,
                     xerr=magerr1[K],
                     yerr=magerr2[K],
                     alpha=0.1)
        plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5)
        plt.xlabel('%s %s (mag)' % (name1, band))
        plt.ylabel('%s %s (mag)' % (name2, band))
        plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
        plt.axis([24, 16, 24, 16])
        plt.title(tt)
        ps.savefig()

        plt.clf()
        plt.errorbar(mag1[K],
                     mag2[K] - mag1[K],
                     fmt='.',
                     color=cc,
                     xerr=magerr1[K],
                     yerr=magerr2[K],
                     alpha=0.1)
        plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5)
        plt.xlabel('%s %s (mag)' % (name1, band))
        plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band))
        plt.axhline(0., color='k', alpha=1.)
        plt.axis([24, 16, -1, 1])
        plt.title(tt)
        ps.savefig()

        magbins = np.arange(16, 24.001, 0.5)

        plt.clf()
        plt.plot(mag1[K],
                 (mag2[K] - mag1[K]) / np.hypot(magerr1[K], magerr2[K]),
                 '.',
                 color=cc,
                 alpha=0.1)
        plt.plot(mag1[P],
                 (mag2[P] - mag1[P]) / np.hypot(magerr1[P], magerr2[P]),
                 'k.',
                 alpha=0.5)

        plt.xlabel('%s %s (mag)' % (name1, band))
        plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
                   (name2, band, name1, band))
        plt.axhline(0., color='k', alpha=1.)
        plt.axis([24, 16, -10, 10])
        plt.title(tt)
        ps.savefig()

        y = (mag2 - mag1) / np.hypot(magerr1, magerr2)

        plt.clf()
        plt.plot(meanmag[P], y[P], 'k.', alpha=0.1)

        midmag = []
        vals = np.zeros((len(magbins) - 1, 5))
        median_err1 = []

        iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)

        # FIXME -- should we do some stats after taking off the mean difference?

        for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
            I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
            midmag.append((mlo + mhi) / 2.)
            median_err1.append(np.median(magerr1[I]))
            if len(I) == 0:
                continue
            # median and +- 1 sigma quantiles
            ybin = y[I]
            vals[bini, 0] = np.percentile(ybin, 16)
            vals[bini, 1] = np.median(ybin)
            vals[bini, 2] = np.percentile(ybin, 84)
            # +- 2 sigma quantiles
            vals[bini, 3] = np.percentile(ybin, 2.3)
            vals[bini, 4] = np.percentile(ybin, 97.7)

            iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)

            print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss,
                  'vs expected for Gaussian;', len(ybin), 'points')

            # if iqd > iqd_gauss:
            #     # What error adding in quadrature would you need to make the IQD match?
            #     err = median_err1[-1]
            #     target_err = err * (iqd / iqd_gauss)
            #     sys_err = np.sqrt(target_err**2 - err**2)
            #     print('--> add systematic error', sys_err)

        # ~ Johan's cuts
        mlo = 21.
        mhi = dict(g=24., r=23.5, z=22.5)[band]
        I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
        ybin = y[I]
        iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
        print('Mag bin', mlo, mhi, 'band', band,
              ': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;',
              len(ybin), 'points')
        if iqd > iqd_gauss:
            # What error adding in quadrature would you need to make
            # the IQD match?
            err = np.median(np.hypot(magerr1[I], magerr2[I]))
            print('Median error (hypot):', err)
            target_err = err * (iqd / iqd_gauss)
            print('Target:', target_err)
            sys_err = np.sqrt((target_err**2 - err**2) / 2.)
            print('--> add systematic error', sys_err)

            # check...
            err_sys = np.hypot(np.hypot(magerr1, sys_err),
                               np.hypot(magerr2, sys_err))
            ysys = (mag2 - mag1) / err_sys
            ysys = ysys[I]
            print('Resulting median error:', np.median(err_sys[I]))
            iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
            print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian')
            # Hmmm, this doesn't work... totally overshoots.

        plt.errorbar(midmag,
                     vals[:, 1],
                     fmt='o',
                     color='b',
                     yerr=(vals[:, 1] - vals[:, 0], vals[:, 2] - vals[:, 1]),
                     capthick=3,
                     zorder=20)
        plt.errorbar(midmag,
                     vals[:, 1],
                     fmt='o',
                     color='b',
                     yerr=(vals[:, 1] - vals[:, 3], vals[:, 4] - vals[:, 1]),
                     capthick=2,
                     zorder=20)
        plt.axhline(1., color='b', alpha=0.2)
        plt.axhline(-1., color='b', alpha=0.2)
        plt.axhline(2., color='b', alpha=0.2)
        plt.axhline(-2., color='b', alpha=0.2)

        for mag, err, y in zip(midmag, median_err1, vals[:, 3]):
            if not np.isfinite(err):
                continue
            if y < -6:
                continue
            plt.text(mag,
                     y - 0.1,
                     '%.3f' % err,
                     va='top',
                     ha='center',
                     color='k',
                     fontsize=10)

        plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band))
        plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
                   (name2, band, name1, band))
        plt.axhline(0., color='k', alpha=1.)

        plt.axvline(21, color='k', alpha=0.3)
        plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3)

        plt.axis([24.1, 16, -6, 6])
        plt.title(tt)
        ps.savefig()

        #magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
        if band == 'g':
            magbins = [20, 24]
        elif band == 'r':
            magbins = [20, 23.5]
        elif band == 'z':
            magbins = [20, 22.5]

        slo, shi = -5, 5
        plt.clf()
        ha = dict(bins=25, range=(slo, shi), histtype='step', normed=True)
        y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
        midmag = []
        nn = []
        rgbs = []
        lt, lp = [], []
        for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
            I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
            if len(I) == 0:
                continue
            ybin = y[I]
            rgb = [0., 0., 0.]
            rgb[0] = float(bini) / (len(magbins) - 1)
            rgb[2] = 1. - rgb[0]
            n, b, p = plt.hist(ybin, color=rgb, **ha)
            lt.append('mag %g to %g' % (mlo, mhi))
            lp.append(p[0])
            midmag.append((mlo + mhi) / 2.)
            nn.append(n)
            rgbs.append(rgb)

        bins = []
        gaussint = []
        for blo, bhi in zip(b, b[1:]):
            #midbin.append((blo+bhi)/2.)
            #gaussint.append(scipy.stats.norm.cdf(bhi) -
            #                scipy.stats.norm.cdf(blo))
            c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
            c /= (bhi - blo)
            bins.extend([blo, bhi])
            gaussint.extend([c, c])
        plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)

        plt.legend(lp, lt)
        plt.title(tt)
        plt.xlim(slo, shi)
        ps.savefig()

        bincenters = b[:-1] + (b[1] - b[0]) / 2.
        plt.clf()
        lp = []
        for n, rgb, mlo, mhi in zip(nn, rgbs, magbins, magbins[1:]):
            p = plt.plot(bincenters, n, '-', color=rgb)
            lp.append(p[0])
        plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2)
        plt.legend(lp, lt)
        plt.title(tt)
        plt.xlim(slo, shi)
        ps.savefig()
Exemple #5
0
def stage0(**kwargs):
    ps = PlotSequence('cfht')

    decals = CfhtDecals()
    B = decals.get_bricks()
    print('Bricks:')
    B.about()

    ra, dec = 190.0, 11.0

    #bands = 'ugri'
    bands = 'gri'

    B.cut(np.argsort(degrees_between(ra, dec, B.ra, B.dec)))
    print('Nearest bricks:', B.ra[:5], B.dec[:5], B.brickid[:5])

    brick = B[0]
    pixscale = 0.186
    #W,H = 1024,1024
    #W,H = 2048,2048
    #W,H = 3600,3600
    W, H = 4800, 4800

    targetwcs = wcs_for_brick(brick, pixscale=pixscale, W=W, H=H)
    ccdfn = 'cfht-ccds.fits'
    if os.path.exists(ccdfn):
        T = fits_table(ccdfn)
    else:
        T = get_ccd_list()
        T.writeto(ccdfn)
    print(len(T), 'CCDs')
    T.cut(ccds_touching_wcs(targetwcs, T))
    print(len(T), 'CCDs touching brick')

    T.cut(np.array([b in bands for b in T.filter]))
    print(len(T), 'in bands', bands)

    ims = []
    for t in T:
        im = CfhtImage(t)
        # magzp = hdr['PHOT_C'] + 2.5 * np.log10(hdr['EXPTIME'])
        # fwhm = t.seeing / (pixscale * 3600)
        # print '-> FWHM', fwhm, 'pix'
        im.seeing = t.seeing
        im.pixscale = t.pixscale
        print('seeing', t.seeing)
        print('pixscale', im.pixscale * 3600, 'arcsec/pix')
        im.run_calibs(t.ra, t.dec, im.pixscale, W=t.width, H=t.height)
        ims.append(im)

    # Read images, clip to ROI
    targetrd = np.array([
        targetwcs.pixelxy2radec(x, y)
        for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
    ])
    keepims = []
    tims = []
    for im in ims:
        print()
        print('Reading expnum', im.expnum, 'name', im.extname, 'band', im.band,
              'exptime', im.exptime)
        band = im.band
        wcs = im.read_wcs()
        imh, imw = wcs.imageh, wcs.imagew
        imgpoly = [(1, 1), (1, imh), (imw, imh), (imw, 1)]
        ok, tx, ty = wcs.radec2pixelxy(targetrd[:-1, 0], targetrd[:-1, 1])
        tpoly = zip(tx, ty)
        clip = clip_polygon(imgpoly, tpoly)
        clip = np.array(clip)
        #print 'Clip', clip
        if len(clip) == 0:
            continue
        x0, y0 = np.floor(clip.min(axis=0)).astype(int)
        x1, y1 = np.ceil(clip.max(axis=0)).astype(int)
        slc = slice(y0, y1 + 1), slice(x0, x1 + 1)

        ## FIXME -- it seems I got lucky and the cross product is
        ## negative == clockwise, as required by clip_polygon. One
        ## could check this and reverse the polygon vertex order.
        # dx0,dy0 = tx[1]-tx[0], ty[1]-ty[0]
        # dx1,dy1 = tx[2]-tx[1], ty[2]-ty[1]
        # cross = dx0*dy1 - dx1*dy0
        # print 'Cross:', cross

        print('Image slice: x [%i,%i], y [%i,%i]' % (x0, x1, y0, y1))
        print('Reading image from', im.imgfn, 'HDU', im.hdu)
        img, imghdr = im.read_image(header=True, slice=slc)
        goodpix = (img != 0)
        print('Number of pixels == 0:', np.sum(img == 0))
        print('Number of pixels != 0:', np.sum(goodpix))
        if np.sum(goodpix) == 0:
            continue
        # print 'Image shape', img.shape
        print('Image range', img.min(), img.max())
        print('Goodpix image range:', (img[goodpix]).min(),
              (img[goodpix]).max())
        if img[goodpix].min() == img[goodpix].max():
            print('No dynamic range in image')
            continue
        # print 'Reading invvar from', im.wtfn, 'HDU', im.hdu
        # invvar = im.read_invvar(slice=slc)
        # # print 'Invvar shape', invvar.shape
        # # print 'Invvar range:', invvar.min(), invvar.max()
        # invvar[goodpix == 0] = 0.
        # if np.all(invvar == 0.):
        #     print 'Skipping zero-invvar image'
        #     continue
        # assert(np.all(np.isfinite(img)))
        # assert(np.all(np.isfinite(invvar)))
        # assert(not(np.all(invvar == 0.)))
        # # Estimate per-pixel noise via Blanton's 5-pixel MAD
        # slice1 = (slice(0,-5,10),slice(0,-5,10))
        # slice2 = (slice(5,None,10),slice(5,None,10))
        # # print 'sliced shapes:', img[slice1].shape, img[slice2].shape
        # # print 'good shape:', (goodpix[slice1] * goodpix[slice2]).shape
        # # print 'good values:', np.unique(goodpix[slice1] * goodpix[slice2])
        # # print 'sliced[good] shapes:', (img[slice1] -  img[slice2])[goodpix[slice1] * goodpix[slice2]].shape
        # mad = np.median(np.abs(img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].ravel())
        # sig1 = 1.4826 * mad / np.sqrt(2.)
        # print 'MAD sig1:', sig1
        # # invvar was 1 or 0
        # invvar *= (1./(sig1**2))
        # medsky = np.median(img[goodpix])

        # Read full image for sig1 and sky estimate
        fullimg = im.read_image()
        fullgood = (fullimg != 0)
        # Estimate per-pixel noise via Blanton's 5-pixel MAD
        slice1 = (slice(0, -5, 10), slice(0, -5, 10))
        slice2 = (slice(5, None, 10), slice(5, None, 10))
        mad = np.median(
            np.abs(fullimg[slice1] -
                   fullimg[slice2])[fullgood[slice1] *
                                    fullgood[slice2]].ravel())
        sig1 = 1.4826 * mad / np.sqrt(2.)
        print('MAD sig1:', sig1)
        medsky = np.median(fullimg[fullgood])
        invvar = np.zeros_like(img)
        invvar[goodpix] = 1. / sig1**2

        # Median-smooth sky subtraction
        plt.clf()
        dimshow(np.round((img - medsky) / sig1), vmin=-3, vmax=5)
        plt.title('Scalar median: %s' % im.name)
        ps.savefig()

        # medsky = np.zeros_like(img)
        # # astrometry.util.util
        # median_smooth(img, np.logical_not(goodpix), 256, medsky)
        fullmed = np.zeros_like(fullimg)
        median_smooth(fullimg - medsky, np.logical_not(fullgood), 256, fullmed)
        fullmed += medsky
        medimg = fullmed[slc]

        plt.clf()
        dimshow(np.round((img - medimg) / sig1), vmin=-3, vmax=5)
        plt.title('Median filtered: %s' % im.name)
        ps.savefig()

        #print 'Subtracting median:', medsky
        #img -= medsky
        img -= medimg

        primhdr = im.read_image_primary_header()

        magzp = decals.get_zeropoint_for(im)
        print('magzp', magzp)
        zpscale = NanoMaggies.zeropointToScale(magzp)
        print('zpscale', zpscale)

        # Scale images to Nanomaggies
        img /= zpscale
        sig1 /= zpscale
        invvar *= zpscale**2
        orig_zpscale = zpscale

        zpscale = 1.
        assert (np.sum(invvar > 0) > 0)
        print('After scaling:')
        print('sig1', sig1)
        print('invvar range', invvar.min(), invvar.max())
        print('image range', img.min(), img.max())

        assert (np.all(np.isfinite(img)))
        assert (np.all(np.isfinite(invvar)))
        assert (np.isfinite(sig1))

        plt.clf()
        lo, hi = -5 * sig1, 10 * sig1
        n, b, p = plt.hist(img[goodpix].ravel(),
                           100,
                           range=(lo, hi),
                           histtype='step',
                           color='k')
        xx = np.linspace(lo, hi, 200)
        plt.plot(xx, max(n) * np.exp(-xx**2 / (2. * sig1**2)), 'r-')
        plt.xlim(lo, hi)
        plt.title('Pixel histogram: %s' % im.name)
        ps.savefig()

        twcs = ConstantFitsWcs(wcs)
        if x0 or y0:
            twcs.setX0Y0(x0, y0)

        info = im.get_image_info()
        fullh, fullw = info['dims']

        # read fit PsfEx model
        psfex = PsfEx.fromFits(im.psffitfn)
        print('Read', psfex)

        # HACK -- highly approximate PSF here!
        #psf_fwhm = imghdr['FWHM']
        #psf_fwhm = im.seeing

        psf_fwhm = im.seeing / (im.pixscale * 3600)
        print('PSF FWHM', psf_fwhm, 'pixels')
        psf_sigma = psf_fwhm / 2.35
        psf = NCircularGaussianPSF([psf_sigma], [1.])

        print('img type', img.dtype)

        tim = Image(img,
                    invvar=invvar,
                    wcs=twcs,
                    psf=psf,
                    photocal=LinearPhotoCal(zpscale, band=band),
                    sky=ConstantSky(0.),
                    name=im.name + ' ' + band)
        tim.zr = [-3. * sig1, 10. * sig1]
        tim.sig1 = sig1
        tim.band = band
        tim.psf_fwhm = psf_fwhm
        tim.psf_sigma = psf_sigma
        tim.sip_wcs = wcs
        tim.x0, tim.y0 = int(x0), int(y0)
        tim.psfex = psfex
        tim.imobj = im
        mn, mx = tim.zr
        tim.ima = dict(interpolation='nearest',
                       origin='lower',
                       cmap='gray',
                       vmin=mn,
                       vmax=mx)
        tims.append(tim)
        keepims.append(im)

    ims = keepims

    print('Computing resampling...')
    # save resampling params
    for tim in tims:
        wcs = tim.sip_wcs
        subh, subw = tim.shape
        subwcs = wcs.get_subimage(tim.x0, tim.y0, subw, subh)
        tim.subwcs = subwcs
        try:
            Yo, Xo, Yi, Xi, rims = resample_with_wcs(targetwcs, subwcs, [], 2)
        except OverlapError:
            print('No overlap')
            continue
        if len(Yo) == 0:
            continue
        tim.resamp = (Yo, Xo, Yi, Xi)

    print('Creating coadds...')
    # Produce per-band coadds, for plots
    coimgs = []
    cons = []
    for ib, band in enumerate(bands):
        coimg = np.zeros((H, W), np.float32)
        con = np.zeros((H, W), np.uint8)
        for tim in tims:
            if tim.band != band:
                continue
            (Yo, Xo, Yi, Xi) = tim.resamp
            if len(Yo) == 0:
                continue
            nn = (tim.getInvvar()[Yi, Xi] > 0)
            coimg[Yo, Xo] += tim.getImage()[Yi, Xi] * nn
            con[Yo, Xo] += nn

            # print
            # print 'tim', tim.name
            # print 'number of resampled pix:', len(Yo)
            # reim = np.zeros_like(coimg)
            # ren  = np.zeros_like(coimg)
            # reim[Yo,Xo] = tim.getImage()[Yi,Xi] * nn
            # ren[Yo,Xo] = nn
            # print 'number of resampled pix with positive invvar:', ren.sum()
            # plt.clf()
            # plt.subplot(2,2,1)
            # mn,mx = [np.percentile(reim[ren>0], p) for p in [25,95]]
            # print 'Percentiles:', mn,mx
            # dimshow(reim, vmin=mn, vmax=mx)
            # plt.colorbar()
            # plt.subplot(2,2,2)
            # dimshow(con)
            # plt.colorbar()
            # plt.subplot(2,2,3)
            # dimshow(reim, vmin=tim.zr[0], vmax=tim.zr[1])
            # plt.colorbar()
            # plt.subplot(2,2,4)
            # plt.hist(reim.ravel(), 100, histtype='step', color='b')
            # plt.hist(tim.getImage().ravel(), 100, histtype='step', color='r')
            # plt.suptitle('%s: %s' % (band, tim.name))
            # ps.savefig()

        coimg /= np.maximum(con, 1)
        coimgs.append(coimg)
        cons.append(con)

    plt.clf()
    dimshow(get_rgb(coimgs, bands))
    ps.savefig()

    plt.clf()
    for i, b in enumerate(bands):
        plt.subplot(2, 2, i + 1)
        dimshow(cons[i], ticks=False)
        plt.title('%s band' % b)
        plt.colorbar()
    plt.suptitle('Number of exposures')
    ps.savefig()

    print('Grabbing SDSS sources...')
    bandlist = [b for b in bands]
    cat, T = get_sdss_sources(bandlist, targetwcs)
    # record coordinates in target brick image
    ok, T.tx, T.ty = targetwcs.radec2pixelxy(T.ra, T.dec)
    T.tx -= 1
    T.ty -= 1
    T.itx = np.clip(np.round(T.tx).astype(int), 0, W - 1)
    T.ity = np.clip(np.round(T.ty).astype(int), 0, H - 1)

    plt.clf()
    dimshow(get_rgb(coimgs, bands))
    ax = plt.axis()
    plt.plot(T.tx, T.ty, 'o', mec=green, mfc='none', ms=10, mew=1.5)
    plt.axis(ax)
    plt.title('SDSS sources')
    ps.savefig()

    print('Detmaps...')
    # Render the detection maps
    detmaps = dict([(b, np.zeros((H, W), np.float32)) for b in bands])
    detivs = dict([(b, np.zeros((H, W), np.float32)) for b in bands])
    for tim in tims:
        iv = tim.getInvvar()
        psfnorm = 1. / (2. * np.sqrt(np.pi) * tim.psf_sigma)
        detim = tim.getImage().copy()
        detim[iv == 0] = 0.
        detim = gaussian_filter(detim, tim.psf_sigma) / psfnorm**2
        detsig1 = tim.sig1 / psfnorm
        subh, subw = tim.shape
        detiv = np.zeros((subh, subw), np.float32) + (1. / detsig1**2)
        detiv[iv == 0] = 0.
        (Yo, Xo, Yi, Xi) = tim.resamp
        detmaps[tim.band][Yo, Xo] += detiv[Yi, Xi] * detim[Yi, Xi]
        detivs[tim.band][Yo, Xo] += detiv[Yi, Xi]

    rtn = dict()
    for k in [
            'T', 'coimgs', 'cons', 'detmaps', 'detivs', 'targetrd', 'pixscale',
            'targetwcs', 'W', 'H', 'bands', 'tims', 'ps', 'brick', 'cat'
    ]:
        rtn[k] = locals()[k]
    return rtn
Exemple #6
0
    psfvar = np.zeros((3, 2, 2))
    psfvar[0, 0, 0] = 1.2**2
    psfvar[0, 1, 1] = psfvar[0, 0, 0]
    psfvar[1, 0, 0] = 2.4**2
    psfvar[1, 1, 1] = psfvar[1, 0, 0]
    psfvar[2, 0, 0] = 3.6**2
    psfvar[2, 1, 1] = psfvar[2, 0, 0]
    psf = MixtureOfGaussians(psfamp, psfmean, psfvar)
    functional_test_patch_maker('test_patch.png')
    functional_test_patch_maker('test_psf_patch.png', psf=psf)

    # functional test: c_gauss_2d_approx
    from tractor.mix import c_gauss_2d_approx, c_gauss_2d_grid, c_gauss_2d_approx2, c_gauss_2d_approx3

    from astrometry.util.plotutils import PlotSequence
    ps = PlotSequence('approx')

    for j in range(100):
        print()
        print('j =', j)
        print()

        x0, x1 = -50, 50
        y0, y1 = -51, 51
        W = x1 - x0
        H = y1 - y0
        result = np.zeros((H, W))

        amp = np.array([1.0])
        mean = np.array([[0.3, 0.7], ])
        minval = 1e-3
Exemple #7
0
import numpy as np
import fitsio
from astrometry.util.fits import fits_table
from astrometry.util.plotutils import PlotSequence
from astrometry.util.multiproc import multiproc
from legacypipe.survey import *
from legacypipe.runs import get_survey

import photutils

import sys
import os

if __name__ == '__main__':

    ps = PlotSequence('psfnorm')

    survey = get_survey('dr4v2')
    ccds = survey.get_ccds_readonly()
    print(len(ccds), 'CCDs')
    ccds = ccds[ccds.camera == '90prime']
    print(len(ccds), 'Bok CCDs')

    ## Skip a few missing ones...
    ccds = ccds[24:]

    for i, ccd in enumerate(ccds):
        print('Reading CCD', i)
        try:
            im = survey.get_image_object(ccd, makeNewWeightMap=False)
            print('Reading', im)
Exemple #8
0
def process_image(fn, ext, nom, sfd, opt, obs, tiles):
    db = opt.db
    print('Reading', fn)

    if sfd is None:
        sfd = gSFD

    # Read primary FITS header
    phdr = fitsio.read_header(fn)

    obstype = phdr.get('OBSTYPE','').strip()
    print('obstype:', obstype)
    exptime = phdr.get('EXPTIME', 0)
    expnum = phdr.get('EXPNUM', 0)

    filt = phdr.get('FILTER', None)
    if filt is not None:
        filt = filt.strip()
        filt = filt.split()[0]
    if filt is None:
        filt = ''

    airmass = phdr.get('AIRMASS', 0.)
    ra  = hmsstring2ra (phdr.get('RA', '0'))
    dec = dmsstring2dec(phdr.get('DEC', '0'))
    
    # Write QA plots to files named by the exposure number
    print('Exposure number:', expnum)

    skip = False
    if obstype in ['zero', 'focus', 'dome flat', '']:
        print('Skipping obstype =', obstype)
        skip = True
    if exptime == 0:
        print('Exposure time EXPTIME in header =', exptime)
        skip = True
    if expnum == '':
        print('No expnum in header')
        skip = True
    if filt == 'solid':
        print('Solid (block) filter.')
        skip = True

    if skip and not db:
        return None

    if db:
        import obsdb
        if ext is None:
            ext = get_default_extension(fn)
        m,created = obsdb.MeasuredCCD.objects.get_or_create(
            filename=fn, extension=ext)
        m.obstype = obstype
        m.camera  = camera_name(phdr)
        m.expnum  = expnum
        m.exptime = exptime
        m.mjd_obs = phdr.get('MJD-OBS', 0.)
        m.airmass = airmass
        m.rabore  = ra
        m.decbore = dec
        m.band = filt
        m.bad_pixcnt = ('PIXCNT1' in phdr)
        m.readtime = phdr.get('READTIME', 0.)

    if opt.focus and obstype == 'focus' and m.camera == 'mosaic3':
        from mosaic_focus import Mosaic3FocusMeas
        show_plot = opt.show
        if show_plot:
            import pylab as plt
            plt.figure(2, figsize=(8,10))
        if ext is None:
            ext = get_default_extension(fn)
        meas = Mosaic3FocusMeas(fn, ext, nom)
        focusfn = 'focus.png'
        meas.run(ps=None, plotfn=focusfn)
        print('Wrote', focusfn)
        if show_plot:
            plt.draw()
            plt.show(block=False)
            plt.pause(0.001)
            plt.figure(1)
        
    if skip:
        m.save()
        return None
        
    if opt.doplots:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence('qa-%i' % expnum)
        ps.printfn = False
    else:
        ps = None

    # Measure the new image
    kwa = {}
    if ext is not None:
        kwa.update(ext=ext)
    if opt.n_fwhm is not None:
        kwa.update(n_fwhm=opt.n_fwhm)
    M = measure_raw(fn, ps=ps, **kwa)

    if opt.doplots:
        from glob import glob
        # Gather all the QAplots into a single pdf and clean them up.
        qafile = 'qa-%i.pdf' % expnum
        pnglist = sorted(glob('qa-%i-??.png' % expnum))
        cmd = 'convert {} {}'.format(' '.join(pnglist), qafile)
        print('Writing out {}'.format(qafile))
        #print(cmd)
        os.system(cmd)
        if not opt.keep_plots:
            [os.remove(png) for png in pnglist]

    # (example results for testig)
    #M = {'seeing': 1.4890481099577366, 'airmass': 1.34,
    #'skybright': 18.383479116033314, 'transparency': 0.94488537276869045,
    #'band': 'z', 'zp': 26.442847814941093}

    #print('Measurements:', M)

    trans = M.get('transparency', 0)
    band = M['band']
    # Look up E(B-V) in SFD map
    ebv = sfd.ebv(ra, dec)[0]
    print('E(B-V): %.3f' % ebv)

    if trans > 0:

        fid = nom.fiducial_exptime(band)

        expfactor = exposure_factor(fid, nom,
                                    airmass, ebv, M['seeing'], M['skybright'],
                                    trans)
        print('Exposure factor:              %6.3f' % expfactor)
        t_exptime = expfactor * fid.exptime
        print('Target exposure time:         %6.1f' % t_exptime)
        t_exptime = np.clip(t_exptime, fid.exptime_min, fid.exptime_max)
        print('Clipped exposure time:        %6.1f' % t_exptime)
    
        if band == 'z':
            t_sat = nom.saturation_time(band, M['skybright'])
            if t_exptime > t_sat:
                t_exptime = t_sat
                print('Reduced exposure time to avoid z-band saturation: %.1f' % t_exptime)

        print

        print('Actual exposure time taken:   %6.1f' % exptime)
    
        print('Depth (exposure time) factor: %6.3f' % (exptime / t_exptime))
        
        # If you were going to re-plan, you would run with these args:
        plandict = dict(seeing=M['seeing'], transparency=trans)
        # Assume the sky is as much brighter than canonical in each band... unlikely
        dsky = M['skybright'] - nom.sky(M['band'])
        for b in 'grz':
            plandict['sb'+b] = nom.sky(b) + dsky
        # Note that nightlystrategy.py takes UTC dates.
        start = datenow()
        # Start the strategy 5 minutes from now.
        start += datetime.timedelta(0, 5*60)
        d = start.date()
        plandict['startdate'] = '%04i-%02i-%02i' % (d.year, d.month, d.day)
        t = start.time()
        plandict['starttime'] = t.strftime('%H:%M:%S')
        # Make an hour-long plan
        end = start + datetime.timedelta(0, 3600)
        d = end.date()
        plandict['enddate'] = '%04i-%02i-%02i' % (d.year, d.month, d.day)
        t = end.time()
        plandict['endtime'] = t.strftime('%H:%M:%S')
    
        # Set "--date" to be the UTC date at previous sunset.
        # (nightlystrategy will ask for the next setting of the sun below
        # -18-degrees from that date to define the sn_18).  We could
        # probably also get away with subtracting, like, 12 hours from
        # now()...
        sun = ephem.Sun()
        obs.date = datenow()
        # not the proper horizon, but this doesn't matter -- just need it to
        # be before -18-degree twilight.
        obs.horizon = 0.
        sunset = obs.previous_setting(sun)
        # pyephem's Date.tuple() splits a date into y,m,d,h,m,s
        d = sunset.tuple()
        #print('Date at sunset, UTC:', d)
        year,month,day = d[:3]
        plandict['date'] = '%04i-%02i-%02i' % (year, month, day)
    
        # Decide the pass.
        goodseeing = plandict['seeing'] < 1.3
        photometric = plandict['transparency'] > 0.9
    
        if goodseeing and photometric:
            passnum = 1
        elif goodseeing or photometric:
            passnum = 2
        else:
            passnum = 3
        plandict['pass'] = passnum
    
        ## ??
        plandict['portion'] = 1.0
        
        print('Replan command:')
        print()
        print('python2.7 nightlystrategy.py --seeg %(seeing).3f --seer %(seeing).3f --seez %(seeing).3f --sbg %(sbg).3f --sbr %(sbr).3f --sbz %(sbz).3f --transparency %(transparency).3f --start-date %(startdate)s --start-time %(starttime)s --end-date %(enddate)s --end-time %(endtime)s --date %(date)s --portion %(portion)f --pass %(pass)i' % plandict) 
        print()
    else:
        plandict = None
        expfactor = 0.

    rtn = (M, plandict, expnum)
    if not db:
        return rtn

    m.racenter  = M['ra_ccd']
    m.deccenter = M['dec_ccd']
    m.ebv  = ebv
    zp = M.get('zp', 0.)
    if zp is None:
        zp = 0.
    m.zeropoint = zp
    m.transparency = trans
    m.seeing = M.get('seeing', 0.)
    m.sky = M['skybright']
    m.expfactor = expfactor
    m.dx = M.get('dx', 0)
    m.dy = M.get('dy', 0)
    m.nmatched = M.get('nmatched',0)

    img = fitsio.read(fn, ext=1)
    cheaphash = np.sum(img)
    # cheaphash becomes an int64.
    m.md5sum = cheaphash

    set_tile_fields(m, phdr, tiles)

    m.save()

    return rtn
Exemple #9
0
    t1 = Time()
    print 'Stage tims:', t1 - t0

    R = stage_srcs(**P)
    P.update(R)
    t2 = Time()
    print 'Stage srcs:', t2 - t1

    R = stage_fitblobs(**P)
    P.update(R)
    t2b = Time()
    print 'Stage fitblobs:', t2b - t2

    P.update(catalogfn=catalogfn)
    stage_writecat(**P)
    t3 = Time()
    print 'Stage writecat:', t3 - t2b

    # Plots

    print
    print 'plots:', P.keys()
    print

    ps = PlotSequence(pspat)
    P.update(ps=ps, outdir='pipebrick-plots')
    stage_fitplots(**P)
    t4 = Time()
    print 'Stage fitplots:', t4 - t3
    print 'Total:', t4 - t0
Exemple #10
0
        if is_galaxy and t.type == 1:
            # deV
            src = DevGalaxy(pos, bright, shape)
        elif is_galaxy and t.type == 2:
            # exp
            src = ExpGalaxy(pos, bright, shape)
        else:
            src = PointSource(pos, bright)
        srcs.append(src)
    return srcs


if __name__ == '__main__':

    ps = PlotSequence('cfht')

    if False:
        # Don't start looking at the ACS image.  Don't.
        imgfn = 'cfht/acs_I_030mas_065_sci.VISRES.fits'
        wtfn = 'cfht/acs_I_030mas_065_wht.VISRES.fits'
        flagfn = 'cfht/acs_I_030mas_065_flg.VISRES.fits'

        catfn = 'cfht/acs_I_030mas_065_sci.VISRES.ldac'
        cat = fits_table(catfn, hdu=2)
        cat.ra = cat.alpha_j2000
        cat.dec = cat.delta_j2000

        img, imghdr = fitsio.read(imgfn, ext=ext, header=True)
        print('Read image', img.shape, img.dtype)
        img = img.astype(np.float32)
Exemple #11
0
        plt.xlabel('Depth: %s band' % band)
        plt.ylabel('Number of pixels')
        plt.title('Depth: Galaxy, %s' % band)

        ps.savefig()


if __name__ == '__main__':
    north = False
    if north == True:
        outfn = 'dr8-north-depth-concat.fits'
        summaryfn = 'dr8-north-depth-summary.fits'
        allfn = 'dr8-north-depth.fits'
        basedir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8/north'
        summarize_depths(basedir, outfn, summaryfn, allfn)

        ps = PlotSequence('depth')
        summary_plots(summaryfn, ps, 'BASS+MzLS DR8')
    else:
        outfn = 'dr8-south-depth-concat.fits'
        summaryfn = 'dr8-south-depth-summary.fits'
        allfn = 'dr8-south-depth.fits'
        basedir = '/global/project/projectdirs/cosmo/work/legacysurvey/dr8/south'
        summarize_depths(basedir, outfn, summaryfn, allfn)

        ps = PlotSequence('depth')
        summary_plots(summaryfn, ps, 'DECaLS DR8')
    import sys
    sys.exit(0)
Exemple #12
0
def main():
    global survey

    init()

    ps = PlotSequence('sky')
    # export LEGACY_SURVEY_DIR=/scratch1/scratchdirs/desiproc/DRs/dr4-bootes/legacypipe-dir/
    #survey = LegacySurveyData()
    survey = get_survey('dr4v2')
    ccds = survey.get_ccds_readonly()
    print(len(ccds), 'CCDs')
    ccds = ccds[ccds.camera == 'mosaic']
    print(len(ccds), 'Mosaic CCDs')
    # plt.clf()
    # plt.hist(ccds.mjd_obs % 1.0, bins=50)
    # plt.xlabel('MJD mod 1')
    # ps.savefig()

    ccds.imjd = np.floor(ccds.mjd_obs).astype(int)

    mjds = np.unique(ccds.imjd)
    print(len(mjds), 'unique MJDs')

    mp = multiproc(nthreads=8, init=init)

    allvals = []
    medmjds = []

    args = []

    for kk, mjd in enumerate(mjds):
        I = np.flatnonzero(ccds.imjd == mjd)
        print('MJD', mjd, ' (%i of %i):' % (kk + 1, len(mjds)), len(I), 'CCDs')
        if len(I) == 0:
            continue

        # pick one near the middle
        #i = I[len(I)/2]
        for i in [I[len(I) / 4], I[len(I) / 2], I[3 * len(I) / 4]]:

            ccd = ccds[i]

            key = dict(expnum=ccd.expnum, ccdname=ccd.ccdname)
            oldvals = key

            vals = cache.find(key)
            print('Got', vals.count(), 'cache hits for', key)
            gotone = False
            for val in vals:
                #if 'median_adu' in val:
                if 'mode_adu' in val:
                    print('cache hit:', val)
                    allvals.append(val)
                    medmjds.append(mjd)
                    gotone = True
                    break
                else:
                    print('partial cache hit:', val)
                    oldvals = val
                    ###!
                    cache.delete_one(val)
            if gotone:
                continue

            print('args: key', key, 'oldvals', oldvals)
            args.append((mjd, key, oldvals, ccd))

            # plt.clf()
            # plt.hist(tim.getImage().ravel(), range=(-0.1, 0.1), bins=50,
            #          histtype='step', color='b')
            # plt.axvline(med, color='b', alpha=0.3, lw=2)
            # plt.axvline(0., color='k', alpha=0.3, lw=2)
            # plt.xlabel('Sky-subtracted pixel values')
            # plt.title('Date ' + ccd.date_obs + ': ' + str(im))
            # ps.savefig()

    if len(args):
        meds = mp.map(read_sky_val, args)
        print('Medians:', meds)
        for (mjd, key, oldvals, ccd), val in zip(args, meds):
            if val is None:
                continue
            allvals.append(val)
            medmjds.append(mjd)

    medians = []
    median_adus = []
    mode_adus = []
    skyadus = []
    keepmjds = []

    for mjd, val in zip(medmjds, allvals):
        madu = val['median_adu']
        if madu is None:
            continue

        if 'median' in val:
            medians.append(val['median'])
        else:
            from tractor.brightness import NanoMaggies
            zpscale = NanoMaggies.zeropointToScale(val['ccdzpt'])
            med = (val['median_adu'] - val['skyadu']) / zpscale
            print('Computed median diff:', med, 'nmgy')
            medians.append(med)

        keepmjds.append(mjd)
        median_adus.append(val['median_adu'])
        mode_adus.append(val['mode_adu'])
        skyadus.append(val['skyadu'])

    medmjds = keepmjds

    median_adus = np.array(median_adus)
    mode_adus = np.array(mode_adus)
    skyadus = np.array(skyadus)
    medmjds = np.array(medmjds)
    medians = np.array(medians)

    plt.clf()
    plt.plot(medmjds, median_adus - skyadus, 'b.')
    plt.xlabel('MJD')
    #plt.ylabel('Image median after sky subtraction (nmgy)')
    plt.ylabel('Image median - SKYADU (ADU)')
    #plt.ylim(-0.03, 0.03)
    #plt.ylim(-10, 10)
    plt.ylim(-1, 1)

    plt.axhline(0, color='k', lw=2, alpha=0.5)
    ps.savefig()

    plt.clf()
    plt.plot(medmjds, mode_adus - skyadus, 'b.')
    plt.xlabel('MJD')
    plt.ylabel('Image mode - SKYADU (ADU)')
    plt.ylim(-1, 1)
    plt.axhline(0, color='k', lw=2, alpha=0.5)
    ps.savefig()

    print('Median pcts:', np.percentile(medians, [0, 2, 50, 98, 100]))
    mlo, mhi = np.percentile(medians, [2, 98])
    I = np.flatnonzero((medians > mlo) * (medians < mhi))

    d0 = np.mean(medmjds)
    dmjd = medmjds[I] - d0
    A = np.zeros((len(dmjd), 2))
    A[:, 0] = 1.
    A[:, 1] = dmjd
    b = np.linalg.lstsq(A, medians[I])[0]
    print('Lstsq:', b)
    offset = b[0]
    slope = b[1]
    xx = np.array([dmjd.min(), dmjd.max()])

    print('Offset', offset)
    print('Slope', slope)

    plt.clf()
    plt.plot(medmjds, medians, 'b.')
    ax = plt.axis()
    plt.plot(xx + d0, offset + slope * xx, 'b-')
    plt.axis(ax)
    plt.xlabel('MJD')
    plt.ylabel('Image median after sky subtraction (nmgy)')
    plt.ylim(-0.03, 0.03)
    plt.axhline(0, color='k', lw=2, alpha=0.5)
    ps.savefig()

    plt.clf()
    plt.plot(median_adus, skyadus, 'b.')
    ax = plt.axis()
    lo = min(ax[0], ax[2])
    hi = max(ax[1], ax[3])
    plt.plot([lo, hi], [lo, hi], 'k-', alpha=0.5)
    plt.xlabel('Median image ADU')
    plt.ylabel('SKYADU')
    plt.axis(ax)
    ps.savefig()

    plt.clf()
    plt.plot(medmjds, skyadus / median_adus, 'b.')
    plt.xlabel('MJD')
    plt.ylim(0.98, 1.02)
    plt.axhline(1.0, color='k', alpha=0.5)
    plt.ylabel('SKYADU / median image ADU')
    ps.savefig()
Exemple #13
0
import matplotlib
matplotlib.use('Agg')
import pylab as plt

import numpy as np
import fitsio
from astrometry.util.fits import fits_table, merge_tables
from astrometry.util.util import wcs_pv2sip_hdr
from astrometry.util.plotutils import PlotSequence

if __name__ == '__main__':

    import astrometry

    ps = PlotSequence('pretty')

    catdir = os.path.join(os.path.dirname(astrometry.__file__), 'catalogs')
    fn = os.path.join(catdir, 'ngc2000.fits')
    NGC = fits_table(fn)
    NGC.name = np.array(['NGC %i' % ngcnum for ngcnum in NGC.ngcnum])
    #NGC.delete_column('ngcnum')

    IC = fits_table(os.path.join(catdir, 'ic2000.fits'))
    IC.name = np.array(['IC %i' % icnum for icnum in IC.icnum])
    #IC.delete_column('icnum')

    UGC = fits_table(os.path.join(catdir, 'ugc.fits'))
    UGC.name = np.array(['UGC %i' % ugcnum for ugcnum in UGC.ugcnum])
    #UGC.delete_column('ugcnum')
Exemple #14
0
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None,
                      use_ceres=True, ceres_block=8,
                      save_fits=False, get_models=False, ps=None,
                      psf_broadening=None,
                      pixelized_psf=False,
                      get_masks=None,
                      move_crpix=False,
                      modelsky_dir=None):
    '''
    Given a list of tractor sources *cat*
    and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id)
    runs forced photometry, returning a FITS table the same length as *cat*.

    *get_masks*: the WCS to resample mask bits into.
    '''
    from tractor import NanoMaggies, PointSource, Tractor, ExpGalaxy, DevGalaxy, FixedCompositeGalaxy

    if not pixelized_psf and psf_broadening is None:
        # PSF broadening in post-reactivation data, by band.
        # Newer version from Aaron's email to decam-chatter, 2018-06-14.
        broadening = { 1: 1.0405, 2: 1.0346, 3: None, 4: None }
        psf_broadening = broadening[band]

    if False:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence('wise-forced-w%i' % band)
    plots = (ps is not None)
    if plots:
        import pylab as plt
    
    wantims = (plots or save_fits or get_models)
    wanyband = 'w'
    if get_models:
        models = {}

    wband = 'w%i' % band

    fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix',
              'pronexp']

    Nsrcs = len(cat)
    phot = fits_table()
    # Filled in based on unique tile overlap
    phot.wise_coadd_id = np.array(['        '] * Nsrcs)
    phot.set(wband + '_psfdepth', np.zeros(len(phot), np.float32))

    ra  = np.array([src.getPosition().ra  for src in cat])
    dec = np.array([src.getPosition().dec for src in cat])

    nexp = np.zeros(Nsrcs, np.int16)
    mjd  = np.zeros(Nsrcs, np.float64)
    central_flux = np.zeros(Nsrcs, np.float32)

    fitstats = {}
    tims = []

    if get_masks:
        mh,mw = get_masks.shape
        maskmap = np.zeros((mh,mw), np.uint32)
    
    for tile in tiles:
        print('Reading WISE tile', tile.coadd_id, 'band', band)

        tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band,
                                       bandname=wanyband, roiradecbox=roiradecbox)
        if tim is None:
            print('Actually, no overlap with tile', tile.coadd_id)
            continue

        if plots:
            sig1 = tim.sig1
            plt.clf()
            plt.imshow(tim.getImage(), interpolation='nearest', origin='lower',
                       cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
            plt.colorbar()
            tag = '%s W%i' % (tile.coadd_id, band)
            plt.title('%s: tim data' % tag)
            ps.savefig()

            plt.clf()
            plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
                     range=(-5,10), bins=100)
            plt.xlabel('Per-pixel intensity (Sigma)')
            plt.title(tag)
            ps.savefig()

        if move_crpix and band in [1, 2]:
            realwcs = tim.wcs.wcs
            x,y = realwcs.crpix
            tile_crpix = tile.get('crpix_w%i' % band)
            dx = tile_crpix[0] - 1024.5
            dy = tile_crpix[1] - 1024.5
            realwcs.set_crpix(x+dx, y+dy)
            #print('CRPIX', x,y, 'shift by', dx,dy, 'to', realwcs.crpix)

        if modelsky_dir and band in [1, 2]:
            fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band))
            if not os.path.exists(fn):
                raise RuntimeError('WARNING: does not exist:', fn)
            x0,x1,y0,y1 = tim.roi
            bg = fitsio.FITS(fn)[2][y0:y1, x0:x1]
            #print('Read background map:', bg.shape, bg.dtype, 'vs image', tim.shape)

            if plots:
                plt.clf()
                plt.subplot(1,2,1)
                plt.imshow(tim.getImage(), interpolation='nearest', origin='lower',
                           cmap='gray', vmin=-3 * sig1, vmax=5 * sig1)
                plt.subplot(1,2,2)
                plt.imshow(bg, interpolation='nearest', origin='lower',
                           cmap='gray', vmin=-3 * sig1, vmax=5 * sig1)
                tag = '%s W%i' % (tile.coadd_id, band)
                plt.suptitle(tag)
                ps.savefig()

                plt.clf()
                ha = dict(range=(-5,10), bins=100, histtype='step')
                plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(),
                         color='b', label='Original', **ha)
                plt.hist(((tim.getImage()-bg) * tim.inverr)[tim.inverr > 0].ravel(),
                         color='g', label='Minus Background', **ha)
                plt.axvline(0, color='k', alpha=0.5)
                plt.xlabel('Per-pixel intensity (Sigma)')
                plt.legend()
                plt.title(tag + ': background')
                ps.savefig()

            # Actually subtract the background!
            tim.data -= bg

        # Floor the per-pixel variances
        if band in [1,2]:
            # in Vega nanomaggies per pixel
            floor_sigma = {1: 0.5, 2: 2.0}
            with np.errstate(divide='ignore'):
                new_ie = 1. / np.hypot(1./tim.inverr, floor_sigma[band])
            new_ie[tim.inverr == 0] = 0.

            if plots:
                plt.clf()
                plt.plot((1. / tim.inverr[tim.inverr>0]).ravel(), (1./new_ie[tim.inverr>0]).ravel(), 'b.')
                plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band))
                plt.xlabel('original')
                plt.ylabel('floored')
                ps.savefig()

            tim.inverr = new_ie

        # Read mask file?
        if get_masks:
            from astrometry.util.resample import resample_with_wcs, OverlapError
            # unwise_dir can be a colon-separated list of paths
            tilemask = None
            for d in tile.unwise_dir.split(':'):
                fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id,
                                  'unwise-%s-msk.fits.gz' % tile.coadd_id)
                if os.path.exists(fn):
                    print('Reading unWISE mask file', fn)
                    x0,x1,y0,y1 = tim.roi
                    tilemask = fitsio.FITS(fn)[0][y0:y1,x0:x1]
                    break
            if tilemask is None:
                print('unWISE mask file for tile', tile.coadd_id, 'does not exist')
            else:
                try:
                    tanwcs = tim.wcs.wcs
                    assert(tanwcs.shape == tilemask.shape)
                    Yo,Xo,Yi,Xi,_ = resample_with_wcs(get_masks, tanwcs, intType=np.int16)
                    # Only deal with mask pixels that are set.
                    I, = np.nonzero(tilemask[Yi,Xi] > 0)
                    # Trim to unique area for this tile
                    rr,dd = get_masks.pixelxy2radec(Yo[I]+1, Xo[I]+1)
                    good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
                    I = I[good]
                    maskmap[Yo[I],Xo[I]] = tilemask[Yi[I], Xi[I]]
                except OverlapError:
                    # Shouldn't happen by this point
                    print('No overlap between WISE tile', tile.coadd_id, 'and brick')

        # The tiles have some overlap, so zero out pixels outside the
        # tile's unique area.
        th,tw = tim.shape
        xx,yy = np.meshgrid(np.arange(tw), np.arange(th))
        rr,dd = tim.wcs.wcs.pixelxy2radec(xx+1, yy+1)
        unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
        #print(np.sum(unique), 'of', (th*tw), 'pixels in this tile are unique')
        tim.inverr[unique == False] = 0.
        del xx,yy,rr,dd,unique

        if plots:
            sig1 = tim.sig1
            plt.clf()
            plt.imshow(tim.getImage() * (tim.inverr > 0),
                       interpolation='nearest', origin='lower',
                       cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
            plt.colorbar()
            tag = '%s W%i' % (tile.coadd_id, band)
            plt.title('%s: tim data (unique)' % tag)
            ps.savefig()

        if pixelized_psf:
            import unwise_psf
            if (band == 1) or (band == 2):
                # we only have updated PSFs for W1 and W2
                psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id, 
                                                   modelname='neo4_unwisecat')
            else:
                psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id)

            if band == 4:
                # oversample (the unwise_psf models are at native W4 5.5"/pix,
                # while the unWISE coadds are made at 2.75"/pix.
                ph,pw = psfimg.shape
                subpsf = np.zeros((ph*2-1, pw*2-1), np.float32)
                from astrometry.util.util import lanczos3_interpolate
                xx,yy = np.meshgrid(np.arange(0., pw-0.51, 0.5, dtype=np.float32),
                                    np.arange(0., ph-0.51, 0.5, dtype=np.float32))
                xx = xx.ravel()
                yy = yy.ravel()
                ix = xx.astype(np.int32)
                iy = yy.astype(np.int32)
                dx = (xx - ix).astype(np.float32)
                dy = (yy - iy).astype(np.float32)
                psfimg = psfimg.astype(np.float32)
                rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg])

                if plots:
                    plt.clf()
                    plt.imshow(psfimg, interpolation='nearest', origin='lower')
                    plt.title('Original PSF model')
                    ps.savefig()
                    plt.clf()
                    plt.imshow(subpsf, interpolation='nearest', origin='lower')
                    plt.title('Subsampled PSF model')
                    ps.savefig()

                psfimg = subpsf
                del xx, yy, ix, iy, dx, dy

            from tractor.psf import PixelizedPSF
            psfimg /= psfimg.sum()
            fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0}
            psfimg *= fluxrescales[band]
            tim.psf = PixelizedPSF(psfimg)

        if psf_broadening is not None and not pixelized_psf:
            # psf_broadening is a factor by which the PSF FWHMs
            # should be scaled; the PSF is a little wider
            # post-reactivation.
            psf = tim.getPsf()
            from tractor import GaussianMixturePSF
            if isinstance(psf, GaussianMixturePSF):
                #
                print('Broadening PSF: from', psf)
                p0 = psf.getParams()
                pnames = psf.getParamNames()
                p1 = [p * psf_broadening**2 if 'var' in name else p
                      for (p, name) in zip(p0, pnames)]
                psf.setParams(p1)
                print('Broadened PSF:', psf)
            else:
                print('WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf))

        wcs = tim.wcs.wcs
        ok,x,y = wcs.radec2pixelxy(ra, dec)
        x = np.round(x - 1.).astype(int)
        y = np.round(y - 1.).astype(int)
        good = (x >= 0) * (x < tw) * (y >= 0) * (y < th)
        # Which sources are in this brick's unique area?
        usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2)
        I, = np.nonzero(good * usrc)

        nexp[I] = tim.nuims[y[I], x[I]]
        if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'):
            mjd[I] = (tim.mjdmin + tim.mjdmax) / 2.
        phot.wise_coadd_id[I] = tile.coadd_id

        central_flux[I] = tim.getImage()[y[I], x[I]]
        del x,y,good,usrc

        # PSF norm for depth
        psf = tim.getPsf()
        h,w = tim.shape
        patch = psf.getPointSourcePatch(h//2, w//2).patch
        psfnorm = np.sqrt(np.sum(patch**2))
        # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags.
        psfdepth = 1. / (tim.sig1 / psfnorm)**2
        phot.get(wband + '_psfdepth')[I] = psfdepth

        tim.tile = tile
        tims.append(tim)

    if plots:
        plt.clf()
        mn,mx = 0.1, 20000
        plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100,
                 range=(np.log10(mn), np.log10(mx)))
        logt = np.arange(0, 5)
        plt.xticks(logt, ['%i' % i for i in 10.**logt])
        plt.title('Central fluxes (W%i)' % band)
        plt.axvline(np.log10(20000), color='k')
        plt.axvline(np.log10(1000), color='k')
        ps.savefig()

    # Eddie's non-secret recipe:
    #- central pixel <= 1000: 19x19 pix box size
    #- central pixel in 1000 - 20000: 59x59 box size
    #- central pixel > 20000 or saturated: 149x149 box size
    #- object near "bright star": 299x299 box size
    nbig = nmedium = nsmall = 0
    for src,cflux in zip(cat, central_flux):
        if cflux > 20000:
            R = 100
            nbig += 1
        elif cflux > 1000:
            R = 30
            nmedium += 1
        else:
            R = 15
            nsmall += 1
        if isinstance(src, PointSource):
            src.fixedRadius = R
        else:
            ### FIXME -- sizes for galaxies..... can we set PSF size separately?
            galrad = 0
            # RexGalaxy is a subclass of ExpGalaxy
            if isinstance(src, (ExpGalaxy, DevGalaxy)):
                galrad = src.shape.re
            elif isinstance(src, FixedCompositeGalaxy):
                galrad = max(src.shapeExp.re, src.shapeDev.re)
            pixscale = 2.75
            src.halfsize = int(np.hypot(R, galrad * 5 / pixscale))

    #print('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small')

    minsb = 0.
    fitsky = False

    tractor = Tractor(tims, cat)
    if use_ceres:
        from tractor.ceres_optimizer import CeresOptimizer
        tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block)
    tractor.freezeParamsRecursive('*')
    tractor.thawPathsTo(wanyband)

    kwa = dict(fitstat_extras=[('pronexp', [tim.nims for tim in tims])])
    t0 = Time()

    R = tractor.optimize_forced_photometry(
        minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True,
        variance=True, shared_params=False,
        wantims=wantims, **kwa)
    print('unWISE forced photometry took', Time() - t0)

    if use_ceres:
        term = R.ceres_status['termination']
        # Running out of memory can cause failure to converge
        # and term status = 2.
        # Fail completely in this case.
        if term != 0:
            print('Ceres termination status:', term)
            raise RuntimeError(
                'Ceres terminated with status %i' % term)

    if wantims:
        ims1 = R.ims1
    flux_invvars = R.IV
    if R.fitstats is not None:
        for k in fskeys:
            x = getattr(R.fitstats, k)
            fitstats[k] = np.array(x).astype(np.float32)

    if save_fits:
        for i,tim in enumerate(tims):
            tile = tim.tile
            (dat, mod, ie, chi, roi) = ims1[i]
            wcshdr = fitsio.FITSHDR()
            tim.wcs.wcs.add_to_header(wcshdr)
            tag = 'fit-%s-w%i' % (tile.coadd_id, band)
            fitsio.write('%s-data.fits' %
                         tag, dat, clobber=True, header=wcshdr)
            fitsio.write('%s-mod.fits' % tag,  mod,
                         clobber=True, header=wcshdr)
            fitsio.write('%s-chi.fits' % tag,  chi,
                         clobber=True, header=wcshdr)

    if plots:
        # Create models for just the brightest sources
        bright_cat = [src for src in cat
                      if src.getBrightness().getBand(wanyband) > 1000]
        print('Bright soures:', len(bright_cat))
        btr = Tractor(tims, bright_cat)
        for tim in tims:
            mod = btr.getModelImage(tim)
            tile = tim.tile
            tag = '%s W%i' % (tile.coadd_id, band)
            sig1 = tim.sig1
            plt.clf()
            plt.imshow(mod, interpolation='nearest', origin='lower',
                       cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
            plt.colorbar()
            plt.title('%s: bright-star models' % tag)
            ps.savefig()

    if get_models:
        for i,tim in enumerate(tims):
            tile = tim.tile
            (dat, mod, ie, chi, roi) = ims1[i]
            models[(tile.coadd_id, band)] = (mod, dat, ie, tim.roi, tim.wcs.wcs)

    if plots:
        for i,tim in enumerate(tims):
            tile = tim.tile
            tag = '%s W%i' % (tile.coadd_id, band)
            (dat, mod, ie, chi, roi) = ims1[i]
            sig1 = tim.sig1
            plt.clf()
            plt.imshow(dat, interpolation='nearest', origin='lower',
                       cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
            plt.colorbar()
            plt.title('%s: data' % tag)
            ps.savefig()
            plt.clf()
            plt.imshow(mod, interpolation='nearest', origin='lower',
                       cmap='gray', vmin=-3 * sig1, vmax=25 * sig1)
            plt.colorbar()
            plt.title('%s: model' % tag)
            ps.savefig()

            plt.clf()
            plt.imshow(chi, interpolation='nearest', origin='lower',
                       cmap='gray', vmin=-5, vmax=+5)
            plt.colorbar()
            plt.title('%s: chi' % tag)
            ps.savefig()


    nm = np.array([src.getBrightness().getBand(wanyband) for src in cat])
    nm_ivar = flux_invvars
    # Sources out of bounds, eg, never change from their default
    # (1-sigma or whatever) initial fluxes.  Zero them out instead.
    nm[nm_ivar == 0] = 0.

    phot.set(wband + '_nanomaggies', nm.astype(np.float32))
    phot.set(wband + '_nanomaggies_ivar', nm_ivar.astype(np.float32))
    dnm = np.zeros(len(nm_ivar), np.float32)
    okiv = (nm_ivar > 0)
    dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32)
    okflux = (nm > 0)
    mag = np.zeros(len(nm), np.float32)
    mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux])
                   ).astype(np.float32)
    dmag = np.zeros(len(nm), np.float32)
    ok = (okiv * okflux)
    dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok])
                ).astype(np.float32)
    mag[np.logical_not(okflux)] = np.nan
    dmag[np.logical_not(ok)] = np.nan

    phot.set(wband + '_mag', mag)
    phot.set(wband + '_mag_err', dmag)

    for k in fskeys:
        phot.set(wband + '_' + k, fitstats[k])
    phot.set(wband + '_nexp', nexp)
    if not np.all(mjd == 0):
        phot.set(wband + '_mjd', mjd)

    rtn = wphotduck()
    rtn.phot = phot
    rtn.models = None
    rtn.maskmap = None
    if get_models:
        rtn.models = models
    if get_masks:
        rtn.maskmap = maskmap
    return rtn
def main():

    # ps = PlotSequence('pro')
    # star_profiles(ps)
    # sys.exit(0)

    #survey_dir = '/project/projectdirs/desiproc/dr3'
    #survey = LegacySurveyData(survey_dir=survey_dir)
    survey = LegacySurveyData()

    ralo, rahi = 240, 245
    declo, dechi = 5, 12

    ps = PlotSequence('comp')

    bands = 'grz'

    ccdfn = 'ccds-forced.fits'
    if not os.path.exists(ccdfn):
        ccds = survey.get_annotated_ccds()
        ccds.cut((ccds.ra > ralo) * (ccds.ra < rahi) * (ccds.dec > declo) *
                 (ccds.dec < dechi))
        print(len(ccds), 'CCDs')

        ccds.path = np.array([
            os.path.join(  #'dr3',
                'forced', ('%08i' % e)[:5], '%08i' % e,
                'decam-%08i-%s-forced.fits' % (e, n.strip()))
            for e, n in zip(ccds.expnum, ccds.ccdname)
        ])
        I, = np.nonzero([os.path.exists(fn) for fn in ccds.path])
        print(len(I), 'CCDs with forced photometry')
        ccds.cut(I)

        #ccds = ccds[:500]
        #e,I = np.unique(ccds.expnum, return_index=True)
        #print(len(I), 'unique exposures')
        #ccds.cut(I)

        FF = read_forcedphot_ccds(ccds, survey)
        FF.writeto('forced-all-matches.fits')

        # - z band -- no trend w/ PS1 mag (brighter-fatter)

        ccds.writeto(ccdfn)

    ccdfn2 = 'ccds-forced-2.fits'
    if not os.path.exists(ccdfn2):
        ccds = fits_table(ccdfn)
        # Split into brighter/fainter halves
        FF = fits_table('forced-all-matches.fits')

        print(len(FF), 'forced measurements')
        FF.cut(FF.masked == False)
        print(len(FF), 'forced measurements not masked')

        ccds.brightest_mdiff = np.zeros(len(ccds))
        ccds.brightest_mscatter = np.zeros(len(ccds))
        ccds.bright_mdiff = np.zeros(len(ccds))
        ccds.bright_mscatter = np.zeros(len(ccds))
        ccds.faint_mdiff = np.zeros(len(ccds))
        ccds.faint_mscatter = np.zeros(len(ccds))
        for iccd in range(len(ccds)):
            I = np.flatnonzero(FF.iforced == iccd)
            if len(I) == 0:
                continue
            if len(I) < 10:
                continue
            F = FF[I]
            b = np.percentile(F.psmag, 10)
            m = np.median(F.psmag)
            print(len(F), 'matches for CCD', iccd, 'median mag', m, '10th pct',
                  b)
            J = np.flatnonzero(F.psmag < b)
            diff = F.mag[J] - F.psmag[J]
            ccds.brightest_mdiff[iccd] = np.median(diff)
            ccds.brightest_mscatter[iccd] = (np.percentile(diff, 84) -
                                             np.percentile(diff, 16)) / 2.
            J = np.flatnonzero(F.psmag < m)
            diff = F.mag[J] - F.psmag[J]
            ccds.bright_mdiff[iccd] = np.median(diff)
            ccds.bright_mscatter[iccd] = (np.percentile(diff, 84) -
                                          np.percentile(diff, 16)) / 2.
            J = np.flatnonzero(F.psmag > m)
            diff = F.mag[J] - F.psmag[J]
            ccds.faint_mdiff[iccd] = np.median(diff)
            ccds.faint_mscatter[iccd] = (np.percentile(diff, 84) -
                                         np.percentile(diff, 16)) / 2.

        ccds.writeto(ccdfn2)

    ccds = fits_table(ccdfn2)

    plt.clf()
    plt.hist(ccds.nforced, bins=100)
    plt.title('nforced')
    ps.savefig()

    plt.clf()
    plt.hist(ccds.nmatched, bins=100)
    plt.title('nmatched')
    ps.savefig()

    #ccds.cut(ccds.nmatched >= 150)
    ccds.cut(ccds.nmatched >= 50)
    print('Cut to', len(ccds), 'with >50 matched')

    ccds.cut(ccds.photometric)
    print('Cut to', len(ccds), 'photometric')

    neff = 1. / ccds.psfnorm_mean**2
    # Narcsec is in arcsec**2
    narcsec = neff * ccds.pixscale_mean**2
    # to arcsec
    narcsec = np.sqrt(narcsec)
    # Correction factor to get back to equivalent of Gaussian sigma
    narcsec /= (2. * np.sqrt(np.pi))
    # Conversion factor to FWHM (2.35)
    narcsec *= 2. * np.sqrt(2. * np.log(2.))
    ccds.psfsize = narcsec

    for band in bands:
        I = np.flatnonzero(
            (ccds.filter == band) * (ccds.photometric) * (ccds.blacklist_ok))

        mlo, mhi = -0.01, 0.05

        plt.clf()
        plt.plot(ccds.ccdzpt[I], ccds.exptime[I], 'k.', alpha=0.1)

        J = np.flatnonzero((ccds.filter == band) * (ccds.photometric == False))
        plt.plot(ccds.ccdzpt[J], ccds.exptime[J], 'r.', alpha=0.1)

        plt.xlabel('Zeropoint (mag)')
        plt.ylabel('exptime')
        plt.title('DR3: EDR region, Forced phot: %s band' % band)
        ps.savefig()

        plt.clf()
        plt.plot(ccds.ccdzpt[I],
                 np.clip(ccds.mdiff[I], mlo, mhi),
                 'k.',
                 alpha=0.1)
        plt.xlabel('Zeropoint (mag)')
        plt.ylabel('DECaLS PSF - PS1 (mag)')
        plt.axhline(0, color='k', alpha=0.2)
        #plt.axis([0, mxsee, mlo,mhi])
        plt.title('DR3: EDR region, Forced phot: %s band' % band)
        ps.savefig()

        plt.clf()
        plt.plot(ccds.ccdzpt[I], ccds.psfsize[I], 'k.', alpha=0.1)
        plt.xlabel('Zeropoint (mag)')
        plt.ylabel('PSF size (arcsec)')
        plt.title('DR3: EDR region, Forced phot: %s band' % band)
        ps.savefig()

        # plt.clf()
        # plt.plot(ccds.avsky[I],
        #          np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1)
        # plt.xlabel('avsky')
        # plt.ylabel('DECaLS PSF - PS1 (mag)')
        # plt.axhline(0, color='k', alpha=0.2)
        # plt.title('DR3: EDR region, Forced phot: %s band' % band)
        # ps.savefig()
        #
        # plt.clf()
        # plt.plot(ccds.meansky[I],
        #          np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1)
        # plt.xlabel('meansky')
        # plt.ylabel('DECaLS PSF - PS1 (mag)')
        # plt.axhline(0, color='k', alpha=0.2)
        # plt.title('DR3: EDR region, Forced phot: %s band' % band)
        # ps.savefig()

        plt.clf()
        lo, hi = (-0.02, 0.05)
        ha = dict(bins=50, histtype='step', range=(lo, hi))
        n, b, p1 = plt.hist(ccds.brightest_mdiff[I], color='r', **ha)
        n, b, p2 = plt.hist(ccds.bright_mdiff[I], color='g', **ha)
        n, b, p3 = plt.hist(ccds.faint_mdiff[I], color='b', **ha)
        plt.legend((p1[0], p2[0], p3[0]),
                   ('Brightest 10%', 'Brightest 50%', 'Faintest 50%'))
        plt.xlabel('DECaLS PSF - PS1 (mag)')
        plt.ylabel('Number of CCDs')
        plt.title('DR3: EDR region, Forced phot: %s band' % band)
        plt.xlim(lo, hi)
        ps.savefig()

    for band in bands:
        I = np.flatnonzero(ccds.filter == band)

        mxsee = 4.
        mlo, mhi = -0.01, 0.05

        plt.clf()
        plt.plot(np.clip(ccds.psfsize[I], 0, mxsee),
                 np.clip(ccds.mdiff[I], mlo, mhi),
                 'k.',
                 alpha=0.1)

        # for p in [1,2,3]:
        #     J = np.flatnonzero(ccds.tilepass[I] == p)
        #     if len(J):
        #         plt.plot(np.clip(ccds.psfsize[I[J]], 0, mxsee),
        #                  np.clip(ccds.mdiff[I[J]], mlo,mhi), '.', color='rgb'[p-1], alpha=0.2)

        #plt.plot(ccds.seeing[I], ccds.mdiff[I], 'b.')
        plt.xlabel('PSF size (arcsec)')
        plt.ylabel('DECaLS PSF - PS1 (mag)')
        plt.axhline(0, color='k', alpha=0.2)
        plt.axis([0, mxsee, mlo, mhi])
        plt.title('DR3: EDR region, Forced phot: %s band' % band)
        ps.savefig()

    # Group by exposure

    for band in bands:
        I = np.flatnonzero(
            (ccds.filter == band) * (ccds.photometric) * (ccds.blacklist_ok))

        E, J = np.unique(ccds.expnum[I], return_index=True)
        print(len(E), 'unique exposures in', band)
        exps = ccds[I[J]]
        print(len(exps), 'unique exposures in', band)
        assert (len(np.unique(exps.expnum)) == len(exps))
        exps.ddiff = np.zeros(len(exps))
        exps.dsize = np.zeros(len(exps))
        exps.nccds = np.zeros(len(exps), int)

        exps.brightest_ddiff = np.zeros(len(exps))
        exps.bright_ddiff = np.zeros(len(exps))
        exps.faint_ddiff = np.zeros(len(exps))

        for iexp, exp in enumerate(exps):
            J = np.flatnonzero(ccds.expnum[I] == exp.expnum)
            J = I[J]
            print(len(J), 'CCDs in exposure', exp.expnum)

            exps.brightest_mdiff[iexp] = np.median(ccds.brightest_mdiff[J])
            exps.bright_mdiff[iexp] = np.median(ccds.bright_mdiff[J])
            exps.faint_mdiff[iexp] = np.median(ccds.faint_mdiff[J])

            exps.brightest_ddiff[iexp] = (
                np.percentile(ccds.brightest_mdiff[J], 84) -
                np.percentile(ccds.brightest_mdiff[J], 16)) / 2.
            exps.bright_ddiff[iexp] = (
                np.percentile(ccds.bright_mdiff[J], 84) -
                np.percentile(ccds.bright_mdiff[J], 16)) / 2.
            exps.faint_ddiff[iexp] = (
                np.percentile(ccds.faint_mdiff[J], 84) -
                np.percentile(ccds.faint_mdiff[J], 16)) / 2.

            exps.mdiff[iexp] = np.median(ccds.mdiff[J])
            exps.ddiff[iexp] = (np.percentile(ccds.mdiff[J], 84) -
                                np.percentile(ccds.mdiff[J], 16)) / 2.
            exps.psfsize[iexp] = np.median(ccds.psfsize[J])
            exps.dsize[iexp] = (np.percentile(ccds.psfsize[J], 84) -
                                np.percentile(ccds.psfsize[J], 16)) / 2.
            exps.nccds[iexp] = len(J)

        mxsee = 4.
        mlo, mhi = -0.01, 0.05

        exps.cut(exps.nccds >= 10)

        plt.clf()
        plt.errorbar(
            np.clip(exps.psfsize, 0, mxsee),
            np.clip(exps.mdiff, mlo, mhi),
            yerr=exps.ddiff,
            #xerr=exps.dsize,
            fmt='.',
            color='k')

        # plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
        #              np.clip(exps.brightest_mdiff, mlo,mhi),
        #              yerr=exps.brightest_ddiff, fmt='r.')
        # plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
        #              np.clip(exps.bright_mdiff, mlo,mhi),
        #              yerr=exps.bright_ddiff, fmt='g.')
        # plt.errorbar(np.clip(exps.psfsize, 0, mxsee),
        #              np.clip(exps.faint_mdiff, mlo,mhi),
        #              yerr=exps.faint_ddiff, fmt='b.')

        # plt.plot(np.clip(exps.psfsize, 0, mxsee),
        #              np.clip(exps.brightest_mdiff, mlo,mhi), 'r.')
        # plt.plot(np.clip(exps.psfsize, 0, mxsee),
        #              np.clip(exps.bright_mdiff, mlo,mhi), 'g.')
        # plt.plot(np.clip(exps.psfsize, 0, mxsee),
        #          np.clip(exps.faint_mdiff, mlo,mhi), 'b.')

        #plt.plot(ccds.seeing[I], ccds.mdiff[I], 'b.')
        plt.xlabel('PSF size (arcsec)')
        plt.ylabel('DECaLS PSF - PS1 (mag)')
        plt.axhline(0, color='k', alpha=0.2)
        plt.axis([0, mxsee, mlo, mhi])
        plt.title('DR3: EDR region, Forced phot: %s band' % band)
        ps.savefig()

        plt.clf()
        p1 = plt.plot(np.clip(exps.psfsize, 0, mxsee),
                      np.clip(exps.brightest_mdiff, mlo, mhi),
                      'r.',
                      alpha=0.5)
        p2 = plt.plot(np.clip(exps.psfsize, 0, mxsee),
                      np.clip(exps.bright_mdiff, mlo, mhi),
                      'g.',
                      alpha=0.5)
        p3 = plt.plot(np.clip(exps.psfsize, 0, mxsee),
                      np.clip(exps.faint_mdiff, mlo, mhi),
                      'b.',
                      alpha=0.5)
        plt.legend((p1[0], p2[0], p3[0]),
                   ('Brightest 10%', 'Brightest 50%', 'Faintest 50%'))
        plt.xlabel('PSF size (arcsec)')
        plt.ylabel('DECaLS PSF - PS1 (mag)')
        plt.axhline(0, color='k', alpha=0.2)
        plt.axis([0, mxsee, mlo, mhi])
        plt.title('DR3: EDR region, Forced phot: %s band' % band)
        ps.savefig()

        J = np.argsort(-exps.mdiff)
        for j in J:
            print('  Photometric diff', exps.mdiff[j], 'PSF size',
                  exps.psfsize[j], 'expnum', exps.expnum[j])

    sys.exit(0)
Exemple #16
0
                        metavar='DECam-filename.fits.fz',
                        nargs='+',
                        help='DECam raw images to process')

    args = parser.parse_args()
    fns = args.images
    #exts = args.ext
    #if len(exts) == 0:
    #    exts = ['N4']

    ext = args.ext

    ps = None
    if args.plots is not None:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence(args.plots)

    measargs = dict()
    if args.aprad is not None:
        measargs.update(aprad=args.aprad)

    vals = {}
    for fn in fns:
        #for ext in exts:
        print()
        print('Measuring', fn, 'ext', ext)
        d = measure_raw(fn, ext=ext, ps=ps, measargs=measargs)
        d.update(filename=fn, ext=ext)
        for k, v in d.items():
            if not k in vals:
                vals[k] = [v]
Exemple #17
0
from tractor import *

from common import *


def subplot_grid(ny, nx, i):
    y = i / nx
    x = i % nx
    plt.subplot(ny, nx, 1 + ((ny - 1) - y) * nx + x)


if __name__ == '__main__':
    decals = Decals()

    ps = PlotSequence('psf')

    #B = decals.get_bricks()

    T = decals.get_ccds()
    T.cut(T.extname == 'S1')
    print 'Cut to', len(T)
    #print 'Expnums:', T[:10]
    T.cut(T.expnum == 348233)
    print 'Cut to', len(T)
    T.about()

    band = T.filter[0]
    print 'Band:', band

    im = DecamImage(T[0])
Exemple #18
0
def rbmain():
    travis = 'travis' in sys.argv

    extra_args = [
        '--old-calibs-ok',
        #'--verbose',
    ]
    if travis:
        extra_args.extend(
            ['--no-wise-ceres', '--no-gaia', '--no-large-galaxies'])

    if 'ceres' in sys.argv:
        surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
        main(args=[
            '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
            '--no-wise', '--force-all', '--no-write', '--ceres',
            '--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres',
            '--no-depth-cut'
        ])
        sys.exit(0)

    # demo RexGalaxy, with plots
    if False:
        from legacypipe.survey import RexGalaxy
        from tractor import NanoMaggies, PixPos
        from tractor import Image, GaussianMixturePSF, LinearPhotoCal
        from legacypipe.survey import LogRadius
        rex = RexGalaxy(PixPos(1., 2.), NanoMaggies(r=3.), LogRadius(0.))
        print('Rex:', rex)
        print('Rex params:', rex.getParams())
        print('Rex nparams:', rex.numberOfParams())
        H, W = 100, 100
        tim = Image(data=np.zeros((H, W), np.float32),
                    inverr=np.ones((H, W), np.float32),
                    psf=GaussianMixturePSF(1., 0., 0., 4., 4., 0.),
                    photocal=LinearPhotoCal(1., band='r'))
        derivs = rex.getParamDerivatives(tim)
        print('Derivs:', len(derivs))
        print('Rex params:', rex.getParamNames())

        import pylab as plt
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence('rex')

        for d, nm in zip(derivs, rex.getParamNames()):
            plt.clf()
            plt.imshow(d.patch, interpolation='nearest', origin='lower')
            plt.title('Derivative %s' % nm)
            ps.savefig()

        sys.exit(0)

    # Test RexGalaxy

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
    outdir = 'out-testcase6-rex'
    main(args=[
        '--brick',
        '1102p240',
        '--zoom',
        '500',
        '600',
        '650',
        '750',
        '--force-all',
        '--no-write',
        '--no-wise',
        #'--rex', #'--plots',
        '--survey-dir',
        surveydir,
        '--outdir',
        outdir
    ] + extra_args)
    fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 2)
    print('Types:', T.type)
    # Since there is a Tycho-2 star in the blob, forced to be PSF.
    assert (T.type[0] == 'PSF ')
    cmd = (
        '(cd %s && sha256sum -c %s)' %
        (outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum')))
    print(cmd)
    rtn = os.system(cmd)
    assert (rtn == 0)

    # Test with a Tycho-2 star in the blob.

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase6')
    outdir = 'out-testcase6'
    main(args=[
        '--brick', '1102p240', '--zoom', '500', '600', '650', '750',
        '--force-all', '--no-write', '--no-wise', '--survey-dir', surveydir,
        '--outdir', outdir
    ] + extra_args)
    fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 2)
    print('Types:', T.type)
    # Since there is a Tycho-2 star in the blob, forced to be PSF.
    assert (T.type[0] == 'PSF ')

    # Test that we can run splinesky calib if required...

    from legacypipe.decam import DecamImage
    DecamImage.splinesky_boxsize = 128

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
    outdir = 'out-testcase4'

    fn = os.path.join(surveydir, 'calib', 'decam', 'splinesky', '00431',
                      '00431608', 'decam-00431608-N3.fits')
    if os.path.exists(fn):
        os.unlink(fn)

    main(args=[
        '--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400',
        '--force-all', '--no-write', '--coadd-bw', '--unwise-dir',
        os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir',
        os.path.join(surveydir, 'images', 'unwise-tr'), '--blob-image',
        '--no-hybrid-psf', '--survey-dir', surveydir, '--outdir', outdir
    ] + extra_args + ['-v'])
    print('Checking for calib file', fn)
    assert (os.path.exists(fn))

    # Wrap-around, hybrid PSF
    surveydir = os.path.join(os.path.dirname(__file__), 'testcase8')
    outdir = 'out-testcase8'

    main(args=[
        '--brick',
        '1209p050',
        '--zoom',
        '720',
        '1095',
        '3220',
        '3500',
        '--force-all',
        '--no-write',
        '--no-wise',  #'--plots',
        '--survey-dir',
        surveydir,
        '--outdir',
        outdir
    ] + extra_args)

    # Test with a Tycho-2 star + another saturated star in the blob.

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase7')
    outdir = 'out-testcase7'
    # remove --no-gaia
    my_extra_args = [a for a in extra_args if a != '--no-gaia']
    os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia-dr2')
    os.environ['GAIA_CAT_VER'] = '2'
    main(args=[
        '--brick',
        '1102p240',
        '--zoom',
        '250',
        '350',
        '1550',
        '1650',
        '--force-all',
        '--no-write',
        '--no-wise',  #'--plots',
        '--survey-dir',
        surveydir,
        '--outdir',
        outdir
    ] + my_extra_args)
    del os.environ['GAIA_CAT_DIR']
    del os.environ['GAIA_CAT_VER']
    fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits')
    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 4)

    # Check skipping blobs outside the brick's unique area.
    # (this now doesn't detect any sources at all, reasonably)
    # surveydir = os.path.join(os.path.dirname(__file__), 'testcase5')
    # outdir = 'out-testcase5'
    #
    # fn = os.path.join(outdir, 'tractor', '186', 'tractor-1867p255.fits')
    # if os.path.exists(fn):
    #     os.unlink(fn)
    #
    # main(args=['--brick', '1867p255', '--zoom', '0', '150', '0', '150',
    #            '--force-all', '--no-write', '--coadd-bw',
    #            '--survey-dir', surveydir,
    #            '--early-coadds',
    #            '--outdir', outdir] + extra_args)
    #
    # assert(os.path.exists(fn))
    # T = fits_table(fn)
    # assert(len(T) == 1)

    # Custom RA,Dec; blob ra,dec.
    surveydir = os.path.join(os.path.dirname(__file__), 'testcase4')
    outdir = 'out-testcase4b'
    # Catalog written with one entry (--blobradec)
    fn = os.path.join(outdir, 'tractor', 'cus',
                      'tractor-custom-186743p25461.fits')
    if os.path.exists(fn):
        os.unlink(fn)
    main(args=[
        '--radec', '186.743965', '25.461788', '--width', '250', '--height',
        '250', '--force-all', '--no-write', '--no-wise', '--blobradec',
        '186.740369', '25.453855', '--survey-dir', surveydir, '--outdir',
        outdir
    ] + extra_args)

    assert (os.path.exists(fn))
    T = fits_table(fn)
    assert (len(T) == 1)

    surveydir = os.path.join(os.path.dirname(__file__), 'testcase3')
    outdir = 'out-testcase3'
    checkpoint_fn = os.path.join(outdir, 'checkpoint.pickle')
    if os.path.exists(checkpoint_fn):
        os.unlink(checkpoint_fn)
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', outdir, '--checkpoint', checkpoint_fn,
        '--checkpoint-period', '1', '--threads', '2'
    ] + extra_args)

    # Read catalog into Tractor sources to test read_fits_catalog
    from legacypipe.catalog import read_fits_catalog
    from legacypipe.survey import LegacySurveyData, GaiaSource
    from tractor.galaxy import DevGalaxy
    from tractor import PointSource

    survey = LegacySurveyData(survey_dir=outdir)
    fn = survey.find_file('tractor', brick='2447p120')
    print('Checking', fn)
    T = fits_table(fn)
    cat = read_fits_catalog(T, fluxPrefix='')
    print('Read catalog:', cat)
    assert (len(cat) == 2)
    src = cat[0]
    assert (type(src) == DevGalaxy)
    assert (np.abs(src.pos.ra - 244.77973) < 0.00001)
    assert (np.abs(src.pos.dec - 12.07233) < 0.00001)
    src = cat[1]
    print('Source', src)
    assert (type(src) in [PointSource, GaiaSource])
    assert (np.abs(src.pos.ra - 244.77830) < 0.00001)
    assert (np.abs(src.pos.dec - 12.07250) < 0.00001)
    # DevGalaxy(pos=RaDecPos[244.77975494973529, 12.072348111713127], brightness=NanoMaggies: g=19.2, r=17.9, z=17.1, shape=re=2.09234, e1=-0.198453, e2=0.023652,
    # PointSource(RaDecPos[244.77833280764278, 12.072521274981987], NanoMaggies: g=25, r=23, z=21.7)

    # Check that we can run again, using that checkpoint file.
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', outdir, '--checkpoint', checkpoint_fn,
        '--checkpoint-period', '1', '--threads', '2'
    ] + extra_args)
    # Assert...... something?

    # Test --checkpoint without --threads
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', outdir, '--checkpoint', checkpoint_fn,
        '--checkpoint-period', '1'
    ] + extra_args)

    # MzLS + BASS data
    # surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass')
    # main(args=['--brick', '3521p002', '--zoom', '2400', '2450', '1200', '1250',
    #            '--no-wise', '--force-all', '--no-write',
    #            '--survey-dir', surveydir2,
    #            '--outdir', 'out-mzlsbass'])

    # From Kaylan's Bootes pre-DR4 run
    # surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass3')
    # main(args=['--brick', '2173p350', '--zoom', '100', '200', '100', '200',
    #            '--no-wise', '--force-all', '--no-write',
    #            '--survey-dir', surveydir2,
    #            '--outdir', 'out-mzlsbass3'] + extra_args)

    # With plots!
    main(args=[
        '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
        '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir,
        '--outdir', 'out-testcase3', '--plots', '--nblobs', '1'
    ] + extra_args)

    # Decals Image Simulations
    # Uncomment WHEN galsim build for Travis
    #os.environ["DECALS_SIM_DIR"]= os.path.join(os.path.dirname(__file__),'image_sims')
    #brick= '2447p120'
    #sim_main(args=['--brick', brick, '-n', '2', '-o', 'STAR', \
    #               '-ic', '1', '--rmag-range', '18', '26', '--threads', '1',\
    #               '--zoom', '1020', '1070', '2775', '2815'])
    # Check if correct files written out
    #rt_dir= os.path.join(os.getenv('DECALS_SIM_DIR'),brick,'star','001')
    #assert( os.path.exists(os.path.join(rt_dir,'../','metacat-'+brick+'-star.fits')) )
    #for fn in ['tractor-%s-star-01.fits' % brick,'simcat-%s-star-01.fits' % brick]:
    #    assert( os.path.exists(os.path.join(rt_dir,fn)) )
    #for fn in ['image','model','resid','simscoadd']:
    #    assert( os.path.exists(os.path.join(rt_dir,'qa-'+brick+'-star-'+fn+'-01.jpg')) )

    if not travis:
        # With ceres
        main(args=[
            '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815',
            '--no-wise', '--force-all', '--no-write', '--ceres',
            '--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres'
        ] + extra_args)
Exemple #19
0
def main():

    max_radius = 4.
    rstep = 0.001
    radii = np.arange(rstep / 2., max_radius, rstep)

    # emcee
    # init_04 = ([25.35, 8.269, -25.22], [0.5045, 0.3587, 0.4013])
    # sersic = 0.4
    # target = ser_model(radii, sersic)
    # initamps,initvars = init_04
    # pars = np.append(initamps, initvars)
    # sample_sersic(radii, target, pars)
    # return

    ### Need to fit parameters for both N-component mixtures at the
    ### transition indices (plus some overlap!)

    #dser = 0.01
    #dser = 0.05
    #all_sersics = np.arange(0.6, 0.3-dser/2., -dser)
    #all_sersics = np.arange(1.0, 0.8-dser/2., -dser)
    #all_sersics = np.arange(0.85, 0.60-dser/2., -dser)
    #all_sersics = np.arange(0.75, 0.50-dser/2., -dser)
    #all_sersics = np.arange(0.4, 0.25-dser/2., -dser)
    # Approaching 0.25, |amps| become large -- exceeding 100

    all_sersics = []
    all_amps = []
    all_vars = []
    all_loglikes = []
    #all_logprobs = []

    sersics7 = np.array([6.1, 6.2, 6.3])
    init_7 = ([
        9.78462,
        5.92346,
        3.08624,
        1.37584,
        0.528399,
        0.17315,
        0.0471669,
        0.0111288,
    ], [
        1.52991,
        0.132595,
        0.0169103,
        0.00245378,
        0.000366884,
        5.27404e-05,
        6.76674e-06,
        6.00242e-07,
    ])

    sersics6 = np.array([6.0, 5.5, 5.0, 4.5, 4.0, 3.5, 3.0, 2.7])
    init_6 = ([
        0.0113067, 0.04898785, 0.18195408, 0.55939775, 1.46288372, 3.28556791,
        6.27896305, 9.86946446
    ], [
        6.07125356e-07, 7.02153046e-06, 5.60375312e-05, 3.98494081e-04,
        2.72853912e-03, 1.93601976e-02, 1.58544866e-01, 1.95149972e+00
    ])

    sersics4 = np.array([3.5, 3.4, 3.3, 3.2, 3.1, 3.0])
    init4 = [
        8.2659,
        5.31123,
        2.79165,
        1.16263,
        0.383863,
        0.0977581,
        0.0176055,
        0.00172958,
    ], [
        2.04792,
        0.283875,
        0.051887,
        0.00977245,
        0.0017372,
        0.000267685,
        3.11052e-05,
        1.88165e-06,
    ]
    n = [None] * 7
    priors4 = [
        n + [1e-1], n + [1e-2], n + [1e-3], n + [1e-4], n + [1e-5], n + [1e-6]
    ]

    sersics3 = np.array([3.2, 3.1, 3., 2.5, 2., 1.9, 1.8, 1.7, 1.6, 1.5])
    init3 = ([7.872, 5.073, 2.661, 1.112, 0.3659, 0.09262, 0.01655],
             [2.095, 0.3306, 0.06875, 0.01458, 0.002892, 0.0004967, 6.458e-05])
    n = [None] * 6
    priors3 = [None] * 5 + [
        n + [1e-4], n + [3e-4], n + [1e-5], n + [3e-5], n + [1e-6]
    ]

    sersics15 = np.array([1.55, 1.51, 1.5, 1.4, 1.3, 1.2, 1.1, 1.0])
    init15 = ([6.653, 4.537, 1.776, 0.5341, 0.121,
               0.01932], [1.83, 0.4273, 0.1187, 0.03255, 0.007875, 0.001491])

    # 5->6 ~ 0.70
    sersicsA = np.array(
        [1.0, 0.95, 0.9, 0.85, 0.8, 0.75, 0.74, 0.73, 0.72, 0.71])
    initA = ([6.0, 4.34, 1.18, 0.223, 0.0308,
              0.00235], [1.5, 0.461, 0.14, 0.0391, 0.00885, 0.0012])
    n = [None] * 5
    priorsA = [None] * 6 + [n + [1e-3], n + [3e-4], n + [1e-4], n + [3e-5]]

    # 4->5 ~ 0.62
    sersicsB = np.array([0.8, 0.75, 0.71, 0.7, 0.65, 0.64, 0.63, 0.62])
    initB = ([5.818, 4.1, 0.7976, 0.0947,
              0.006378], [1.272, 0.4728, 0.1475, 0.03664, 0.005635])
    priorsB = [None] * 4 + [
        [None, None, None, None, 0.01],
        [None, None, None, None, 0.003],
        [None, None, None, None, 0.001],
        [None, None, None, None, 0.0003],
    ]

    # 3->4 ~ 0.57
    #sersicsC = np.array([0.75, 0.7, 0.63, 0.62, 0.6, 0.58, 0.575, 0.57])
    #initC = ([ 6.06735, 3.75046, 0.485251, 0.0287147,  ],
    #         [ 1.12455, 0.455509, 0.12943, 0.0216252,  ])
    sersicsC = np.array([0.63, 0.62, 0.6, 0.58, 0.575, 0.57])
    initC = [
        6.24609,
        3.29044,
        0.320663,
        0.016228,
    ], [
        0.995781,
        0.471937,
        0.144342,
        0.0256225,
    ]
    priorsC = [None] * 3 + [
        [None, None, None, 1e-3],
        [None, None, None, 3e-4],
        [None, None, None, 1e-4],
    ]

    ### USE THIS 2->3 ~ 0.53
    sersicsC2 = np.array([0.57, 0.56, 0.55, 0.54, 0.53, 0.52, 0.515, 0.51])
    initC2 = [6.44901, 2.83301, 0.218781], [0.892218, 0.508317, 0.17374]
    priorsC2 = [None] * 4 + [[None, None, 0.1], [None, None, 0.01],
                             [None, None, 0.003], [None, None, 0.001]]

    dser = 0.01
    sersicsD = np.arange(0.61, 0.50 + dser / 2., -dser)
    init_055 = ([7.747, 1.589], [0.8195, 0.4178])

    sersicsE = np.array([0.5])
    init_05 = ([9.065], [0.7213])

    dser = 0.01
    sersicsF = np.arange(0.5 - dser, 0.4 - dser / 2., -dser)
    init_049 = ([9.24, -0.23], [0.71, 0.33])

    sersicsG = np.arange(0.42, 0.29 - dser / 2., -dser)
    init_04 = ([
        13.5581,
        0.433216,
        -5.44774,
    ], [
        0.575113,
        0.304652,
        0.385431,
    ])

    # Ramp between N=3 and N=2 around n~0.4
    sersicsFG = np.array([0.39, 0.40, 0.41, 0.42])
    initFG = ([
        15.4473,
        1.43828,
        -8.47514,
    ], [
        0.540383,
        0.28262,
        0.365116,
    ])
    priorsFG = [[None, 1., None], [None, 0.5, None], [None, 0.25, None],
                [None, 0.125, None]]

    # Ramp between N=4 and N=2, n~0.6
    sersicsCD = np.array([0.6, 0.59, 0.58, 0.57, 0.57,
                          0.57])  #, 0.56, 0.55, 0.54])
    initCD = ([
        6.39787,
        3.02775,
        0.256118,
        0.0121954,
    ], [
        0.942441,
        0.480798,
        0.152128,
        0.027722,
    ])
    priorsCD = [
        [None, None, None, 0.01],
        [None, None, None, 0.003],
        [None, None, None, 0.001],
        [None, None, None, 0.0003],
        [None, None, None, 0.0001],
        [None, None, None, 0.00003],
    ]
    # priorsCD = [ [None,None, 0.3, 0.01],
    #              [None,None, 0.1, 0.003],
    #              [None,None, 0.03, 0.001],
    #              [None,None, 0.01, 0.0003],
    #              [None,None, 0.003, 0.0001],
    #              [None,None, 0.001, 0.00003],
    #              [None,None, 0.0003, 0.00003],
    #              ]

    sersicsCD2 = np.array([0.57, 0.56, 0.55, 0.54])
    initCD2 = [7.82365, 1.63757, 0.0411003], [0.840165, 0.373703, 0.0649509]
    priorsCD2 = [
        [None, None, 0.3],
        [None, None, 0.1],
        [None, None, 0.03],
        [None, None, 0.01],
        # [None,None, 0.003],
        # [None,None, 0.001],
        # [None,None, 0.0003],
    ]

    from astrometry.util.plotutils import PlotSequence
    ps = PlotSequence('fit')

    for sersics, ini, priors, patch in [
            # (sersics7, init_7,   None, False),
            # (sersics6, init_6,   None, False),
        (sersics4, init4, priors4, False),
            #(sersics3, init3, priors3, False),
            #(sersics15, init15, None, False),
            #(sersicsA, initA,  priorsA, False),
            #(sersicsB, initB, priorsB, False),
            #(sersicsC, initC,  priorsC, False),
            # (sersicsD, init_055, None, False),
            # (sersicsE, init_05,  None, False),
            # (sersicsF, init_049, None, True),
            # (sersicsF, init_049, None, False),
            # (sersicsG, init_04, None, False),
            #(sersicsFG, initFG, priorsFG, False),
            #(sersicsC, init_07,  None, False),
            #(sersicsC2, initC2,  priorsC2, False),
            #(sersicsD, init_055, None, False),
            #(sersicsCD, initCD, priorsCD, False),
            #(sersicsCD2, initCD2, priorsCD2, False),
    ]:
        amps, vars, loglikes = fit_range(sersics,
                                         radii,
                                         ini,
                                         priors=priors,
                                         ps=ps)

        # amps2,vars2,loglikes2 = fit_range(reversed(sersics), radii,
        #                                   (amps[-1], vars[-1]))
        #
        # amps3,vars3,loglikes3 = fit_range(sersics, radii,
        #                                   (amps2[-1], vars2[-1]))
        #
        # amps4,vars4,loglikes4 = fit_range(reversed(sersics), radii,
        #                                   (amps3[-1], vars3[-1]))
        # amps,vars = amps4,vars4

        print('After fitting range forward:', sersics)
        #print('amps:', amps)
        #print('vars:', vars)
        print('log-likes:', loglikes)

        # print('After fitting range backwards:', sersics)
        # #print('amps:', amps2)
        # #print('vars:', vars2)
        # print('log-likes:', loglikes2)
        #
        # print('After fitting range forwards again:', sersics)
        # print('log-likes:', loglikes3)
        #
        # print('After fitting range backwards again:', sersics)
        # print('log-likes:', loglikes4)

        for sersic, fitamps, fitvars in reversed(list(zip(sersics, amps,
                                                          vars))):
            print('(%.2f' % sersic, ', [',
                  ''.join(['%g, ' % a for a in fitamps]), '], [',
                  ''.join(['%g, ' % v for v in fitvars]), ']),')

        # the one after 0.5: patch across 0.5
        if patch:
            amps_before = all_amps[-2]
            vars_before = all_vars[-2]

            amps_after = amps[0]
            vars_after = vars[0]

            amps_half = all_amps[-1]
            vars_half = all_vars[-1]
            assert (len(amps_before) == len(amps_after))

            #print('vars before:', vars_before)
            #print('vars after:', vars_after)
            #print('vars half:', vars_half)

            patchvars = np.append([vars_half[0]],
                                  np.sqrt(vars_before[1:] * vars_after[1:]))
            #print('Patch variances:', patchvars)

            patchamps = np.array([amps_half[0]] + [0.] * (len(amps_after) - 1))
            #print('Patch amps:', patchamps)

            #all_sersics.append(sersics[0])
            #all_amps.append(patchamps)
            #all_vars.append(patchvars)
            #all_loglikes.append(loglikes[0])

            # Replace 0.5 fit
            all_amps[-1] = patchamps
            all_vars[-1] = patchvars

        all_sersics.extend(sersics)
        all_amps.extend(amps)
        all_vars.extend(vars)
        all_loglikes.extend(loglikes)

    print()
    print('All fit results:')
    print()
    for sersic, fitamps, fitvars in reversed(
            list(zip(all_sersics, all_amps, all_vars))):
        print('(%.3g' % sersic, ', [', ''.join(['%g, ' % a for a in fitamps]),
              '], [', ''.join(['%g, ' % v for v in fitvars]), ']),')

    all_sersics = np.array(all_sersics)

    plt.figure(figsize=(12, 6))
    plt.subplot(1, 2, 1)
    #for i in range(3):
    for i in range(8):
        J, = np.nonzero([len(a) > i for a in all_amps])
        plt.plot(all_sersics[J], np.array([all_amps[j][i] for j in J]), 'o-')
    plt.xlabel('Sersic index')
    plt.ylabel('Mixture amplitudes')
    plt.axhline(0., color='k')
    plt.yscale('symlog', linthreshy=1e-3)

    plt.subplot(1, 2, 2)
    #for i in range(3):
    for i in range(8):
        J, = np.nonzero([len(a) > i for a in all_vars])
        plt.plot(all_sersics[J], np.array([all_vars[j][i] for j in J]), 'o-')
    plt.xlabel('Sersic index')
    plt.ylabel('Mixture variances')
    plt.yscale('log')
    plt.suptitle('Gaussian Mixture approximations to Sersic profiles')
    plt.savefig('/tmp/mix.png')

    plt.xscale('log')
    xt = [6, 5, 4, 3, 2, 1, 0.5, 0.4, 0.3]
    plt.xticks(xt, ['%g' % x for x in xt])
    plt.subplot(1, 2, 1)
    plt.xscale('log')
    plt.xticks(xt, ['%g' % x for x in xt])
    plt.savefig('/tmp/mix2.png')

    plt.clf()
    plt.subplot(1, 2, 1)
    for s, a, v in zip(all_sersics, all_amps, all_vars):
        plt.plot(s + np.zeros_like(a), a, '-')
    plt.xlabel('Sersic index')
    plt.ylabel('Mixture amplitudes')
    plt.axhline(0., color='k')
    plt.yscale('symlog', linthreshy=1e-3)
    plt.xscale('log')
    xt = [6, 5, 4, 3, 2, 1, 0.5, 0.4, 0.3]
    plt.xticks(xt, ['%g' % x for x in xt])
    plt.subplot(1, 2, 2)
    for s, a, v in zip(all_sersics, all_amps, all_vars):
        plt.plot(s + np.zeros_like(v), v, '.')
    plt.xlabel('Sersic index')
    plt.ylabel('Mixture variances')
    plt.yscale('log')
    plt.xscale('log')
    xt = [6, 5, 4, 3, 2, 1, 0.5, 0.4, 0.3]
    plt.xticks(xt, ['%g' % x for x in xt])
    plt.savefig('/tmp/mix3.png')

    return

    #all_sersics = np.arange(1.0, 0.25-dser/2, -dser)

    # (Re-)Initializations: iser -> (amps, vars)
    inits = dict([
        # 1.0 -- paper 1 values for lux, 6 components
        # (0, ([2.34853813e-03,   3.07995260e-02,   2.23364214e-01,
        #       1.17949102e+00,   4.33873750e+00,   5.99820770e+00],
        #      [1.20078965e-03,   8.84526493e-03,   3.91463084e-02,
        #       1.39976817e-01,   4.60962500e-01,   1.50159566e+00])),
        (np.argmin(np.abs(all_sersics - 1.0)),
         ([6.0, 4.34, 1.18, 0.223, 0.0308,
           0.00235], [1.5, 0.461, 0.14, 0.0391, 0.00885, 0.0012])),
        # 0.85 -- one amplitude goes negative.  Smallest amplitude ~ 1e-3,
        # variances reasonably spaced
        # 5 components
        (np.argmin(np.abs(all_sersics - 0.85)),
         ([5.81, 4.2, 0.946, 0.148,
           0.018], [1.35, 0.475, 0.152, 0.044, 0.0104])),
        # 0.6
        #(np.argmin(np.abs(all_sersics - 0.6)),
        # ([8.5, 1.15], [0.83, 0.25])),
        # Nothing wrong with the 5-component fit at 0.6; smallest component ~ 1e-3
        (np.argmin(np.abs(all_sersics - 0.6)),
         ([6.236, 3.169, 0.3216, 0.02589], [0.9646, 0.4898, 0.1714, 0.04615])),
        # Pattern changes at 0.55; all amps still positive, smallest ~ 1e-2.5
        (np.argmin(np.abs(all_sersics - 0.55)), ([6.522, 2.721, 0.1968],
                                                 [0.873, 0.5162, 0.1796])),

        # 0.5: one component only
        (np.argmin(np.abs(all_sersics - 0.5)), ([9.], [0.72])),
        # First one below 0.5: flip one amp negative.
        (np.argmin(np.abs(all_sersics - (0.5 - dser))), ([9.24,
                                                          -0.23], [0.71,
                                                                   0.33])),
        # Switch to 3 components
        #(np.argmin(np.abs(all_sersics - 0.37)),
        #    ([33.85, -35.89, 10.25], [0.46, 0.37, 0.31])),
        #([15, -15, 10], [0.5, 0.35, 0.3])),
        #(np.argmin(np.abs(all_sersics - 0.39)),
        # ([31.13, -31.29, 8.43], [0.47, 0.38, 0.32])),
        (np.argmin(np.abs(all_sersics - 0.40)), ([31.34, -31.92,
                                                  8.92], [0.48, 0.39, 0.34])),
        #(np.argmin(np.abs(all_sersics - 0.41)),
        # ([25.57, -25.94, 8.78], [0.50, 0.40, 0.36])),
        # They cross over!
        #(np.argmin(np.abs(all_sersics - 0.42)),
        # ([24.05, -18.77, 3.19], [0.52, 0.43, 0.38])),
    ])

    for iser, sersic in enumerate(all_sersics):
        #for iser,sersic in enumerate(np.append(all_sersics, list(reversed(all_sersics[:-1])))):
        #all_ser2.append(sersic)
        target = ser_model(radii, sersic)

        if iser in inits:
            a, v = inits[iser]
            initamps = np.array(a)
            initvars = np.array(v)
            print('Sersic index %.2f' % sersic,
                  ': resetting initialization to', initamps, initvars)

        print('Sersic %.2f' % sersic, 'initialized at amps',
              ', '.join(['%.4g' % a for a in initamps]), 'vars',
              ', '.join(['%.4g' % v for v in initvars]))
        pars = np.append(initamps, initvars)
        pars = optimize_mixture(pars, radii, target)
        fitamps = pars[:len(pars) // 2].copy()
        fitvars = pars[len(pars) // 2:].copy()
        all_amps.append(fitamps)
        all_vars.append(fitvars)
        #all_logprobs.append(mixture_logprob(pars, radii, target))
        all_loglikes.append(mixture_loglikelihood(pars, radii, target))

        initamps = fitamps
        initvars = fitvars
        print('Sersic %.2f' % sersic, 'amps',
              ', '.join(['%.4g' % a for a in fitamps]), 'vars',
              ', '.join(['%.4g' % v for v in fitvars]))
        #print('Sersic %.2f' % sersic, 'amps', ', '.join(['%g'%a for a in fitamps]), 'vars', ', '.join(['%g'%v for v in fitvars]))

    all_amps = np.array(all_amps)
    all_vars = np.array(all_vars)
    #all_logprobs = np.array(all_logprobs)
    all_loglikes = np.array(all_loglikes)

    plt.figure(figsize=(12, 6))
    plt.subplot(1, 2, 1)
    #for i in range(3):
    for i in range(6):
        J, = np.nonzero([len(a) > i for a in all_amps])
        plt.plot(all_sersics[J], np.array([all_amps[j][i] for j in J]), 'o-')
    plt.xlabel('Sersic index')
    plt.ylabel('Mixture amplitudes')
    plt.axhline(0., color='k')
    plt.yscale('symlog', linthreshy=1e-3)

    plt.subplot(1, 2, 2)
    #for i in range(3):
    for i in range(6):
        J, = np.nonzero([len(a) > i for a in all_vars])
        plt.plot(all_sersics[J], np.array([all_vars[j][i] for j in J]), 'o-')
    plt.xlabel('Sersic index')
    plt.ylabel('Mixture variances')
    plt.yscale('log')
    plt.suptitle('Gaussian Mixture approximations to Sersic profiles')
    plt.savefig('/tmp/mix.png')

    plt.clf()
    plt.plot(all_sersics, all_loglikes, 'ko-')
    plt.xlabel('Sersic index')
    plt.ylabel('Mixture log-likelihood')
    plt.suptitle('Gaussian Mixture approximations to Sersic profiles')
    mx = max(all_loglikes)
    plt.ylim(mx - 200, mx)
    plt.savefig('/tmp/loglike.png')

    plt.figure(figsize=(16, 4))

    xx = radii[radii <= 4]
    for i, (n, a, v) in enumerate(zip(all_sersics, all_amps, all_vars)):
        plt.clf()
        plot_one_row(n, a, v, xx)
        #plt.suptitle('Logprob %.2f' % lnp)
        plt.savefig('/tmp/ser-%02i.png' % i)
    p3 = plt.plot(dists, corrs_y, 'g.-')
    if medians:
        p4 = plt.plot(dists, rcorrs, 'b.--')
        p5 = plt.plot(dists, rcorrs_x, 'r.--')
        p6 = plt.plot(dists, rcorrs_y, 'g.--')
    plt.xlabel('Pixel offset')
    plt.ylabel('Correlation')
    plt.axhline(0, color='k', alpha=0.3)
    plt.legend([p1[0], p2[0], p3[0]], ['Diagonal', 'X', 'Y'],
               loc='upper right')

    return (dists, corrs, corrs_x, corrs_y, rcorrs, rcorrs_x, rcorrs_y, mads,
            mads_x, mads_y, mad_random)


ps = PlotSequence('noise')

# fitsgetext -i /global/homes/d/dstn/cosmo/staging/decam/DECam_Raw/20160107/c4d_160107_213046_fri.fits.fz -e 0 -e 1 -o flat.fits

# img = fitsio.read('flat.fits').astype(np.float32)
# # trim the edges
# img = img[100:-100, 100:-100]
# img -= np.median(img)

# Pixel correlations in a flat image:
if False:

    from obsbot.measure_raw import DECamMeasurer
    from obsbot.decam import DecamNominalCalibration

    nom = DecamNominalCalibration()
Exemple #21
0
    catfn = args[2]
    outfn = args[3]

    if os.path.exists(outfn):
        print 'Ouput file exists:', outfn
        sys.exit(0)

    zoomslice = None
    if opt.zoom is not None:
        (x0,x1,y0,y1) = opt.zoom
        zoomslice = (slice(y0,y1), slice(x0,x1))

    ps = None
    if opt.plots is not None:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence(opt.plots)

    T = exposure_metadata([filename], hdus=[hdu])
    print 'Metadata:'
    T.about()

    decals = Decals()
    im = DecamImage(decals, T[0])
    tim = im.get_tractor_image(slc=zoomslice, const2psf=True)
    print 'Got tim:', tim

    if catfn == 'DR1':
        margin = 20
        TT = []
        chipwcs = tim.subwcs
        bricks = bricks_touching_wcs(chipwcs, decals=decals)
            cmd = 'wget -O %s "%s"' % (fn, url)
            print(cmd)
            os.system(cmd)
        img = plt.imread(fn)

        M = 20
        img = img[M:-M, M:-M, :]
        
        plt.subplot(NR, NC, j+1)
        plt.imshow(img, interpolation='nearest', origin='lower')
        plt.xticks([])
        plt.yticks([])

    
if __name__ == '__main__':
    ps = PlotSequence('morph')

    from glob import glob
    from astrometry.util.fits import merge_tables, fits_table

    HST = fits_table('acs-gc.fits')
    print(len(HST), 'ACS sources')
    HST.cut(HST.imaging == 'COSMOS ')
    print(len(HST), 'in COSMOS')
    HST.about()

    for dirnm in ['cosmos-50-rex', 'cosmos-51-rex', 'cosmos-52-rex']:
        fns = glob(os.path.join(dirnm, 'metrics', '*', 'all-models-*.fits'))
        T = merge_tables([fits_table(fn) for fn in fns])
        print(len(T), 'sources')
        T.about()
Exemple #23
0
plt.savefig('diffrd.png')

plt.clf()
p1 = plt.plot(A.expnum, A.dx * 0.262, 'b.')
p2 = plt.plot(A.expnum, A.dy * 0.262, 'r.')
p3 = plt.plot(A.expnum, A.ra_offset, 'g.')
p4 = plt.plot(A.expnum, A.dec_offset, 'm.')
plt.legend([p1[0], p2[0], p3[0], p4[0]], ['dx', 'dy', 'dRA', 'dDec'],
           'lower right')
plt.xlabel('expnum')
plt.savefig('difft.png')

# Now, look at each copilot extension vs im16.
from astrometry.util.plotutils import PlotSequence

ps = PlotSequence('astrom')

C.affdx = C.affine[:, 2]
C.affdy = C.affine[:, 5]

refext = 'im16'
Cref = C[C.extension == refext]
print(len(Cref), 'im16 exposures')
ref_expnum = dict([(expnum, i) for i, expnum in enumerate(Cref.expnum)])

plt.clf()
p1 = plt.plot(C.expnum, C.affine[:, 3] - 1, 'b.')
p2 = plt.plot(C.expnum, C.affine[:, 4], 'r.')
plt.plot(Cref.expnum, Cref.affine[:, 4], 'k-')
p3 = plt.plot(C.expnum, C.affine[:, 6], 'c.')
p4 = plt.plot(C.expnum, C.affine[:, 7] - 1, 'm.')
Exemple #24
0
def main():
    import optparse
    from astrometry.util.plotutils import PlotSequence
    from astrometry.util.util import Tan

    parser = optparse.OptionParser(usage='%prog [options] incat.fits out.fits')
    parser.add_option('-r', '--ralo',  dest='ralo',  type=float,
                      help='Minimum RA')
    parser.add_option('-R', '--rahi',  dest='rahi',  type=float,
                      help='Maximum RA')
    parser.add_option('-d', '--declo', dest='declo', type=float,
                      help='Minimum Dec')
    parser.add_option('-D', '--dechi', dest='dechi', type=float,
                      help='Maximum Dec')

    parser.add_option('-b', '--band', dest='bands', action='append', type=int,
                      default=[], help='WISE band to photometer (default: 1,2)')

    parser.add_option('-u', '--unwise', dest='unwise_dir',
                      default='unwise-coadds',
                      help='Directory containing unWISE coadds')

    parser.add_option('--no-ceres', dest='ceres', action='store_false',
                      default=True,
                      help='Use scipy lsqr rather than Ceres Solver?')

    parser.add_option('--ceres-block', '-B', dest='ceresblock', type=int,
                      default=8,
                      help='Ceres image block size (default: %default)')

    parser.add_option('--plots', dest='plots',
                      default=False, action='store_true')
    parser.add_option('--save-fits', dest='save_fits',
                      default=False, action='store_true')

    # parser.add_option('--ellipses', action='store_true',
    #                  help='Assume catalog shapes are ellipse descriptions (not r,ab,phi)')

    # parser.add_option('--ra', help='Center RA')
    # parser.add_option('--dec', help='Center Dec')
    # parser.add_option('--width', help='Degrees width (in RA*cos(Dec))')
    # parser.add_option('--height', help='Degrees height (Dec)')
    opt, args = parser.parse_args()
    if len(args) != 2:
        parser.print_help()
        sys.exit(-1)

    if len(opt.bands) == 0:
        opt.bands = [1, 2]
    # Allow specifying bands like "123"
    bb = []
    for band in opt.bands:
        for s in str(band):
            bb.append(int(s))
    opt.bands = bb
    print('Bands', opt.bands)

    ps = None
    if opt.plots:
        ps = PlotSequence('unwise')

    infn, outfn = args

    T = fits_table(infn)
    print('Read', len(T), 'sources from', infn)
    if opt.declo is not None:
        T.cut(T.dec >= opt.declo)
    if opt.dechi is not None:
        T.cut(T.dec <= opt.dechi)

    # Let's be a bit smart about RA wrap-around.  Compute the 'center'
    # of the RA points, use the cross product against that to define
    # inequality (clockwise-of).
    r = np.deg2rad(T.ra)
    x = np.mean(np.cos(r))
    y = np.mean(np.sin(r))
    rr = np.hypot(x, y)
    x /= rr
    y /= rr
    midra = np.rad2deg(np.arctan2(y, x))
    midra += 360. * (midra < 0)
    xx = np.cos(r)
    yy = np.sin(r)
    T.cross = x * yy - y * xx
    minra = T.ra[np.argmin(T.cross)]
    maxra = T.ra[np.argmax(T.cross)]

    if opt.ralo is not None:
        r = np.deg2rad(opt.ralo)
        xx = np.cos(r)
        yy = np.sin(r)
        crosscut = x * yy - y * xx
        T.cut(T.cross >= crosscut)
        print('Cut to', len(T), 'with RA >', opt.ralo)

    if opt.rahi is not None:
        r = np.deg2rad(opt.rahi)
        xx = np.cos(r)
        yy = np.sin(r)
        crosscut = x * yy - y * xx
        T.cut(T.cross <= crosscut)
        print('Cut to', len(T), 'with RA <', opt.rahi)
    if opt.declo is None:
        opt.declo = T.dec.min()
    if opt.dechi is None:
        opt.dechi = T.dec.max()
    if opt.ralo is None:
        opt.ralo = T.ra[np.argmin(T.cross)]
    if opt.rahi is None:
        opt.rahi = T.ra[np.argmax(T.cross)]
    T.delete_column('cross')

    print('RA range:', opt.ralo, opt.rahi)
    print('Dec range:', opt.declo, opt.dechi)

    x = np.mean([np.cos(np.deg2rad(r)) for r in (opt.ralo, opt.rahi)])
    y = np.mean([np.sin(np.deg2rad(r)) for r in (opt.ralo, opt.rahi)])
    midra = np.rad2deg(np.arctan2(y, x))
    midra += 360. * (midra < 0)
    middec = (opt.declo + opt.dechi) / 2.

    print('RA,Dec center:', midra, middec)

    pixscale = 2.75 / 3600.
    H = (opt.dechi - opt.declo) / pixscale
    dra = 2. * min(np.abs(midra - opt.ralo), np.abs(midra - opt.rahi))
    W = dra * np.cos(np.deg2rad(middec)) / pixscale

    margin = 5
    W = int(W) + margin * 2
    H = int(H) + margin * 2
    print('W,H', W, H)
    targetwcs = Tan(midra, middec, (W + 1) / 2., (H + 1) / 2.,
                    -pixscale, 0., 0., pixscale, float(W), float(H))
    #print('Target WCS:', targetwcs)

    ra0, dec0 = targetwcs.pixelxy2radec(0.5, 0.5)
    ra1, dec1 = targetwcs.pixelxy2radec(W + 0.5, H + 0.5)
    roiradecbox = [ra0, ra1, dec0, dec1]
    #print('ROI RA,Dec box', roiradecbox)

    tiles = unwise_tiles_touching_wcs(targetwcs)
    print('Cut to', len(tiles), 'unWISE tiles')

    disable_galaxy_cache()

    cols = T.get_columns()
    all_ptsrcs = not('type' in cols)
    if not all_ptsrcs:
        assert('shapeexp' in cols)
        assert('shapedev' in cols)
        assert('fracdev' in cols)

    wanyband = 'w'

    print('Creating Tractor catalog...')
    cat = []
    for i, t in enumerate(T):
        pos = RaDecPos(t.ra, t.dec)
        flux = NanoMaggies(**{wanyband: 1.})
        if all_ptsrcs:
            cat.append(PointSource(pos, flux))
            continue

        tt = t.type.strip()
        if tt in ['PTSRC', 'STAR', 'S']:
            cat.append(PointSource(pos, flux))
        elif tt in ['EXP', 'E']:
            shape = EllipseE(*t.shapeexp)
            cat.append(ExpGalaxy(pos, flux, shape))
        elif tt in ['DEV', 'D']:
            shape = EllipseE(*t.shapedev)
            cat.append(DevGalaxy(pos, flux, shape))
        elif tt in ['COMP', 'C']:
            eshape = EllipseE(*t.shapeexp)
            dshape = EllipseE(*t.shapedev)
            cat.append(FixedCompositeGalaxy(pos, flux, t.fracdev,
                                            eshape, dshape))
        else:
            print('Did not understand row', i, 'of input catalog:')
            t.about()
            assert(False)

    W = unwise_forcedphot(cat, tiles, roiradecbox=roiradecbox,
                          bands=opt.bands, unwise_dir=opt.unwise_dir,
                          use_ceres=opt.ceres, ceres_block=opt.ceresblock,
                          save_fits=opt.save_fits, ps=ps)
    W.writeto(outfn)
Exemple #25
0
def run_one_brick(X):
    brick, ibrick, nbricks, plots, kwargs = X

    survey = LegacySurveyData()

    print()
    print()
    print('Brick', (ibrick + 1), 'of', nbricks, ':', brick.brickname)

    dirnm = os.path.join('depthcuts', brick.brickname[:3])
    outfn = os.path.join(dirnm, 'ccds-%s.fits' % brick.brickname)
    if os.path.exists(outfn):
        print('Exists:', outfn)
        return 0

    H, W = 3600, 3600
    pixscale = 0.262
    bands = ['g', 'r', 'z']

    # Get WCS object describing brick
    targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale)
    targetrd = np.array([
        targetwcs.pixelxy2radec(x, y)
        for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
    ])
    gitver = get_git_version()

    ccds = survey.ccds_touching_wcs(targetwcs)
    if ccds is None:
        print('No CCDs actually touching brick')
        return 0
    print(len(ccds), 'CCDs actually touching brick')

    ccds.cut(np.in1d(ccds.filter, bands))
    print('Cut on filter:', len(ccds), 'CCDs remain.')

    if 'ccd_cuts' in ccds.get_columns():
        norig = len(ccds)
        ccds.cut(ccds.ccd_cuts == 0)
        print(len(ccds), 'of', norig, 'CCDs pass cuts')
    else:
        print('No CCD cuts')

    if len(ccds) == 0:
        print('No CCDs left')
        return 0

    ps = None
    if plots:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence('depth-%s' % brick.brickname)

    splinesky = True
    gaussPsf = False
    pixPsf = True
    do_calibs = False
    normalizePsf = True

    get_depth_maps = kwargs.pop('get_depth_maps', False)

    try:
        D = make_depth_cut(survey,
                           ccds,
                           bands,
                           targetrd,
                           brick,
                           W,
                           H,
                           pixscale,
                           plots,
                           ps,
                           splinesky,
                           gaussPsf,
                           pixPsf,
                           normalizePsf,
                           do_calibs,
                           gitver,
                           targetwcs,
                           get_depth_maps=get_depth_maps,
                           **kwargs)
        if get_depth_maps:
            keep, overlapping, depthmaps = D
        else:
            keep, overlapping = D
    except:
        print('Failed to make_depth_cut():')
        import traceback
        traceback.print_exc()
        return -1

    print(np.sum(overlapping), 'CCDs overlap the brick')
    print(np.sum(keep), 'CCDs passed depth cut')
    ccds.overlapping = overlapping
    ccds.passed_depth_cut = keep

    if not os.path.exists(dirnm):
        try:
            os.makedirs(dirnm)
        except:
            pass

    if get_depth_maps:
        for band, depthmap in depthmaps:
            doutfn = os.path.join(dirnm,
                                  'depth-%s-%s.fits' % (brick.brickname, band))
            hdr = fitsio.FITSHDR()
            # Plug the WCS header cards into these images
            targetwcs.add_to_header(hdr)
            hdr.delete('IMAGEW')
            hdr.delete('IMAGEH')
            hdr.add_record(dict(name='EQUINOX', value=2000.))
            hdr.add_record(dict(name='FILTER', value=band))
            fitsio.write(doutfn, depthmap, header=hdr)
            print('Wrote', doutfn)

    tmpfn = os.path.join(os.path.dirname(outfn),
                         'tmp-' + os.path.basename(outfn))
    ccds.writeto(tmpfn)
    os.rename(tmpfn, outfn)
    print('Wrote', outfn)

    return 0
def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--plots', action='store_true')
    parser.add_argument('--brick', help='Brick name to run')
    parser.add_argument(
        '--input-dir',
        default='/global/projecta/projectdirs/cosmo/work/legacysurvey/dr7')
    #/global/cscratch1/sd/desiproc/dr7out')
    parser.add_argument('--survey-dir',
                        default='/global/cscratch1/sd/dstn/dr7-depthcut')
    parser.add_argument('--output-dir',
                        default='/global/cscratch1/sd/dstn/bright')
    opt = parser.parse_args()

    plots = opt.plots
    ps = PlotSequence('bright')
    brickname = opt.brick

    insurvey = LegacySurveyData(opt.input_dir, cache_dir=opt.survey_dir)
    outsurvey = LegacySurveyData(opt.output_dir, output_dir=opt.output_dir)

    bfn = insurvey.find_file('blobmap', brick=brickname)
    print('Found blob map', bfn)
    blobs = fitsio.read(bfn)
    h, w = blobs.shape

    brick = insurvey.get_brick_by_name(brickname)
    brickwcs = wcs_for_brick(brick)
    radius = np.sqrt(2.) * 0.25 * 1.01
    neighbors = insurvey.get_bricks_near(brick.ra, brick.dec, radius)
    print(len(neighbors), 'bricks nearby')

    def showbool(X):
        d = downsample_max(X, 8)
        h, w = X.shape
        plt.imshow(d,
                   interpolation='nearest',
                   origin='lower',
                   vmin=0,
                   vmax=1,
                   extent=[0, w, 0, h],
                   cmap='gray')

    brightblobs = set()

    for nb in neighbors:
        if nb.brickname == brickname:
            # ignore myself!
            continue
        print('Neighbor:', nb.brickname)

        mfn = insurvey.find_file('maskbits', brick=nb.brickname)
        if not os.path.exists(mfn):
            print('No maskbits file:', mfn)
            continue
        maskbits = fitsio.read(mfn)
        bright = ((maskbits & MASKBITS['BRIGHT']) > 0)
        print(np.sum(bright > 0), 'BRIGHT pixels set')
        primary = (maskbits & MASKBITS['NPRIMARY'] == 0)
        print(np.sum(primary), 'PRIMARY pixels set')
        edge = binary_dilation(primary, structure=np.ones((3, 3), bool))
        edge = edge * np.logical_not(primary)
        brightedge = edge & bright

        if plots:
            plt.clf()
            showbool(bright)
            plt.title('bright: brick %s' % nb.brickname)
            ps.savefig()

            # plt.clf()
            # showbool(primary)
            # plt.title('PRIMARY, brick %s' % nb.brickname)
            # ps.savefig()
            #
            # plt.clf()
            # showbool(edge)
            # plt.title('boundary, brick %s' % nb.brickname)
            # ps.savefig()

            plt.clf()
            showbool(brightedge)
            plt.title('bright at edge, brick %s' % nb.brickname)
            ps.savefig()

        nwcs = wcs_for_brick(nb)

        yy, xx = np.nonzero(brightedge)
        print(len(yy), 'bright edge pixels')
        if len(yy) == 0:
            continue
        rr, dd = nwcs.pixelxy2radec(xx + 1, yy + 1)
        print('RA range', rr.min(), rr.max(), 'vs brick', brick.ra1, brick.ra2)
        print('Dec range', dd.min(), dd.max(), 'vs brick', brick.dec1,
              brick.dec2)
        # Find pixels that are within this brick's unique area
        I, = np.nonzero((rr >= brick.ra1) * (rr <= brick.ra2) *
                        (dd >= brick.dec1) * (dd <= brick.dec2))

        if plots:
            plt.clf()
            plt.plot(
                [brick.ra1, brick.ra1, brick.ra2, brick.ra2, brick.ra1],
                [brick.dec1, brick.dec2, brick.dec2, brick.dec1, brick.dec1],
                'b-')
            plt.plot(rr, dd, 'k.')
            plt.plot(rr[I], dd[I], 'r.')
            plt.title('Bright pixels from %s' % nb.brickname)
            ps.savefig()

        if len(I) == 0:
            print('No edge pixels touch')
            #plt.plot(br,bd, 'b-')
            continue
        #print('Edge pixels touch!')
        #plt.plot(br,bd, 'r-', zorder=20)

        ok, x, y = brickwcs.radec2pixelxy(rr[I], dd[I])
        x = np.round(x).astype(int) - 1
        y = np.round(y).astype(int) - 1
        print('Pixel ranges X', x.min(), x.max(), 'Y', y.min(), y.max())
        assert (np.all((x >= 0) * (x < w) * (y >= 0) * (y < h)))
        print('Adding blobs:', np.unique(blobs[y, x]))
        brightblobs.update(blobs[y, x])
        print('Blobs touching bright pixels:', brightblobs)

    print()
    brightblobs.discard(-1)
    if len(brightblobs) == 0:
        print('No neighboring bright blobs to update!')
        return
    print('Updating', len(brightblobs), 'blobs:', brightblobs)

    tmap = np.zeros(blobs.max() + 2, bool)
    for b in brightblobs:
        tmap[b + 1] = True
    touching = tmap[blobs + 1]

    if plots:
        plt.clf()
        showbool(touching)
        plt.title('Blobs touching bright, brick %s' % brickname)
        ps.savefig()

    mfn = insurvey.find_file('maskbits', brick=brickname)
    maskbits, hdr = fitsio.read(mfn, header=True)
    updated = maskbits | (MASKBITS['BRIGHT'] * touching)
    if np.all(maskbits == updated):
        print('No bits updated!  (Bright stars were already masked)')
        return
    maskbits = updated

    if plots:
        plt.clf()
        showbool((maskbits & MASKBITS['BRIGHT']) > 0)
        plt.title('New maskbits map for BRIGHT, brick %s' % brickname)
        ps.savefig()

    with outsurvey.write_output('maskbits', brick=brickname) as out:
        out.fits.write(maskbits, hdr=hdr)

    tfn = insurvey.find_file('tractor', brick=brickname)
    phdr = fitsio.read_header(tfn, ext=0)
    hdr = fitsio.read_header(tfn, ext=1)
    T = fits_table(tfn)
    print('Read', len(T), 'sources')
    print('Bright:', Counter(T.brightstarinblob))
    iby = np.clip(np.round(T.by), 0, h - 1).astype(int)
    ibx = np.clip(np.round(T.bx), 0, w - 1).astype(int)
    if plots:
        before = np.flatnonzero(T.brightstarinblob)
    T.brightstarinblob |= touching[iby, ibx]
    print('Bright:', Counter(T.brightstarinblob))

    # yuck -- copy the TUNIT headers from input to output.
    units = [
        hdr.get('TUNIT%i' % (i + 1), '') for i in range(len(T.get_columns()))
    ]

    if plots:
        plt.clf()
        showbool((maskbits & MASKBITS['BRIGHT']) > 0)
        ax = plt.axis()
        after = np.flatnonzero(T.brightstarinblob)
        plt.plot(T.bx[before], T.by[before], 'gx')
        plt.plot(T.bx[after], T.by[after], 'r.')
        plt.axis(ax)
        plt.title('sources with brightstarinblob, brick %s' % brickname)
        ps.savefig()

    with outsurvey.write_output('tractor', brick=brickname) as out:
        T.writeto(None,
                  fits_object=out.fits,
                  primheader=phdr,
                  header=hdr,
                  units=units)
Exemple #27
0
def main():
    ps = PlotSequence('cov')

    survey = LegacySurveyData()

    ra, dec = 242.0, 10.2

    fn = 'coverage-ccds.fits'
    if not os.path.exists(fn):
        ccds = survey.get_ccds()
        ccds.cut(ccds.filter == 'r')
        ccds.cut(ccds.propid == '2014B-0404')
        ccds.cut(np.hypot(ccds.ra_bore - ra, ccds.dec_bore - dec) < 2.5)
        print(np.unique(ccds.expnum), 'unique exposures')
        print('propids', np.unique(ccds.propid))
        ccds.writeto(fn)
    else:
        ccds = fits_table(fn)

    plt.clf()
    for e in np.unique(ccds.expnum):
        I = np.flatnonzero(ccds.expnum == e)
        plt.plot(ccds.ra[I], ccds.dec[I], '.')
    ps.savefig()

    degw = 3.0
    pixscale = 10.

    W = degw * 3600 / 10.
    H = W

    hi = 6
    cmap = cmap_discretize('jet', hi + 1)

    wcs = Tan(ra, dec, W / 2. + 0.5, H / 2. + 0.5, -pixscale / 3600., 0., 0.,
              pixscale / 3600., float(W), float(H))

    r0, d0 = wcs.pixelxy2radec(1, 1)
    r1, d1 = wcs.pixelxy2radec(W, H)
    extent = [min(r0, r1), max(r0, r1), min(d0, d1), max(d0, d1)]

    for expnums in [
        [348666],
        [348666, 348710, 348686],
        [348659, 348667, 348658, 348666, 348665, 348669, 348668],
            None,
        [
            348683, 348687, 347333, 348686, 348685, 348692, 348694, 348659,
            348667, 348658, 348666, 348665, 348669, 348668, 348707, 348709,
            348708, 348710, 348711, 348716, 348717
        ],
    ]:

        nexp = np.zeros((H, W), np.uint8)

        for ccd in ccds:
            if expnums is not None and not ccd.expnum in expnums:
                continue

            ccdwcs = survey.get_approx_wcs(ccd)
            r, d = ccdwcs.pixelxy2radec(1, 1)
            ok, x0, y0 = wcs.radec2pixelxy(r, d)
            r, d = ccdwcs.pixelxy2radec(ccd.width, ccd.height)
            ok, x1, y1 = wcs.radec2pixelxy(r, d)
            xlo = np.clip(int(np.round(min(x0, x1))) - 1, 0, W - 1)
            xhi = np.clip(int(np.round(max(x0, x1))) - 1, 0, W - 1)
            ylo = np.clip(int(np.round(min(y0, y1))) - 1, 0, H - 1)
            yhi = np.clip(int(np.round(max(y0, y1))) - 1, 0, H - 1)
            nexp[ylo:yhi + 1, xlo:xhi + 1] += 1

        plt.clf()
        plt.imshow(nexp,
                   interpolation='nearest',
                   origin='lower',
                   vmin=-0.5,
                   vmax=hi + 0.5,
                   cmap=cmap,
                   extent=extent)
        plt.colorbar(ticks=np.arange(hi + 1))
        ps.savefig()

    O = fits_table('obstatus/decam-tiles_obstatus.fits')
    O.cut(np.hypot(O.ra - ra, O.dec - dec) < 2.5)

    for p in [1, 2, 3]:
        print('Pass', p, 'exposures:', O.r_expnum[O.get('pass') == p])

    O.cut(O.get('pass') == 2)
    print(len(O), 'pass 2 nearby')

    d = np.hypot(O.ra - ra, O.dec - dec)
    print('Dists:', d)

    I = np.flatnonzero(d < 0.5)
    assert (len(I) == 1)
    ocenter = O[I[0]]
    print('Center expnum', ocenter.r_expnum)

    I = np.flatnonzero(d >= 0.5)
    O.cut(I)

    #center = ccds[ccds.expnum == ocenter.r_expnum]
    #p2 = ccds[ccds.

    ok, xc, yc = wcs.radec2pixelxy(ocenter.ra, ocenter.dec)

    xx, yy = np.meshgrid(np.arange(W) + 1, np.arange(H) + 1)
    c_d2 = (xc - xx)**2 + (yc - yy)**2

    best = np.ones((H, W), bool)

    for o in O:
        ok, x, y = wcs.radec2pixelxy(o.ra, o.dec)
        d2 = (x - xx)**2 + (y - yy)**2
        best[d2 < c_d2] = False
        del d2

    del c_d2, xx, yy

    # plt.clf()
    # plt.imshow(best, interpolation='nearest', origin='lower', cmap='gray',
    #            vmin=0, vmax=1)
    # ps.savefig()

    plt.clf()
    plt.imshow(nexp * best,
               interpolation='nearest',
               origin='lower',
               vmin=-0.5,
               vmax=hi + 0.5,
               cmap=cmap,
               extent=extent)
    plt.colorbar(ticks=np.arange(hi + 1))
    ps.savefig()

    plt.clf()
    n, b, p = plt.hist(np.clip(nexp[best], 0, hi),
                       range=(-0.5, hi + 0.5),
                       bins=hi + 1)
    plt.xlim(-0.5, hi + 0.5)
    ps.savefig()

    print('b', b)
    print('n', n)
    print('fracs', np.array(n) / np.sum(n))

    print('pcts',
          ', '.join(['%.1f' % f for f in 100. * np.array(n) / np.sum(n)]))
Exemple #28
0
def main(survey=None, opt=None, args=None):
    '''Driver function for forced photometry of individual Legacy
    Survey images.
    '''
    if args is None:
        args = sys.argv[1:]
    print('forced_photom.py', ' '.join(args))

    if opt is None:
        parser = get_parser()
        opt = parser.parse_args(args)

    import logging
    if opt.verbose == 0:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG
    logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
    # tractor logging is *soooo* chatty
    logging.getLogger('tractor.engine').setLevel(lvl + 10)

    t0 = Time()
    if survey is None:
        survey = LegacySurveyData(survey_dir=opt.survey_dir,
                                  cache_dir=opt.cache_dir,
                                  output_dir=opt.out_dir)
    if opt.skip:
        if opt.out is not None:
            outfn = opt.out
        else:
            outfn = survey.find_file('forced',
                                     output=True,
                                     camera=opt.camera,
                                     expnum=opt.expnum)
        if os.path.exists(outfn):
            print('Ouput file exists:', outfn)
            return 0

    if opt.derivs and opt.agn:
        print('Sorry, can\'t do --derivs AND --agn')
        return -1

    if opt.out is None and opt.out_dir is None:
        print('Must supply either --out or --out-dir')
        return -1

    if opt.expnum is None and opt.out is None:
        print('If no --expnum is given, must supply --out filename')
        return -1

    if not opt.forced:
        opt.apphot = True

    zoomslice = None
    if opt.zoom is not None:
        (x0, x1, y0, y1) = opt.zoom
        zoomslice = (slice(y0, y1), slice(x0, x1))

    ps = None
    if opt.plots is not None:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence(opt.plots)

    # Cache CCDs files before the find_ccds call...
    # Copy required files into the cache?
    if opt.pre_cache:

        def copy_files_to_cache(fns):
            for fn in fns:
                cachefn = fn.replace(survey.survey_dir, survey.cache_dir)
                if not cachefn.startswith(survey.cache_dir):
                    print('Skipping', fn)
                    continue
                outdir = os.path.dirname(cachefn)
                trymakedirs(outdir)
                print('Copy', fn)
                print('  to', cachefn)
                shutil.copyfile(fn, cachefn)

        assert (survey.cache_dir is not None)
        fnset = set()
        fn = survey.find_file('bricks')
        fnset.add(fn)
        fns = survey.find_file('ccd-kds')
        fnset.update(fns)
        copy_files_to_cache(fnset)

    # Read metadata from survey-ccds.fits table
    ccds = survey.find_ccds(camera=opt.camera,
                            expnum=opt.expnum,
                            ccdname=opt.ccdname)
    print(len(ccds), 'with camera', opt.camera, 'and expnum', opt.expnum,
          'and ccdname', opt.ccdname)
    # sort CCDs
    ccds.cut(np.lexsort((ccds.ccdname, ccds.expnum, ccds.camera)))

    # If there is only one catalog survey_dir, we pass it to get_catalog_in_wcs
    # as the northern survey.
    catsurvey_north = survey
    catsurvey_south = None

    if opt.catalog_dir_north is not None:
        assert (opt.catalog_dir_south is not None)
        assert (opt.catalog_resolve_dec_ngc is not None)
        catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
        catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)
    elif opt.catalog_dir is not None:
        catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)

    # Copy required CCD & calib files into the cache?
    if opt.pre_cache:
        assert (survey.cache_dir is not None)
        fnset = set()
        for ccd in ccds:
            im = survey.get_image_object(ccd)
            for key in im.get_cacheable_filename_variables():
                fn = getattr(im, key)
                if fn is None or not (os.path.exists(fn)):
                    continue
                fnset.add(fn)
        copy_files_to_cache(fnset)

    args = []
    for ccd in ccds:
        args.append((survey, catsurvey_north, catsurvey_south,
                     opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))

    if opt.threads:
        from astrometry.util.multiproc import multiproc
        from astrometry.util.timingpool import TimingPool, TimingPoolMeas
        pool = TimingPool(opt.threads)
        poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
        Time.add_measurement(poolmeas)
        mp = multiproc(None, pool=pool)
        tm = Time()
        FF = mp.map(bounce_one_ccd, args)
        print('Multi-processing forced-phot:', Time() - tm)
        del mp
        Time.measurements.remove(poolmeas)
        del poolmeas
        pool.close()
        pool.join()
        del pool
    else:
        FF = map(bounce_one_ccd, args)

    FF = [F for F in FF if F is not None]
    if len(FF) == 0:
        print('No photometry results to write.')
        return 0
    # Keep only the first header
    _, version_hdr, _, _ = FF[0]
    # unpack results
    outlier_masks = [m for _, _, m, _ in FF]
    outlier_hdrs = [h for _, _, _, h in FF]
    FF = [F for F, _, _, _ in FF]
    F = merge_tables(FF)

    if len(ccds):
        version_hdr.delete('CPHDU')
        version_hdr.delete('CCDNAME')

    from legacypipe.utils import add_bits
    from legacypipe.bits import DQ_BITS
    add_bits(version_hdr, DQ_BITS, 'DQMASK', 'DQ', 'D')
    from legacyzpts.psfzpt_cuts import CCD_CUT_BITS
    add_bits(version_hdr, CCD_CUT_BITS, 'CCD_CUTS', 'CC', 'C')
    for i, ap in enumerate(apertures_arcsec):
        version_hdr.add_record(
            dict(name='APRAD%i' % i,
                 value=ap,
                 comment='(optical) Aperture radius, in arcsec'))

    unitmap = {
        'exptime': 'sec',
        'flux': 'nanomaggy',
        'flux_ivar': '1/nanomaggy^2',
        'apflux': 'nanomaggy',
        'apflux_ivar': '1/nanomaggy^2',
        'psfdepth': '1/nanomaggy^2',
        'galdepth': '1/nanomaggy^2',
        'sky': 'nanomaggy/arcsec^2',
        'psfsize': 'arcsec',
        'fwhm': 'pixels',
        'ccdrarms': 'arcsec',
        'ccddecrms': 'arcsec',
        'ra': 'deg',
        'dec': 'deg',
        'skyrms': 'counts/sec',
        'dra': 'arcsec',
        'ddec': 'arcsec',
        'dra_ivar': '1/arcsec^2',
        'ddec_ivar': '1/arcsec^2'
    }

    columns = F.get_columns()
    order = [
        'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
        'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'fwhm', 'ccd_cuts',
        'airmass', 'sky', 'skyrms', 'psfdepth', 'galdepth', 'ccdzpt',
        'ccdrarms', 'ccddecrms', 'ccdphrms', 'ra', 'dec', 'flux', 'flux_ivar',
        'fracflux', 'rchisq', 'fracmasked', 'fracin', 'apflux', 'apflux_ivar',
        'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar', 'ddec_ivar'
    ]
    columns = [c for c in order if c in columns]
    units = [unitmap.get(c, '') for c in columns]

    if opt.out is not None:
        outdir = os.path.dirname(opt.out)
        if len(outdir):
            trymakedirs(outdir)
        tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.out))
        fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
        F.writeto(tmpfn, units=units, append=True, columns=columns)
        os.rename(tmpfn, opt.out)
        print('Wrote', opt.out)
    else:
        with survey.write_output('forced',
                                 camera=opt.camera,
                                 expnum=opt.expnum) as out:
            F.writeto(None,
                      fits_object=out.fits,
                      primheader=version_hdr,
                      units=units,
                      columns=columns)
            print('Wrote', out.real_fn)

    if opt.outlier_mask is not None:
        # Add outlier bit meanings to the primary header
        version_hdr.add_record(
            dict(name='COMMENT', value='Outlier mask bit meanings'))
        version_hdr.add_record(
            dict(name='OUTL_POS',
                 value=1,
                 comment='Outlier mask bit for Positive outlier'))
        version_hdr.add_record(
            dict(name='OUTL_NEG',
                 value=2,
                 comment='Outlier mask bit for Negative outlier'))

    if opt.outlier_mask == 'default':
        outdir = os.path.join(opt.out_dir, 'outlier-masks')
        camexp = set(zip(ccds.camera, ccds.expnum))
        for c, e in camexp:
            I = np.flatnonzero((ccds.camera == c) * (ccds.expnum == e))
            ccd = ccds[I[0]]
            imfn = ccd.image_filename.strip()
            outfn = os.path.join(outdir, imfn.replace('.fits',
                                                      '-outlier.fits'))
            trymakedirs(outfn, dir=True)
            tempfn = outfn.replace('.fits', '-tmp.fits')
            with fitsio.FITS(tempfn, 'rw', clobber=True) as fits:
                fits.write(None, header=version_hdr)
                for i in I:
                    mask = outlier_masks[i]
                    _, _, _, meth, tile = survey.get_compression_args(
                        'outliers_mask', shape=mask.shape)
                    fits.write(mask,
                               header=outlier_hdrs[i],
                               extname=ccds.ccdname[i],
                               compress=meth,
                               tile_dims=tile)
            os.rename(tempfn, outfn)
            print('Wrote', outfn)
    elif opt.outlier_mask is not None:
        with fitsio.FITS(opt.outlier_mask, 'rw', clobber=True) as F:
            F.write(None, header=version_hdr)
            for i, (hdr, mask) in enumerate(zip(outlier_hdrs, outlier_masks)):
                _, _, _, meth, tile = survey.get_compression_args(
                    'outliers_mask', shape=mask.shape)
                F.write(mask,
                        header=hdr,
                        extname=ccds.ccdname[i],
                        compress=meth,
                        tile_dims=tile)
        print('Wrote', opt.outlier_mask)

    tnow = Time()
    print('Total:', tnow - t0)
    return 0
Exemple #29
0
def main(survey=None, opt=None):

    print(' '.join(sys.argv))
    '''Driver function for forced photometry of individual Legacy
    Survey images.
    '''
    if opt is None:
        parser = get_parser()
        opt = parser.parse_args()

    Time.add_measurement(MemMeas)
    t0 = tlast = Time()

    if opt.skip and os.path.exists(opt.outfn):
        print('Ouput file exists:', opt.outfn)
        sys.exit(0)

    if opt.derivs and opt.agn:
        print('Sorry, can\'t do --derivs AND --agn')
        sys.exit(0)

    if not opt.forced:
        opt.apphot = True

    zoomslice = None
    if opt.zoom is not None:
        (x0, x1, y0, y1) = opt.zoom
        zoomslice = (slice(y0, y1), slice(x0, x1))

    ps = None
    if opt.plots is not None:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence(opt.plots)

    # Try parsing first arg as exposure number (otherwise, it's a filename)
    try:
        expnum = int(opt.expnum)
        filename = None
    except:
        # make this 'None' for survey.find_ccds()
        expnum = None
        filename = opt.expnum

    # Try parsing HDU: "all" or HDU name or HDU number.
    all_hdus = (opt.ccdname == 'all')
    hdu = -1
    ccdname = None
    if not all_hdus:
        try:
            hdu = int(opt.ccdname)
        except:
            ccdname = opt.ccdname

    if survey is None:
        survey = LegacySurveyData(survey_dir=opt.survey_dir)

    catsurvey_north = survey
    catsurvey_south = None

    if opt.catalog_dir_north is not None:
        assert (opt.catalog_dir_south is not None)
        assert (opt.catalog_resolve_dec_ngc is not None)
        catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
        catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)

    if opt.catalog_dir is not None:
        catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)

    if filename is not None and hdu >= 0:
        # FIXME -- try looking up in CCDs file?
        # Read metadata from file
        print('Warning: faking metadata from file contents')
        T = exposure_metadata([filename], hdus=[hdu])
        print('Metadata:')
        T.about()

        if not 'ccdzpt' in T.columns():
            phdr = fitsio.read_header(filename)
            T.ccdzpt = np.array([phdr['MAGZERO']])
            print('WARNING: using header MAGZERO')
            T.ccdraoff = np.array([0.])
            T.ccddecoff = np.array([0.])
            print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.')

    else:
        # Read metadata from survey-ccds.fits table
        T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
        print(len(T), 'with expnum', expnum, 'and ccdname', ccdname)
        if hdu >= 0:
            T.cut(T.image_hdu == hdu)
            print(len(T), 'with HDU', hdu)
        if filename is not None:
            T.cut(np.array([f.strip() == filename for f in T.image_filename]))
            print(len(T), 'with filename', filename)
        if opt.camera is not None:
            T.cut(T.camera == opt.camera)
            print(len(T), 'with camera', opt.camera)
        if not all_hdus:
            assert (len(T) == 1)

    args = []
    for ccd in T:
        args.append((survey, catsurvey_north, catsurvey_south,
                     opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))

    if opt.threads:
        from astrometry.util.multiproc import multiproc
        from astrometry.util.timingpool import TimingPool, TimingPoolMeas
        pool = TimingPool(opt.threads)
        poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
        Time.add_measurement(poolmeas)
        mp = multiproc(None, pool=pool)
        tm = Time()
        FF = mp.map(bounce_one_ccd, args)
        print('Multi-processing forced-phot:', Time() - tm)
    else:
        FF = map(bounce_one_ccd, args)

    FF = [F for F in FF if F is not None]
    if len(FF) == 0:
        print('No photometry results to write.')
        return 0
    # Keep only the first header
    _, version_hdr = FF[0]
    FF = [F for F, hdr in FF]
    F = merge_tables(FF)

    if all_hdus:
        version_hdr.delete('CPHDU')
        version_hdr.delete('CCDNAME')

    units = {
        'exptime': 'sec',
        'flux': 'nanomaggy',
        'flux_ivar': '1/nanomaggy^2',
        'apflux': 'nanomaggy',
        'apflux_ivar': '1/nanomaggy^2',
        'psfdepth': '1/nanomaggy^2',
        'galdepth': '1/nanomaggy^2',
        'sky': 'nanomaggy/arcsec^2',
        'psfsize': 'arcsec'
    }
    if opt.derivs:
        units.update({
            'dra': 'arcsec',
            'ddec': 'arcsec',
            'dra_ivar': '1/arcsec^2',
            'ddec_ivar': '1/arcsec^2'
        })

    columns = F.get_columns()
    order = [
        'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
        'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'ccd_cuts',
        'airmass', 'sky', 'psfdepth', 'galdepth', 'ra', 'dec', 'flux',
        'flux_ivar', 'fracflux', 'rchisq', 'fracmasked', 'apflux',
        'apflux_ivar', 'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar',
        'ddec_ivar'
    ]
    columns = [c for c in order if c in columns]

    # Set units headers (must happen after column ordering is set!)
    hdr = fitsio.FITSHDR()
    for i, col in enumerate(columns):
        if col in units:
            hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))

    outdir = os.path.dirname(opt.outfn)
    if len(outdir):
        trymakedirs(outdir)
    tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.outfn))
    fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
    F.writeto(tmpfn, header=hdr, append=True, columns=columns)
    os.rename(tmpfn, opt.outfn)
    print('Wrote', opt.outfn)

    tnow = Time()
    print('Total:', tnow - t0)
    return 0
def main(passnum, threads):

    global udecs
    global P3
    global T
    global tilewcs
    global exps
    global bad_expids
    global tileid_to_depth

    ps = PlotSequence('covfill-p%i' % passnum)

    retirablefn = 'retirable-p%i.fits' % passnum
    depthsfn = 'all-depths-p%i.fits' % passnum

    if os.path.exists(retirablefn):
        R = fits_table(retirablefn)
        pcts = np.arange(0, 101)
        target = 22.5
        req_pcts = [0, 2, 2, 5, 5, 10, 10, 100]
        req_depths = [0, 0, target-0.6, target-0.6,
                      target-0.3, target-0.3, target, target]

        maglo, maghi = 21,23

        plt.clf()
        for depths in R.depths:
            plt.plot(pcts, np.clip(depths, maglo, maghi), 'b-',
                     alpha=0.1)
        plt.plot(req_pcts, np.clip(req_depths, maglo, maghi), 'k-',
                 lw=2, alpha=0.5)
        plt.ylim(maglo, maghi)
        plt.xlim(0, 100)
        plt.xlabel('Coverage fraction')
        plt.ylabel('Existing depth')
        plt.suptitle('MzLS: retirable pass-%i tiles: %i' % (passnum,len(R)))
        ps.savefig()

        # Where are they on the sky?
        T = fits_table('obstatus/mosaic-tiles_obstatus.fits')

        T.cut(T.in_desi == 1)
        T.cut(T.get('pass') <= 3)

        tileid_to_index = np.zeros(T.tileid.max()+1, int)
        tileid_to_index[T.tileid] = np.arange(len(T))
        R.ra  = T.ra [tileid_to_index[R.tileid]]
        R.dec = T.dec[tileid_to_index[R.tileid]]
        
        plt.clf()
        plt.plot(T.ra, T.dec, 'k.', alpha=0.02)
        I = (T.z_done == 1)
        plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.1)
        plt.plot(R.ra, R.dec, 'b.')
        ax = [310,80,30,85]
        #xl,xh = plt.xlim()
        #plt.xlim(xh,xl)
        plt.xlabel('RA (deg)')
        plt.ylabel('Dec (deg)')
        plt.title('MzLS: retirable pass-%i tiles' % passnum)
        plt.axis(ax)
        ps.savefig()

        for p in [1,2,3]:
            plt.clf()
            plt.plot(T.ra, T.dec, 'k.', alpha=0.02)
            #plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.1)
            I = np.flatnonzero((T.get('pass') == p) * (T.z_done == 1)
                               * (T.z_depth > 1) * (T.z_depth < 30))
            plt.scatter(T.ra[I], T.dec[I], c=T.z_depth[I],
                        vmin=20, vmax=23, s=4)
            I = np.flatnonzero((T.get('pass') == p) * (T.z_done == 1)
                               * (T.z_depth == 30))
            plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.5)
            plt.colorbar()
            plt.title('MzLS: Finished tiles in pass %i' % p)
            plt.axis(ax)
            ps.savefig()
            
        sys.exit(0)
    
    # NERSC: export LEGACY_SURVEY_DIR=/global/cscratch1/sd/dstn/dr4plus
    # (dstn laptop: export LEGACY_SURVEY_DIR=~/legacypipe-dir-mzls/)

    survey = LegacySurveyData()
    ccds = survey.get_annotated_ccds()
    print('Annotated CCDs:', len(ccds))
    ccds.cut(ccds.camera == 'mosaic')
    print(len(ccds), 'Mosaic')
    print('Unique exposures:', len(np.unique(ccds.expnum)))

    ccds.cut(ccds.exptime > 60)
    print('Exptime > 60 sec:', len(ccds))
    
    nccds = Counter(ccds.expnum)
    for k,v in nccds.most_common():
        if v <= 4:
            break
        print('Expnum', k, 'appears', v, 'times')
    print('Tile pass numbers:', Counter(ccds.tilepass).most_common())

    # Fix parsing of OBJECT field to tileid...
    from obsbot import get_tile_id_from_name
    tileids = []
    for o in ccds.object:
        tid = get_tile_id_from_name(o.strip())
        if tid is None:
            tid = 0
        tileids.append(tid)
    tileids = np.array(tileids)
    print(len(np.unique(tileids)), 'unique tile ids in annotated file, from OBJECT')
    print(len(np.unique(ccds.tileid)), 'unique tile ids in ann file from TILEID')
    D = np.flatnonzero(tileids != ccds.tileid)
    print(len(D), 'different tileids')
    print('From OBJECT:', tileids[D])
    print('From TILEID:', ccds.tileid[D])
    ccds.tileid = tileids

    T = fits_table('obstatus/mosaic-tiles_obstatus.fits')

    f = open('obstatus/bad_expid.txt')
    bad_expids = []
    for line in f:
        line = line.strip()
        if len(line) == 0:
            continue
        if line[0] == '#':
            continue
        words = line.split()
        try:
            expnum = int(words[0])
        except:
            print('Skipping line:', line)
            continue
        bad_expids.append(expnum)
    print('Read', len(bad_expids), 'bad exposure numbers')

    # Update ccds.tilepass from ccds.tileid
    tileidtopass = dict(zip(T.tileid, T.get('pass')))
    tileidtoebv = dict(zip(T.tileid, T.ebv_med))
    ccds.tilepass = np.array([tileidtopass.get(tileid, 0)
                              for tileid in ccds.tileid])
    ccds.tileebv = np.array([tileidtoebv.get(tileid, 0)
                             for tileid in ccds.tileid])
    print('Tile pass numbers after update:', Counter(ccds.tilepass).most_common())
    e,I = np.unique(ccds.expnum, return_index=True)
    exps = ccds[I]
    #print('Object names,exptimes for tilepass==0:', zip(exps.object[exps.tilepass == 0], exps.exptime[exps.tilepass == 0]))

    # Average the depth per exposure
    for j,expnum in enumerate(exps.expnum):
        I = np.flatnonzero(ccds.expnum == expnum)
        if len(I) != 4:
            print('Exposure', expnum, 'has', len(I), 'CCD entries')
            continue
        # Don't include zeros in computing average depths!
        Igood = I[(ccds.galdepth[I] > 0) * (ccds.ccdzpt[I] < 30)]
        if len(Igood) > 0:
            exps.galdepth[j] = np.mean(ccds.galdepth[Igood])
        else:
            exps.galdepth[j] = 0.

    # CCDs-table-based mapping from tileid to depth.
    I = np.flatnonzero((exps.tilepass > 0) * (exps.galdepth > 0) *
                       (exps.tileid > 0))
    tileid_to_depth = dict(zip(exps.tileid[I], exps.galdepth[I]))

    
    T.cut(T.in_desi == 1)
    T.cut(T.get('pass') <= 3)
    
    # The tiles we'll examine
    P3 = T[T.get('pass') == passnum]
    print(len(P3), 'pass', passnum, 'and in DESI')
    todo = P3[P3.z_done == 0]
    print(len(todo), 'pass', passnum, 'tiles to do (Z_DONE=0)')

    # Tiles with measured depths
    T.cut((T.z_depth > 15) * (T.z_depth < 30))
    print(len(T), 'tiles with measured depths')
    # Passes other than 3... they ~ only barely overlap, so don't
    # contribute significant depth.
    #T.cut(T.get('pass') < 3)

    udecs = np.unique(P3.dec)
    print(len(udecs), 'unique Dec values in pass', passnum)

    # Grab an arbitrary weight-map image and use that as a proxy!
    wtfn = 'k4m_170501_112501_oow_zd_v1.fits.fz'
    F = fitsio.FITS(wtfn)
    tilewcs = []

    # Read the WCS headers for each chip.
    # They make the CRVAL be the boresight for all 4 chips... perfect!
    # (because this means we can just set CRVAL = RA,Dec to shift the WCSes)
    for i in range(1, len(F)):
        hdr = F[i].read_header()
        wcs = wcs_pv2sip_hdr(hdr)
        tilewcs.append(wcs)

    mp = multiproc(threads)

    #args = [(i,t,udecs,P3,T,tilewcs,exps,bad_expids,tileid_to_depth)
    args = [(i,t)
            for i,t in enumerate(todo)]
    thedepths = mp.map(one_tile, args)

    alldepths = []
    retirable = []
    for arg,depths in zip(args, thedepths):
        if depths is None:
            continue
        itile,tile = arg[:2]
        target = 22.5
        req_pcts = [0, 2, 2, 5, 5, 10, 10, 100]
        req_depths = [0, 0, target-0.6, target-0.6,
                      target-0.3, target-0.3, target, target]

        print('  Depths at 2, 5, and 10th percentile vs target:',
              '%.2f' % (depths[2]  - (target - 0.6)),
              '%.2f' % (depths[5]  - (target - 0.3)),
              '%.2f' % (depths[10] -  target))

        alldepths.append((tile.tileid, depths))
        
        if not ((depths[2]  > target - 0.6) and
                (depths[5]  > target - 0.3) and
                (depths[10] > target)):
            continue

        retirable.append((tile.tileid, depths))

        #if len(retirable) == 10:
        #    break
        # if ps.ploti >= 100:
        #     continue
        # 
        # maglo, maghi = 21,23
        # 
        # plt.clf()
        # #plt.subplot(1,2,1)
        # plt.subplot2grid((2,2), (0,0))
        # plt.imshow(depth, interpolation='nearest', origin='lower',
        #            vmin=maglo, vmax=maghi)
        # plt.colorbar(ticks=[np.arange(maglo, maghi+0.01, 0.5)])
        # plt.xticks([]); plt.yticks([])
        # #plt.subplot(1,2,2)
        # plt.subplot2grid((2,2), (1,0))
        # plt.imshow(nexp, interpolation='nearest', origin='lower',
        #            vmin=0, vmax=4)
        # plt.colorbar(ticks=[0,1,2,3,4])
        # plt.xticks([]); plt.yticks([])
        # 
        # ax = plt.subplot2grid((2,2), (0,1), rowspan=2)
        # plt.plot(req_pcts, np.clip(req_depths, maglo, maghi), 'k-',
        #          lw=2, alpha=0.5)
        # plt.plot(pcts, np.clip(depths, maglo, maghi), 'b-')
        # ax.yaxis.tick_right()
        # plt.ylim(maglo, maghi)
        # plt.xlim(0, 100)
        # plt.xlabel('Coverage fraction')
        # plt.ylabel('Existing depth')
        # plt.suptitle('Tile %i' % tile.tileid)
        # ps.savefig()

        #if ps.ploti == 100:
        #    break
        
    # print('Tiles that could be retired:')
    # print('# Tileid 0th-percentile-extcorr-depth 1st-pctile 2nd-pctile ...')
    # for tileid, depths in retirable:
    #     print(tileid, ' '.join(['%.3f' % d for d in depths]))

    R = fits_table()
    R.tileid = np.array ([t for t,d in alldepths])
    R.depths = np.vstack([d for t,d in alldepths])
    R.writeto(depthsfn)

    if len(retirable):
        R = fits_table()
        R.tileid = np.array([t for t,d in retirable])
        R.depths = np.vstack([d for t,d in retirable])
        R.writeto(retirablefn)
    else:
        print('No tiles in pass', passnum, 'are retirable')