Example #1
0
def compare_brick_to_ps1(brickname, ps, name='', basedir=''):
    decals = Decals()
    brick = decals.get_brick_by_name(brickname)
    wcs = wcs_for_brick(brick)

    magrange = (15,20)
    ps1 = ps1cat(ccdwcs=wcs)
    ps1 = ps1.get_stars(magrange=magrange)
    print 'Got', len(ps1), 'PS1 stars'

    T = fits_table(os.path.join(basedir, 'tractor', brickname[:3],
                                'tractor-%s.fits' % brickname))
    I,J,d = match_radec(T.ra, T.dec, ps1.ra, ps1.dec, 1./3600.)
    print 'Matched', len(I), 'stars to PS1'

    T.cut(I)
    ps1.cut(J)

    bands = 'z'
    ap = 5    

    allbands = 'ugrizY'
    mags = np.arange(magrange[0], 1+magrange[1])
    
    for band in bands:
        iband = allbands.index(band)
        piband = ps1cat.ps1band[band]
        T.flux = T.decam_flux[:,iband]
        T.mag = NanoMaggies.nanomaggiesToMag(T.flux)
        print 'apflux shape', T.decam_apflux.shape
        
        T.apflux = T.decam_apflux[:, iband, ap]
        T.apmag = NanoMaggies.nanomaggiesToMag(T.apflux)

        ps1mag = ps1.median[:,piband]

        plt.clf()

        for cc,mag,label in [('b', T.mag, 'Mag'), ('r', T.apmag, 'Aper mag')]:
            plt.plot(ps1mag, mag - ps1mag, '.', color=cc, label=label, alpha=0.6)

            mm,dd = [],[]
            for mlo,mhi in zip(mags, mags[1:]):
                I = np.flatnonzero((ps1mag > mlo) * (ps1mag <= mhi))
                mm.append((mlo+mhi)/2.)
                dd.append(np.median(mag[I] - ps1mag[I]))
            plt.plot(mm, dd, 'o-', color=cc)
            
        plt.xlabel('PS1 %s mag' % band)
        plt.ylabel('Mag - PS1 (mag)')
        plt.title('%sPS1 comparison: brick %s' % (name, brickname))
        plt.ylim(-0.2, 0.2)
        mlo,mhi = magrange
        plt.xlim(mhi, mlo)
        plt.axhline(0., color='k', alpha=0.1)
        plt.legend()
        ps.savefig()
Example #2
0
 def set_mags(self, cat):
     '''adds columns to fits_table cat'''
     # Remove white spaces
     cat.set('type', np.char.strip(cat.get('type')))
     # AB mags
     # Two kinds, as observed (including dust) and instrinsic (dust removed)
     for whichmag in ['wdust', 'nodust']:
         # DECam
         for band in ['g', 'r', 'z', 'w1', 'w2']:
             if whichmag == 'wdust':
                 flux = cat.get('flux_%s' % band)
                 flux_ivar = cat.get('flux_ivar_%s' % band)
             elif whichmag == 'nodust':
                 flux = cat.get('flux_%s' % band) / cat.get(
                     'mw_transmission_%s' % band)
                 flux_ivar= cat.get('flux_ivar_%s' % band)*\
                             np.power(cat.get('mw_transmission_%s' % band),2)
             else:
                 raise ValueError()
             mag, mag_err = NanoMaggies.fluxErrorsToMagErrors(
                 flux, flux_ivar)
             cat.set('mag_%s_%s' % (whichmag, band), mag)
             cat.set('mag_ivar_%s_%s' % (whichmag, band),
                     1. / np.power(mag_err, 2))
     # Instrinsic fluxes
     whichmag = 'nodust'
     # DECam
     for band in ['g', 'r', 'z', 'w1', 'w2']:
         flux = cat.get('flux_%s' % band) / cat.get(
             'mw_transmission_%s' % band)
         flux_ivar= cat.get('flux_ivar_%s' % band)*\
                                 np.power(cat.get('mw_transmission_%s' % band),2)
     cat.set('flux_%s_%s' % (whichmag, band), flux)
     cat.set('flux_ivar_%s_%s' % (whichmag, band), flux_ivar)
Example #3
0
    def __init__(self, hdr, bandname=None, scale=1.):
        import numpy as np
        from tractor.brightness import NanoMaggies

        if hdr is not None:
            self.exptime = hdr['EXPTIME']
            self.phot_c = hdr['PHOT_C']
            self.phot_k = hdr['PHOT_K']
            self.airmass = hdr['AIRMASS']
            print('CFHT photometry:', self.exptime, self.phot_c, self.phot_k,
                  self.airmass)

            zpt = (2.5 * np.log10(self.exptime) + self.phot_c + self.phot_k *
                   (self.airmass - 1))
            print('-> zeropoint', zpt)
            scale = NanoMaggies.zeropointToScale(zpt)
            print('-> scale', scale)

        super(CfhtLinearPhotoCal, self).__init__(scale, band=bandname)
Example #4
0
 def set_mags_OldDataModel(self, cat):
     '''for tractor catalogues with columns like decam_flux.shape(many,6)
     adds columns to fits_table cat'''
     # Remove white spaces
     cat.set('type', np.char.strip(cat.get('type')))
     # AB mags
     # Two kinds, as observed (including dust) and instrinsic (dust removed)
     for whichmag in ['wdust', 'nodust']:
         # DECam
         shp = cat.get('decam_flux').shape
         mag, mag_err = np.zeros(shp), np.zeros(shp)
         for iband in range(shp[1]):
             if whichmag == 'wdust':
                 flux = cat.get('decam_flux')[:, iband]
                 flux_ivar = cat.get('decam_flux_ivar')[:, iband]
             elif whichmag == 'nodust':
                 flux = cat.get('decam_flux')[:, iband] / cat.get(
                     'decam_mw_transmission')[:, iband]
                 flux_ivar= cat.get('decam_flux_ivar')[:,iband]*\
                             np.power(cat.get('decam_mw_transmission')[:,iband],2)
             else:
                 raise ValueError()
             mag[:,
                 iband], mag_err[:,
                                 iband] = NanoMaggies.fluxErrorsToMagErrors(
                                     flux, flux_ivar)
         cat.set('decam_mag_%s' % whichmag, mag)
         cat.set('decam_mag_ivar_%s' % whichmag, 1. / np.power(mag_err, 2))
         # WISE
         if 'wise_flux' in cat.get_columns():
             shp = cat.get('wise_flux').shape
             mag, mag_err = np.zeros(shp), np.zeros(shp)
             for iband in range(shp[1]):
                 if whichmag == 'wdust':
                     flux = cat.get('wise_flux')[:, iband]
                     flux_ivar = cat.get('wise_flux_ivar')[:, iband]
                 elif whichmag == 'nodust':
                     flux = cat.get('wise_flux')[:, iband] / cat.get(
                         'wise_mw_transmission')[:, iband]
                     flux_ivar= cat.get('wise_flux_ivar')[:,iband]*\
                                 np.power(cat.get('wise_mw_transmission')[:,iband],2)
                 mag[:,
                     iband], mag_err[:,
                                     iband] = NanoMaggies.fluxErrorsToMagErrors(
                                         flux, flux_ivar)
             cat.set('wise_mag_%s' % whichmag, mag)
             cat.set('wise_mag_ivar_%s' % whichmag,
                     1. / np.power(mag_err, 2))
     # Instrinsic fluxes
     whichmag = 'nodust'
     # DECam
     shp = cat.get('decam_flux').shape
     flux, flux_ivar = np.zeros(shp), np.zeros(shp)
     for iband in range(shp[1]):
         flux[:, iband] = cat.get('decam_flux')[:, iband] / cat.get(
             'decam_mw_transmission')[:, iband]
         flux_ivar[:,iband]= cat.get('decam_flux_ivar')[:,iband]*\
                                 np.power(cat.get('decam_mw_transmission')[:,iband],2)
     cat.set('decam_flux_%s' % whichmag, flux)
     cat.set('decam_flux_ivar_%s' % whichmag, flux_ivar)
     # WISE
     if 'wise_flux' in cat.get_columns():
         shp = cat.get('wise_flux').shape
         flux, flux_err = np.zeros(shp), np.zeros(shp)
         for iband in range(shp[1]):
             flux[:, iband] = cat.get('wise_flux')[:, iband] / cat.get(
                 'wise_mw_transmission')[:, iband]
             flux_ivar[:,iband]= cat.get('wise_flux_ivar')[:,iband]*\
                                     np.power(cat.get('wise_mw_transmission')[:,iband],2)
         cat.set('wise_flux_%s' % whichmag, flux)
         cat.set('wise_flux_ivar_%s' % whichmag, flux_ivar)
Example #5
0
    
    plt.figure(figsize=(10,10))
    plt.subplots_adjust(left=0.05, right=0.95, bottom=0.05, top=0.95,
                        hspace=0, wspace=0)
    
    fn = os.path.join('%s/tractor/%s/tractor-%s.fits' %
                      (base, brick[:3], brick))
    print 'Reading', fn
    T = fits_table(fn)
    print len(T), 'sources'
    jpeg = os.path.join('%s/coadd/%s/%s/decals-%s-image.jpg' %
                        (base, brick[:3], brick, brick))
    img = plt.imread(jpeg)
    img = np.flipud(img)

    mags = NanoMaggies.nanomaggiesToMag(T.decam_flux)
    mags[np.logical_not(np.isfinite(mags))] = 99.
    
    T.g = mags[:,1]
    T.r = mags[:,2]
    T.z = mags[:,4]

    # Convert 
    I = np.flatnonzero((T.r < 15) * (T.type == 'SIMP'))
    print 'Converting', len(I), 'bright SIMP objects into PSFs'
    T.type[I] = 'PSF '
    

    if False:
        P = T[T.type == 'PSF ']
        print len(P), 'PSFs'
Example #6
0
def gim2d_catalog(cat, band):
    '''
    http://irsa.ipac.caltech.edu/data/COSMOS/tables/morphology/cosmos_morph_zurich_colDescriptions.html
    '''
    '''
    ACS_MU_CLASS 	float 	  	Type of object.
    1 = galaxy
    2 = star
    3 = spurious
    '''
    '''
    ACS_CLEAN 	float 	  	Object useable flag.
    0 = do not use this object
    1 = use this object
    '''
    '''
    FLUX_GIM2D 	float 	counts 	GIM2D total flux
    R_GIM2D 	float 	arcseconds 	GIM2D psf-convolved half-light radius of object
    ELL_GIM2D 	float 	  	GIM2D ellipticity = 1-b/a of object
    PA_GIM2D 	float 	degrees 	GIM2D position angle of object - cw from +y-axis
    DX_GIM2D 	float 	arcseconds 	x-offset of GIM2D-model center from ACS-coordinate center
    DY_GIM2D 	float 	arcseconds 	y-offset of GIM2D-model center from ACS-coordinate center
    SERSIC_N_GIM2D 	float 	  	GIM2D Sersic index
    R_0P5_GIM2D 	float 	arcseconds 	GIM2D half-light radius of object without PSF convolution

    TYPE 	float 	  	ZEST Type CLASS
    1 = Early type
    2 = Disk
    3 = Irregular Galaxy
    9 = no classification
    '''
    '''
    BULG 	float 	  	ZEST "Bulgeness" CLASS - only for Type 2 (disk) galaxies.
    0 = bulge dominated galaxy
    1,2 = intermediate-bulge galaxies
    3 = pure disk galaxy
    9 = no classification
    '''
    '''
    STELLARITY 	float 	  	Visual Stellarity flag.

    0 if ACS_CLASS_STAR<0.6 (object is ASSUMED to be a galaxy; no visual inspection)
    0 if ACS_CLASS_STAR>=0.6 AND object visually identified as a galaxy.
    1 if ACS_CLASS_STAR>=0.6 AND visually identified as a star.
    2 if ACS_CLASS_STAR>=0.8 (object is assumed to be a star and was not visually inspected)
    3 if ACS_CLASS_STAR<0.6 but object is visually identified as a star (e.g. saturated star, etc)

    JUNKFLAG 	float 	  	
    0 = good object
    1 = spurious
    '''

    print('Classifications:', Counter(cat.type).most_common())

    cat.is_galaxy = (cat.stellarity == 0)
    srcs = []
    for t in cat:
        pos = RaDecPos(t.ra, t.dec)
        bright = NanoMaggies(
            **{band: NanoMaggies.magToNanomaggies(t.acs_mag_auto)})
        shape = GalaxyShape(t.r_0p5_gim2d, 1. - t.ell_gim2d, 90. + t.pa_gim2d)

        is_galaxy = (t.is_galaxy * (shape.re >= 0) * (shape.ab <= 1.) *
                     (shape.phi > -999))

        if is_galaxy and t.type == 1:
            # deV
            src = DevGalaxy(pos, bright, shape)
        elif is_galaxy and t.type == 2:
            # exp
            src = ExpGalaxy(pos, bright, shape)
        else:
            src = PointSource(pos, bright)
        srcs.append(src)
    return srcs
Example #7
0
    print('Tim photocal:', tim.photocal)
    print('Subtim photocal:', subtim.photocal)

    # # Order by flux
    I = np.argsort([-src.getBrightness().getFlux(band) for src in srcs])
    for i in I:
        src = srcs[i]
        print('Source:', src)
        print('-> counts',
              subtim.photocal.brightnessToCounts(src.getBrightness()))

    # flux in nanomaggies
    flux = np.array([src.getBrightness().getFlux(band) for src in srcs])
    fluxiv = R.IV
    mag, magerr = NanoMaggies.fluxErrorsToMagErrors(flux, fluxiv)

    typemap = {ExpGalaxy: 'E', DevGalaxy: 'D', PointSource: 'P'}

    cat.tractor_type = np.array([typemap[type(src)] for src in srcs])
    cat.set('cfht_forced_mag_%s' % band, mag)
    cat.set('cfht_forced_magerr_%s' % band, magerr)
    cat.writeto('cfht-forced.fits')

    plt.clf()
    plt.plot(cat.acs_mag_auto, mag, 'b.')
    plt.xlabel('ACS I-band (mag)')
    plt.ylabel('CFHT %s-band forced phot (mag)' % band)
    plt.title('CFHT forced phot')
    ps.savefig()
Example #8
0
def main():

    ps = PlotSequence('dr3-summary')

    #fns = glob('/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-*.fits')
    #fns = glob('/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-240p005-250p010.fits')
    fns = ['dr3.1/sweep/3.1/sweep-240p005-250p010.fits',
#           '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-240p010-250p015.fits',
#           '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-230p005-240p010.fits',
#           '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-230p010-240p015.fits'
    ]

    plt.figure(2, figsize=(6,4))
    plt.subplots_adjust(left=0.1, bottom=0.15, right=0.98, top=0.98)
    plt.figure(1)


    TT = []
    for fn in fns:
        T = fits_table(fn)
        print(len(T), 'from', fn)
        TT.append(T)
    T = merge_tables(TT)
    del TT

    print('Types:', Counter(T.type))

    T.flux_g = T.decam_flux[:,1]
    T.flux_r = T.decam_flux[:,2]
    T.flux_z = T.decam_flux[:,4]
    T.flux_ivar_g = T.decam_flux_ivar[:,1]
    T.flux_ivar_r = T.decam_flux_ivar[:,2]
    T.flux_ivar_z = T.decam_flux_ivar[:,4]
    
    T.mag_g, T.magerr_g = NanoMaggies.fluxErrorsToMagErrors(T.flux_g, T.flux_ivar_g)
    T.mag_r, T.magerr_r = NanoMaggies.fluxErrorsToMagErrors(T.flux_r, T.flux_ivar_r)
    T.mag_z, T.magerr_z = NanoMaggies.fluxErrorsToMagErrors(T.flux_z, T.flux_ivar_z)

    T.deflux_g = T.flux_g / T.decam_mw_transmission[:,1]
    T.deflux_r = T.flux_r / T.decam_mw_transmission[:,2]
    T.deflux_z = T.flux_z / T.decam_mw_transmission[:,4]
    T.deflux_ivar_g = T.flux_ivar_g * T.decam_mw_transmission[:,1]**2
    T.deflux_ivar_r = T.flux_ivar_r * T.decam_mw_transmission[:,2]**2
    T.deflux_ivar_z = T.flux_ivar_z * T.decam_mw_transmission[:,4]**2

    T.demag_g, T.demagerr_g = NanoMaggies.fluxErrorsToMagErrors(T.deflux_g, T.deflux_ivar_g)
    T.demag_r, T.demagerr_r = NanoMaggies.fluxErrorsToMagErrors(T.deflux_r, T.deflux_ivar_r)
    T.demag_z, T.demagerr_z = NanoMaggies.fluxErrorsToMagErrors(T.deflux_z, T.deflux_ivar_z)

    T.typex = np.array([t[0] for t in T.type])


    T.mag_w1, T.magerr_w1 = NanoMaggies.fluxErrorsToMagErrors(
        T.wise_flux[:,0], T.wise_flux_ivar[:,0])
    T.mag_w2, T.magerr_w2 = NanoMaggies.fluxErrorsToMagErrors(
        T.wise_flux[:,1], T.wise_flux_ivar[:,1])
    tt = 'P'
    I2 = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) *
                        (T.flux_ivar_z > 0) *
                        (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
                        (T.magerr_g < 0.05) *
                        (T.magerr_r < 0.05) *
                        (T.magerr_z < 0.05) *
                        (T.typex == tt))
    print(len(I2), 'with < 0.05-mag errs and type PSF')
    plt.clf()
    plt.hist(T.mag_w1[I2], bins=100, range=[12,24], histtype='step', color='b')
    plt.hist(T.mag_w2[I2], bins=100, range=[12,24], histtype='step', color='g')
    plt.xlabel('WISE mag')
    ps.savefig()

    plt.clf()
    plt.hist(T.magerr_w1[I2], bins=100, range=[0,1], histtype='step', color='b')
    plt.hist(T.magerr_w2[I2], bins=100, range=[0,1], histtype='step', color='g')
    plt.xlabel('WISE mag errors')
    ps.savefig()

    I2 = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) *
                        (T.flux_ivar_z > 0) *
                        (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
                        (T.magerr_g < 0.05) *
                        (T.magerr_r < 0.05) *
                        (T.magerr_z < 0.05) *
                        (T.typex == tt) *
                        (T.magerr_w1 < 0.05))
    #(T.magerr_w2 < 0.1) *
    print(len(I2), 'with < 0.05-mag errs and type PSF and W1 errors < 0.1')

    Q1 = fits_table('dr3.1/external/survey-dr3-DR12Q.fits')
    Q2 = fits_table('dr3.1/external/DR12Q.fits', columns=['z_pipe'])
    Q1.add_columns_from(Q2)
    I = np.flatnonzero(Q1.objid >= 0)
    Q1.cut(I)
    print(len(Q1), 'sources with DR12Q matches')
    Q = Q1
    Q.typex = np.array([t[0] for t in Q.type])
    Q.mag_w1, Q.magerr_w1 = NanoMaggies.fluxErrorsToMagErrors(
        Q.wise_flux[:,0], Q.wise_flux_ivar[:,0])
    Q.mag_w2, Q.magerr_w2 = NanoMaggies.fluxErrorsToMagErrors(
        Q.wise_flux[:,1], Q.wise_flux_ivar[:,1])
    Q.flux_g = Q.decam_flux[:,1]
    Q.flux_r = Q.decam_flux[:,2]
    Q.flux_z = Q.decam_flux[:,4]
    Q.flux_ivar_g = Q.decam_flux_ivar[:,1]
    Q.flux_ivar_r = Q.decam_flux_ivar[:,2]
    Q.flux_ivar_z = Q.decam_flux_ivar[:,4]
    Q.deflux_g = Q.flux_g / Q.decam_mw_transmission[:,1]
    Q.deflux_r = Q.flux_r / Q.decam_mw_transmission[:,2]
    Q.deflux_z = Q.flux_z / Q.decam_mw_transmission[:,4]
    Q.deflux_ivar_g = Q.flux_ivar_g * Q.decam_mw_transmission[:,1]**2
    Q.deflux_ivar_r = Q.flux_ivar_r * Q.decam_mw_transmission[:,2]**2
    Q.deflux_ivar_z = Q.flux_ivar_z * Q.decam_mw_transmission[:,4]**2
    Q.demag_g, Q.demagerr_g = NanoMaggies.fluxErrorsToMagErrors(Q.deflux_g, Q.deflux_ivar_g)
    Q.demag_r, Q.demagerr_r = NanoMaggies.fluxErrorsToMagErrors(Q.deflux_r, Q.deflux_ivar_r)
    Q.demag_z, Q.demagerr_z = NanoMaggies.fluxErrorsToMagErrors(Q.deflux_z, Q.deflux_ivar_z)
    I = np.flatnonzero((Q.flux_ivar_g > 0) * (Q.flux_ivar_r > 0) *
                        (Q.flux_ivar_z > 0) *
                        (Q.flux_g > 0) * (Q.flux_r > 0) * (Q.flux_z > 0) *
                        (Q.demagerr_g < 0.05) *
                        (Q.demagerr_r < 0.05) *
                        (Q.demagerr_z < 0.05) *
                        (Q.typex == tt) *
                        (Q.magerr_w1 < 0.05))    
    Q.cut(I)
    print(len(Q), 'DR12Q-matched entries of type PSF with g,r,z,W1 errs < 0.05')
    
    plt.clf()
    loghist(T.demag_g[I2] - T.demag_r[I2], T.demag_z[I2] - T.mag_w1[I2],
             nbins=200, range=((-0.5,2.0),(-3,4)))
    plt.xlabel('g - r (mag)')
    plt.ylabel('z - W1 (mag)')
    ps.savefig()

    #Q.cut(np.argsort(Q.z_pipe))
    plt.clf()
    plt.scatter(Q.demag_g - Q.demag_r, Q.demag_z - Q.mag_w1, c=Q.z_pipe, s=5)
    plt.colorbar(label='QSO redshift')
    plt.xlabel('g - r (mag)')
    plt.ylabel('z - W1 (mag)')
    ps.savefig()

    plt.figure(2)
    from scipy.ndimage.filters import gaussian_filter
    import matplotlib.cm
    xlo,xhi,ylo,yhi = [-0.25, 1.75, -2.5, 2.5]
    plt.clf()
    H,xe,ye = loghist(T.demag_g[I2] - T.demag_r[I2], T.demag_z[I2] - T.mag_w1[I2],
                      nbins=(400,200), range=((xlo,xhi),(ylo,yhi)), imshowargs=dict(cmap='Greys'),
                      hot=False, docolorbar=False)
    I = np.flatnonzero(Q.z_pipe < 10.)
    QH,nil,nil = np.histogram2d(Q.demag_g[I] - Q.demag_r[I], Q.demag_z[I] - Q.mag_w1[I],
                       bins=200, range=((xlo,xhi),(ylo,yhi)))
    QH = gaussian_filter(QH.T, 3.)
    c = plt.contour(QH, 8, extent=[xe.min(), xe.max(), ye.min(), ye.max()], colors='k')
    plt.axis([xlo,xhi,ylo,yhi])
    plt.xlabel('g - r (mag)')
    plt.ylabel('z - W1 (mag)')

    mx,my,mz = [],[],[]
    # I = np.argsort(Q.z_pipe)
    # for i in range(len(Q)/1000):
    #     J = I[i*1000:][:1000]
    #     mx.append(np.median(Q.demag_g[J] - Q.demag_r[J]))
    #     my.append(np.median(Q.demag_z[J] - Q.mag_w1[J]))
    #     mz.append(np.median(Q.z_pipe[J]))
    # plt.scatter(mx,my, c=mz)
    # plt.colorbar(label='QSO Redshift')
    # ps.savefig()

        
    zvals = np.arange(0, 3.61, 0.1)
    for zlo,zhi in zip(zvals, zvals[1:]):
         J = np.flatnonzero((Q.z_pipe >= zlo) * (Q.z_pipe < zhi))
         mx.append(np.median(Q.demag_g[J] - Q.demag_r[J]))
         my.append(np.median(Q.demag_z[J] - Q.mag_w1[J]))
         mz.append(np.median(Q.z_pipe[J]))
    # plt.scatter(mx,my, c=mz, marker='o-')
    # plt.colorbar(label='QSO Redshift')
    for i in range(len(mx)-1):
        c = matplotlib.cm.viridis(0.8 * i / (len(mx)-1))
        plt.plot([mx[i], mx[i+1]], [my[i],my[i+1]], '-', color=c, lw=4)
    plt.savefig('qso-wise.png')
    plt.savefig('qso-wise.pdf')

    xlo,xhi,ylo,yhi = [-0.25, 1.75, -0.25, 2.5]
    plt.clf()
    H,xe,ye = loghist(T.demag_g[I2] - T.demag_r[I2], T.demag_r[I2] - T.demag_z[I2],
                      nbins=(400,200), range=((xlo,xhi),(ylo,yhi)), imshowargs=dict(cmap='Greys'),
                      hot=False, docolorbar=False)

    #I3 = np.random.permutation(I2)[:1000]
    #plt.plot(T.demag_g[I3] - T.demag_r[I3], T.demag_r[I3] - T.demag_z[I3], 'r.')
    
    I = np.flatnonzero(Q.z_pipe < 10.)
    QH,nil,nil = np.histogram2d(Q.demag_g[I] - Q.demag_r[I], Q.demag_r[I] - Q.demag_z[I],
                       bins=200, range=((xlo,xhi),(ylo,yhi)))
    QH = gaussian_filter(QH.T, 3.)
    c = plt.contour(QH, 8, extent=[xe.min(), xe.max(), ye.min(), ye.max()], colors='k')
    plt.axis([xlo,xhi,ylo,yhi])
    plt.xlabel('g - r (mag)')
    plt.ylabel('r - z (mag)')
    mx,my,mz = [],[],[]
    zvals = np.arange(0, 3.61, 0.1)
    for zlo,zhi in zip(zvals, zvals[1:]):
         J = np.flatnonzero((Q.z_pipe >= zlo) * (Q.z_pipe < zhi))
         mx.append(np.median(Q.demag_g[J] - Q.demag_r[J]))
         my.append(np.median(Q.demag_r[J] - Q.demag_z[J]))
         mz.append(np.median(Q.z_pipe[J]))
    for i in range(len(mx)-1):
        c = matplotlib.cm.viridis(0.8 * i / (len(mx)-1))
        plt.plot([mx[i], mx[i+1]], [my[i],my[i+1]], '-', color=c, lw=4)

    plt.savefig('qso-optical.png')
    plt.savefig('qso-optical.pdf')
    plt.figure(1)


    # I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0))
    # print(len(I), 'with g,r,z fluxes')
    # I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) *
    #                    (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0))
    # print(len(I), 'with positive g,r,z fluxes')
    # I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) *
    #                    (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
    #                    (T.magerr_g < 0.2) * (T.magerr_r < 0.2) * (T.magerr_z < 0.2))
    # print(len(I), 'with < 0.2-mag errs')
    # I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) *
    #                    (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
    #                    (T.magerr_g < 0.1) * (T.magerr_r < 0.1) * (T.magerr_z < 0.1))
    # print(len(I), 'with < 0.1-mag errs')
    # 
    # I2 = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) *
    #                     (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
    #                     (T.magerr_g < 0.05) * (T.magerr_r < 0.05) * (T.magerr_z < 0.05))
    # print(len(I2), 'with < 0.05-mag errs')

    #ha = dict(nbins=200, range=((-0.5, 2.5), (-0.5, 2.5)))
    ha = dict(nbins=400, range=((-0.5, 2.5), (-0.5, 2.5)))
    plt.clf()
    #plothist(T.mag_g[I] - T.mag_r[I], T.mag_r[I] - T.mag_z[I], 200)
    #loghist(T.mag_g[I] - T.mag_r[I], T.mag_r[I] - T.mag_z[I], **ha)
    loghist(T.demag_g[I] - T.demag_r[I], T.demag_r[I] - T.demag_z[I], **ha)
    plt.xlabel('g - r (mag)')
    plt.ylabel('r - z (mag)')
    ps.savefig()

    hists = []
    hists2 = []
    for tt,name in [('P', 'PSF'), ('E', 'EXP'), ('D', 'DeV')]:
        I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) *
                           (T.flux_ivar_z > 0) *
                           (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
                           (T.magerr_g < 0.1) * (T.magerr_r < 0.1) *(T.magerr_z < 0.1) *
                           (T.typex == tt))
        print(len(I), 'with < 0.1-mag errs and type', name)

        # plt.clf()
        # H,xe,ye = loghist(T.mag_g[I] - T.mag_r[I], T.mag_r[I] - T.mag_z[I], **ha)
        # plt.xlabel('g - r (mag)')
        # plt.ylabel('r - z (mag)')
        # plt.title('Type = %s' % name)
        # ps.savefig()

        plt.clf()
        H,xe,ye = loghist(T.demag_g[I] - T.demag_r[I], T.demag_r[I] - T.demag_z[I], **ha)

        # Keeping the dereddened color-color histograms
        hists.append(H.T)

        plt.xlabel('g - r (mag)')
        plt.ylabel('r - z (mag)')
        plt.title('Type = %s, dereddened' % name)
        ps.savefig()

        I2 = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) *
                            (T.flux_ivar_z > 0) *
                            (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
                            (T.magerr_g < 0.05) *
                            (T.magerr_r < 0.05) *
                            (T.magerr_z < 0.05) *
                            (T.typex == tt))
        print(len(I2), 'with < 0.05-mag errs and type', name)

        plt.clf()
        H,xe,ye = loghist(T.demag_g[I2] - T.demag_r[I2], T.demag_r[I2] - T.demag_z[I2], **ha)
        hists2.append(H.T)
        plt.xlabel('g - r (mag)')
        plt.ylabel('r - z (mag)')
        plt.title('Type = %s, dereddened, 5%% errs' % name)
        ps.savefig()

    for H in hists:
        H /= H.max()
    #for H in hists2:
    #    H /= H.max()
    #hists2 = [np.clip(H / np.percentile(H.ravel(), 99), 0, 1) for H in hists2]
    hists2 = [np.clip(H / np.percentile(H.ravel(), 99.9), 0, 1) for H in hists2]

    rgbmap = np.dstack((hists[2], hists[0], hists[1]))
    plt.clf()
    ((xlo,xhi),(ylo,yhi)) = ha['range']
    plt.imshow(rgbmap, interpolation='nearest', origin='lower', extent=[xlo,xhi,ylo,yhi])
    plt.axis([0,2,0,2])
    plt.xlabel('g - r (mag)')
    plt.ylabel('r - z (mag)')
    plt.title('Red: DeV, Blue: Exp, Green: PSF')
    ps.savefig()

    plt.clf()
    plt.imshow(np.sqrt(rgbmap), interpolation='nearest', origin='lower', extent=[xlo,xhi,ylo,yhi])
    plt.axis([0,2,0,2])
    plt.xlabel('g - r (mag)')
    plt.ylabel('r - z (mag)')
    plt.title('Red: DeV, Blue: Exp, Green: PSF')
    ps.savefig()

    # For a white-background plot, mix colors like paint
    for hr,hg,hb in [(hists[2], hists[0], hists[1]),
                     (hists2[2], hists2[0], hists2[1]),]:
        H,W = hr.shape
        for mapping in [lambda x: x]: #, lambda x: np.sqrt(x), lambda x: x**2]:
            red = np.ones((H,W,3))
            red[:,:,1] = 1 - mapping(hr)
            red[:,:,2] = 1 - mapping(hr)
            # in matplotlib, 'green'->(0, 0.5, 0)
            green = np.ones((H,W,3))
            green[:,:,0] = 1 - mapping(hg)
            green[:,:,1] = 1 - 0.5*mapping(hg)
            green[:,:,2] = 1 - mapping(hg)
            blue = np.ones((H,W,3))
            blue[:,:,0] = 1 - mapping(hb)
            blue[:,:,1] = 1 - mapping(hb)
            plt.clf()
            plt.imshow(red * green * blue, origin='lower', interpolation='nearest',
                       extent=[xlo,xhi,ylo,yhi], aspect='auto')
            plt.xlabel('g - r (mag)')
            plt.ylabel('r - z (mag)')
            plt.xticks(np.arange(0, 2.1, 0.5))
            plt.yticks(np.arange(0, 2.1, 0.5))
            plt.axis([-0.25,2,0,2])
            plt.title('Red: DeV, Blue: Exp, Green: PSF')
            ps.savefig()

            plt.figure(2)
            plt.clf()
            plt.imshow(red * green * blue, origin='lower', interpolation='nearest',
                       extent=[xlo,xhi,ylo,yhi], aspect='auto')
            plt.xlabel('g - r (mag)')
            plt.ylabel('r - z (mag)')
            plt.xticks(np.arange(0, 2.1, 0.5))
            plt.yticks(np.arange(0, 2.1, 0.5))
            plt.axis([0,2,-0.25,2])
            plt.text(0.7, 0.1, 'PSF', ha='center', va='center', color='k')
            plt.text(0.6, 0.7, 'EXP', ha='center', va='center', color='k')
            plt.text(1.5, 0.6, 'DEV', ha='center', va='center', color='k')
            plt.savefig('color-type.pdf')
            plt.figure(1)

    sys.exit(0)

            
    np.random.seed(42)

    T.gr = T.mag_g - T.mag_r
    T.rz = T.mag_r - T.mag_z

    for tt,name in [('P', 'PSF'), ('E', 'EXP'), ('D', 'DeV')]:
        I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) *
                           (T.flux_ivar_z > 0) *
                           (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
                           (T.magerr_g < 0.1) * (T.magerr_r < 0.1) *(T.magerr_z < 0.1) *
                           # Bright cut
                           (T.mag_r > 16.) *

                           # Contamination cut
                           (T.decam_fracflux[:,2] < 0.5) *

                           # Mag range cut
                           (T.mag_r > 19.) *
                           (T.mag_r < 20.) *
                           
                           (T.typex == tt))
        print(len(I), 'with < 0.1-mag errs and type', name)

        # plt.clf()
        # ha = dict(histtype='step', range=(16, 24), bins=100)
        # plt.hist(T.mag_g[I], color='g', **ha)
        # plt.hist(T.mag_r[I], color='r', **ha)
        # plt.hist(T.mag_z[I], color='m', **ha)
        # plt.title('Type %s' % name)
        # ps.savefig()

        J = np.random.permutation(I)

        if tt == 'D':
            J = J[T.shapedev_r[J] < 3.]

        J = J[:100]
        rzbin = np.argsort(T.rz[J])
        rzblock = (rzbin / 10).astype(int)
        K = np.lexsort((T.gr[J], rzblock))
        #print('sorted rzblock', rzblock[K])
        #print('sorted rzbin', rzbin[K])
        J = J[K]

        # plt.clf()
        # plt.hist(T.decam_fracflux[J,2], label='fracflux_r', histtype='step', bins=20)
        # plt.hist(T.decam_fracmasked[J,2], label='fracmasked_r', histtype='step', bins=20)
        # plt.legend()
        # plt.title('Type %s' % name)
        # ps.savefig()

        #print('fracflux r', T.decam_fracflux[J,2])
        #print('fracmasked r', T.decam_fracmasked[J,2])
        #print('anymasked r', T.decam_anymask[J,2])

        imgs = []
        imgrow = []
        stackrows = []
        for i in J:
            fn = 'cutouts/cutout_%.4f_%.4f.jpg' % (T.ra[i], T.dec[i])
            if not os.path.exists(fn):
                url = 'http://legacysurvey.org/viewer/jpeg-cutout/?layer=decals-dr3&pixscale=0.262&size=25&ra=%f&dec=%f' % (T.ra[i], T.dec[i])
                print('URL', url)
                cmd = 'wget -O %s.tmp "%s" && mv %s.tmp %s' % (fn, url, fn, fn)
                os.system(cmd)
            img = plt.imread(fn)
            #print('Image', img.shape)
            extra = ''
            if tt == 'D':
                extra = ', shapedev_r %.2f' % T.shapedev_r[i]
            elif tt == 'E':
                extra = ', shapeexp_r %.2f' % T.shapeexp_r[i]

            print(name, 'RA,Dec %.4f,%.4f, g/r/z %.2f, %.2f, %.2f, fracflux_r %.3f, fracmasked_r %.3f, anymasked_r %4i%s' % (
                T.ra[i], T.dec[i], T.mag_g[i], T.mag_r[i], T.mag_z[i],
                T.decam_fracflux[i,2], T.decam_fracmasked[i,2], T.decam_anymask[i,2], extra))

            imgrow.append(img)
            if len(imgrow) == 10:
                stack = np.hstack(imgrow)
                print('stacked row:', stack.shape)
                stackrows.append(stack)

                imgs.append(imgrow)
                imgrow = []


        stack = np.vstack(stackrows)
        print('Stack', stack.shape)
        plt.clf()
        plt.imshow(stack, interpolation='nearest', origin='lower')
        plt.xticks([])
        plt.yticks([])
        plt.title('Type %s' % name)
        ps.savefig()
        

    # Wrap-around at RA=300
    # rax = T.ra + (-360 * T.ra > 300.)
    # 
    # rabins = np.arange(-60., 300., 0.2)
    # decbins = np.arange(-20, 35, 0.2)
    # 
    # print('RA bins:', len(rabins))
    # print('Dec bins:', len(decbins))
    #
    # for name,cut in [(None, 'All sources'),
    #                  (T.type == 'PSF '), 'PSF'),
    #                  (T.type == 'SIMP'), 'SIMP'),
    #                  (T.type == 'DEV '), 'DEV'),
    #                  (T.type == 'EXP '), 'EXP'),
    #                  (T.type == 'COMP'), 'COMP'),
    #                  ]:
    #                  
    # H,xe,ye = np.histogram2d(T.ra, T.dec, bins=(rabins, decbins))
    # print('H shape', H.shape)
    # 
    # H = H.T
    # 
    # H /= (rabins[1:] - rabins[:-1])[np.newaxis, :]
    # H /= ((decbins[1:] - decbins[:-1]) * np.cos(np.deg2rad((decbins[1:] + decbins[:-1]) / 2.))))[:, np.newaxis]
    # 
    # plt.clf()
    # plt.imshow(H, extent=(rabins[0], rabins[-1], decbins[0], decbins[-1]), interpolation='nearest', origin='lower',
    #            vmin=0)
    # cbar = plt.colorbar()
    # cbar.set_label('Source density per square degree')
    # plt.xlim(rabins[-1], rabins[0])
    # plt.xlabel('RA (deg)')
    # plt.ylabel('Dec (deg)')
    # ps.savefig()



    '''
def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--name1', help='Name for first data set')
    parser.add_argument('--name2', help='Name for second data set')
    parser.add_argument('--plot-prefix',
                        default='compare',
                        help='Prefix for plot filenames; default "%default"')
    parser.add_argument('--match',
                        default=1.0,
                        help='Astrometric cross-match distance in arcsec')
    parser.add_argument('dir1', help='First directory to compare')
    parser.add_argument('dir2', help='Second directory to compare')

    opt = parser.parse_args()

    ps = PlotSequence(opt.plot_prefix)

    name1 = opt.name1
    if name1 is None:
        name1 = os.path.basename(opt.dir1)
        if not len(name1):
            name1 = os.path.basename(os.path.dirname(opt.dir1))
    name2 = opt.name2
    if name2 is None:
        name2 = os.path.basename(opt.dir2)
        if not len(name2):
            name2 = os.path.basename(os.path.dirname(opt.dir2))
    tt = 'Comparing %s to %s' % (name1, name2)

    # regex for tractor-*.fits catalog filename
    catre = re.compile('tractor-.*.fits')

    cat1, cat2 = [], []
    for basedir, cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
        for dirpath, dirnames, filenames in os.walk(basedir, followlinks=True):
            for fn in filenames:
                if not catre.match(fn):
                    print('Skipping', fn, 'due to filename')
                    continue
                fn = os.path.join(dirpath, fn)
                t = fits_table(fn)
                print(len(t), 'from', fn)
                cat.append(t)
    cat1 = merge_tables(cat1, columns='fillzero')
    cat2 = merge_tables(cat2, columns='fillzero')
    print('Total of', len(cat1), 'from', name1)
    print('Total of', len(cat2), 'from', name2)
    cat1.cut(cat1.brick_primary)
    cat2.cut(cat2.brick_primary)
    print('Total of', len(cat1), 'BRICK_PRIMARY from', name1)
    print('Total of', len(cat2), 'BRICK_PRIMARY from', name2)

    cat1.cut((cat1.decam_anymask[:, 1] == 0) *
             (cat1.decam_anymask[:, 2] == 0) * (cat1.decam_anymask[:, 4] == 0))
    cat2.cut((cat2.decam_anymask[:, 1] == 0) *
             (cat2.decam_anymask[:, 2] == 0) * (cat2.decam_anymask[:, 4] == 0))
    print('Total of', len(cat1), 'unmasked from', name1)
    print('Total of', len(cat2), 'unmasked from', name2)

    I, J, d = match_radec(cat1.ra,
                          cat1.dec,
                          cat2.ra,
                          cat2.dec,
                          opt.match / 3600.,
                          nearest=True)
    print(len(I), 'matched')

    plt.clf()
    plt.hist(d * 3600., 100)
    plt.xlabel('Match distance (arcsec)')
    plt.title(tt)
    ps.savefig()

    matched1 = cat1[I]
    matched2 = cat2[J]

    for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
        K = np.flatnonzero((matched1.decam_flux_ivar[:, iband] > 0) *
                           (matched2.decam_flux_ivar[:, iband] > 0))

        print('Median mw_trans', band, 'is',
              np.median(matched1.decam_mw_transmission[:, iband]))

        plt.clf()
        plt.errorbar(
            matched1.decam_flux[K, iband],
            matched2.decam_flux[K, iband],
            fmt='.',
            color=cc,
            xerr=1. / np.sqrt(matched1.decam_flux_ivar[K, iband]),
            yerr=1. / np.sqrt(matched2.decam_flux_ivar[K, iband]),
            alpha=0.1,
        )
        plt.xlabel('%s flux: %s' % (name1, band))
        plt.ylabel('%s flux: %s' % (name2, band))
        plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
        plt.axis([-100, 1000, -100, 1000])
        plt.title(tt)
        ps.savefig()

    for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
        good = ((matched1.decam_flux_ivar[:, iband] > 0) *
                (matched2.decam_flux_ivar[:, iband] > 0))
        K = np.flatnonzero(good)
        psf1 = (matched1.type == 'PSF ')
        psf2 = (matched2.type == 'PSF ')
        P = np.flatnonzero(good * psf1 * psf2)

        mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
            matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])

        iv1 = matched1.decam_flux_ivar[:, iband]
        iv2 = matched2.decam_flux_ivar[:, iband]
        std = np.sqrt(1. / iv1 + 1. / iv2)

        plt.clf()
        plt.plot(
            mag1[K],
            (matched2.decam_flux[K, iband] - matched1.decam_flux[K, iband]) /
            std[K],
            '.',
            alpha=0.1,
            color=cc)
        plt.plot(
            mag1[P],
            (matched2.decam_flux[P, iband] - matched1.decam_flux[P, iband]) /
            std[P],
            '.',
            alpha=0.1,
            color='k')
        plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' %
                   (name2, name1, band))
        plt.xlabel('%s mag: %s' % (name1, band))
        plt.axhline(0, color='k', alpha=0.5)
        plt.axis([24, 16, -10, 10])
        plt.title(tt)
        ps.savefig()

    plt.clf()
    lp, lt = [], []
    for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
        good = ((matched1.decam_flux_ivar[:, iband] > 0) *
                (matched2.decam_flux_ivar[:, iband] > 0))
        #good = True
        psf1 = (matched1.type == 'PSF ')
        psf2 = (matched2.type == 'PSF ')
        mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
            matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
        iv1 = matched1.decam_flux_ivar[:, iband]
        iv2 = matched2.decam_flux_ivar[:, iband]
        std = np.sqrt(1. / iv1 + 1. / iv2)
        #std = np.hypot(std, 0.01)
        G = np.flatnonzero(good * psf1 * psf2 * np.isfinite(mag1) *
                           (mag1 >= 20) *
                           (mag1 < dict(g=24, r=23.5, z=22.5)[band]))

        n, b, p = plt.hist(
            (matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) /
            std[G],
            range=(-4, 4),
            bins=50,
            histtype='step',
            color=cc,
            normed=True)

        sig = (matched2.decam_flux[G, iband] -
               matched1.decam_flux[G, iband]) / std[G]
        print('Raw mean and std of points:', np.mean(sig), np.std(sig))
        med = np.median(sig)
        rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.
        print('Median and percentile-based sigma:', med, rsigma)
        lp.append(p[0])
        lt.append('%s: %.2f +- %.2f' % (band, med, rsigma))

    bins = []
    gaussint = []
    for blo, bhi in zip(b, b[1:]):
        c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
        c /= (bhi - blo)
        #bins.extend([blo,bhi])
        #gaussint.extend([c,c])
        bins.append((blo + bhi) / 2.)
        gaussint.append(c)
    plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)

    plt.title(tt)
    plt.xlabel('Flux difference / error (sigma)')
    plt.axvline(0, color='k', alpha=0.1)
    plt.ylim(0, 0.45)
    plt.legend(lp, lt, loc='upper right')
    ps.savefig()

    for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
        plt.clf()
        mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
            matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
        mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
            matched2.decam_flux[:, iband], matched2.decam_flux_ivar[:, iband])

        meanmag = NanoMaggies.nanomaggiesToMag(
            (matched1.decam_flux[:, iband] + matched2.decam_flux[:, iband]) /
            2.)

        psf1 = (matched1.type == 'PSF ')
        psf2 = (matched2.type == 'PSF ')
        good = ((matched1.decam_flux_ivar[:, iband] > 0) *
                (matched2.decam_flux_ivar[:, iband] > 0) * np.isfinite(mag1) *
                np.isfinite(mag2))
        K = np.flatnonzero(good)
        P = np.flatnonzero(good * psf1 * psf2)

        plt.errorbar(mag1[K],
                     mag2[K],
                     fmt='.',
                     color=cc,
                     xerr=magerr1[K],
                     yerr=magerr2[K],
                     alpha=0.1)
        plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5)
        plt.xlabel('%s %s (mag)' % (name1, band))
        plt.ylabel('%s %s (mag)' % (name2, band))
        plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
        plt.axis([24, 16, 24, 16])
        plt.title(tt)
        ps.savefig()

        plt.clf()
        plt.errorbar(mag1[K],
                     mag2[K] - mag1[K],
                     fmt='.',
                     color=cc,
                     xerr=magerr1[K],
                     yerr=magerr2[K],
                     alpha=0.1)
        plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5)
        plt.xlabel('%s %s (mag)' % (name1, band))
        plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band))
        plt.axhline(0., color='k', alpha=1.)
        plt.axis([24, 16, -1, 1])
        plt.title(tt)
        ps.savefig()

        magbins = np.arange(16, 24.001, 0.5)

        plt.clf()
        plt.plot(mag1[K],
                 (mag2[K] - mag1[K]) / np.hypot(magerr1[K], magerr2[K]),
                 '.',
                 color=cc,
                 alpha=0.1)
        plt.plot(mag1[P],
                 (mag2[P] - mag1[P]) / np.hypot(magerr1[P], magerr2[P]),
                 'k.',
                 alpha=0.5)

        plt.xlabel('%s %s (mag)' % (name1, band))
        plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
                   (name2, band, name1, band))
        plt.axhline(0., color='k', alpha=1.)
        plt.axis([24, 16, -10, 10])
        plt.title(tt)
        ps.savefig()

        y = (mag2 - mag1) / np.hypot(magerr1, magerr2)

        plt.clf()
        plt.plot(meanmag[P], y[P], 'k.', alpha=0.1)

        midmag = []
        vals = np.zeros((len(magbins) - 1, 5))
        median_err1 = []

        iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)

        # FIXME -- should we do some stats after taking off the mean difference?

        for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
            I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
            midmag.append((mlo + mhi) / 2.)
            median_err1.append(np.median(magerr1[I]))
            if len(I) == 0:
                continue
            # median and +- 1 sigma quantiles
            ybin = y[I]
            vals[bini, 0] = np.percentile(ybin, 16)
            vals[bini, 1] = np.median(ybin)
            vals[bini, 2] = np.percentile(ybin, 84)
            # +- 2 sigma quantiles
            vals[bini, 3] = np.percentile(ybin, 2.3)
            vals[bini, 4] = np.percentile(ybin, 97.7)

            iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)

            print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss,
                  'vs expected for Gaussian;', len(ybin), 'points')

            # if iqd > iqd_gauss:
            #     # What error adding in quadrature would you need to make the IQD match?
            #     err = median_err1[-1]
            #     target_err = err * (iqd / iqd_gauss)
            #     sys_err = np.sqrt(target_err**2 - err**2)
            #     print('--> add systematic error', sys_err)

        # ~ Johan's cuts
        mlo = 21.
        mhi = dict(g=24., r=23.5, z=22.5)[band]
        I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
        ybin = y[I]
        iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
        print('Mag bin', mlo, mhi, 'band', band,
              ': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;',
              len(ybin), 'points')
        if iqd > iqd_gauss:
            # What error adding in quadrature would you need to make
            # the IQD match?
            err = np.median(np.hypot(magerr1[I], magerr2[I]))
            print('Median error (hypot):', err)
            target_err = err * (iqd / iqd_gauss)
            print('Target:', target_err)
            sys_err = np.sqrt((target_err**2 - err**2) / 2.)
            print('--> add systematic error', sys_err)

            # check...
            err_sys = np.hypot(np.hypot(magerr1, sys_err),
                               np.hypot(magerr2, sys_err))
            ysys = (mag2 - mag1) / err_sys
            ysys = ysys[I]
            print('Resulting median error:', np.median(err_sys[I]))
            iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
            print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian')
            # Hmmm, this doesn't work... totally overshoots.

        plt.errorbar(midmag,
                     vals[:, 1],
                     fmt='o',
                     color='b',
                     yerr=(vals[:, 1] - vals[:, 0], vals[:, 2] - vals[:, 1]),
                     capthick=3,
                     zorder=20)
        plt.errorbar(midmag,
                     vals[:, 1],
                     fmt='o',
                     color='b',
                     yerr=(vals[:, 1] - vals[:, 3], vals[:, 4] - vals[:, 1]),
                     capthick=2,
                     zorder=20)
        plt.axhline(1., color='b', alpha=0.2)
        plt.axhline(-1., color='b', alpha=0.2)
        plt.axhline(2., color='b', alpha=0.2)
        plt.axhline(-2., color='b', alpha=0.2)

        for mag, err, y in zip(midmag, median_err1, vals[:, 3]):
            if not np.isfinite(err):
                continue
            if y < -6:
                continue
            plt.text(mag,
                     y - 0.1,
                     '%.3f' % err,
                     va='top',
                     ha='center',
                     color='k',
                     fontsize=10)

        plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band))
        plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
                   (name2, band, name1, band))
        plt.axhline(0., color='k', alpha=1.)

        plt.axvline(21, color='k', alpha=0.3)
        plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3)

        plt.axis([24.1, 16, -6, 6])
        plt.title(tt)
        ps.savefig()

        #magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
        if band == 'g':
            magbins = [20, 24]
        elif band == 'r':
            magbins = [20, 23.5]
        elif band == 'z':
            magbins = [20, 22.5]

        slo, shi = -5, 5
        plt.clf()
        ha = dict(bins=25, range=(slo, shi), histtype='step', normed=True)
        y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
        midmag = []
        nn = []
        rgbs = []
        lt, lp = [], []
        for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
            I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
            if len(I) == 0:
                continue
            ybin = y[I]
            rgb = [0., 0., 0.]
            rgb[0] = float(bini) / (len(magbins) - 1)
            rgb[2] = 1. - rgb[0]
            n, b, p = plt.hist(ybin, color=rgb, **ha)
            lt.append('mag %g to %g' % (mlo, mhi))
            lp.append(p[0])
            midmag.append((mlo + mhi) / 2.)
            nn.append(n)
            rgbs.append(rgb)

        bins = []
        gaussint = []
        for blo, bhi in zip(b, b[1:]):
            #midbin.append((blo+bhi)/2.)
            #gaussint.append(scipy.stats.norm.cdf(bhi) -
            #                scipy.stats.norm.cdf(blo))
            c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
            c /= (bhi - blo)
            bins.extend([blo, bhi])
            gaussint.extend([c, c])
        plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)

        plt.legend(lp, lt)
        plt.title(tt)
        plt.xlim(slo, shi)
        ps.savefig()

        bincenters = b[:-1] + (b[1] - b[0]) / 2.
        plt.clf()
        lp = []
        for n, rgb, mlo, mhi in zip(nn, rgbs, magbins, magbins[1:]):
            p = plt.plot(bincenters, n, '-', color=rgb)
            lp.append(p[0])
        plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2)
        plt.legend(lp, lt)
        plt.title(tt)
        plt.xlim(slo, shi)
        ps.savefig()
Example #10
0
def plot_light_curves(pfn, ucal=False):
    lightcurves = unpickle_from_file(pfn)

    if ucal:
        tag = 'ucal-'
    else:
        tag = ''

    survey = LegacySurveyData()
    brickname = '0364m042'
    catfn = survey.find_file('tractor', brick=brickname)
    print('Reading catalog from', catfn)
    cat = fits_table(catfn)
    print(len(cat), 'catalog entries')
    cat.cut(cat.brick_primary)
    print(len(cat), 'brick primary')

    I = []
    for i, oid in enumerate(cat.objid):
        if (brickname, oid) in lightcurves:
            I.append(i)
    I = np.array(I)
    cat.cut(I)
    print('Cut to', len(cat), 'with light curves')

    S = fits_table('specObj-dr12-trim-2.fits')

    from astrometry.libkd.spherematch import match_radec
    I, J, d = match_radec(S.ra, S.dec, cat.ra, cat.dec, 2. / 3600.)
    print('Matched', len(I), 'to spectra')

    plt.subplots_adjust(hspace=0)

    movie_jpegs = []
    movie_wcs = None

    for i in range(28):
        fn = os.path.join('des-sn-movie', 'epoch%i' % i, 'coadd',
                          brickname[:3], brickname,
                          'legacysurvey-%s-image.jpg' % brickname)
        print(fn)
        if not os.path.exists(fn):
            continue

        img = plt.imread(fn)
        img = np.flipud(img)
        h, w, d = img.shape

        fn = os.path.join('des-sn-movie', 'epoch%i' % i, 'coadd',
                          brickname[:3], brickname,
                          'legacysurvey-%s-image-r.fits' % brickname)
        if not os.path.exists(fn):
            continue
        wcs = Tan(fn)

        movie_jpegs.append(img)
        movie_wcs = wcs

    plt.figure(figsize=(8, 6), dpi=100)
    n = 0

    fluxtags = [('flux', 'flux_ivar', '', 'a')]
    if ucal:
        fluxtags.append(('uflux', 'uflux_ivar', ': ucal', 'b'))

    for oid, ii in zip(cat.objid[J], I):
        print('Objid', oid)
        spec = S[ii]
        k = (brickname, oid)
        v = lightcurves[k]

        # Cut bad CCDs
        v.cut(np.array([e not in [230151, 230152, 230153] for e in v.expnum]))

        plt.clf()
        print('obj', k, 'has', len(v), 'measurements')
        T = v

        for fluxtag, fluxivtag, fluxname, plottag in fluxtags:
            plt.clf()

            filts = np.unique(T.filter)
            for i, f in enumerate(filts):
                from tractor.brightness import NanoMaggies

                plt.subplot(len(filts), 1, i + 1)

                fluxes = np.hstack(
                    [T.get(ft[0])[T.filter == f] for ft in fluxtags])
                fluxes = fluxes[np.isfinite(fluxes)]
                mn, mx = np.percentile(fluxes, [5, 95])
                print('Flux percentiles for filter', f, ':', mn, mx)
                # note swap
                mn, mx = NanoMaggies.nanomaggiesToMag(
                    mx), NanoMaggies.nanomaggiesToMag(mn)
                print('-> mags', mn, mx)

                cut = (T.filter == f) * (T.flux_ivar > 0)
                if ucal:
                    cut *= np.isfinite(T.uflux)
                I = np.flatnonzero(cut)

                print('  ', len(I), 'in', f, 'band')
                I = I[np.argsort(T.mjd[I])]
                mediv = np.median(T.flux_ivar[I])
                # cut really noisy ones
                I = I[T.flux_ivar[I] > 0.25 * mediv]

                #plt.plot(T.mjd[I], T.flux[I], '.-', color=dict(g='g',r='r',z='m')[f])
                # plt.errorbar(T.mjd[I], T.flux[I], yerr=1/np.sqrt(T.fluxiv[I]),
                #              fmt='.-', color=dict(g='g',r='r',z='m')[f])
                #plt.errorbar(T.mjd[I], T.flux[I], yerr=1/np.sqrt(T.fluxiv[I]),
                #             fmt='.', color=dict(g='g',r='r',z='m')[f])

                # if ucal:
                #     mag,dmag = NanoMaggies.fluxErrorsToMagErrors(T.flux[I], T.flux_ivar[I])
                # else:
                #     mag,dmag = NanoMaggies.fluxErrorsToMagErrors(T.uflux[I], T.uflux_ivar[I])
                mag, dmag = NanoMaggies.fluxErrorsToMagErrors(
                    T.get(fluxtag)[I],
                    T.get(fluxivtag)[I])

                plt.errorbar(T.mjd[I],
                             mag,
                             yerr=dmag,
                             fmt='.',
                             color=dict(g='g', r='r', z='m')[f])
                #yl,yh = plt.ylim()
                #plt.ylim(yh,yl)
                plt.ylim(mx, mn)

                plt.ylabel(f)

                if i + 1 < len(filts):
                    plt.xticks([])
                #plt.yscale('symlog')

            outfn = 'cutout_%.4f_%.4f.jpg' % (spec.ra, spec.dec)
            if not os.path.exists(outfn):
                url = 'http://legacysurvey.org/viewer/jpeg-cutout/?ra=%.4f&dec=%.4f&zoom=14&layer=sdssco&size=128' % (
                    spec.ra, spec.dec)
                cmd = 'wget -O %s "%s"' % (outfn, url)
                print(cmd)
                os.system(cmd)
            pix = plt.imread(outfn)
            h, w, d = pix.shape
            fig = plt.gcf()

            #print('fig bbox:', fig.bbox)
            #print('xmax, ymax', fig.bbox.xmax, fig.bbox.ymax)
            #plt.figimage(pix, 0, fig.bbox.ymax - h, zorder=10)
            #plt.figimage(pix, 0, fig.bbox.ymax, zorder=10)
            #plt.figimage(pix, fig.bbox.xmax - w, fig.bbox.ymax, zorder=10)
            plt.figimage(pix,
                         fig.bbox.xmax - (w + 2),
                         fig.bbox.ymax - (h + 2),
                         zorder=10)

            plt.suptitle('SDSS spectro object: %s at (%.4f, %.4f)%s' %
                         (spec.label.strip(), spec.ra, spec.dec, fluxname))
            plt.savefig('forced-%s%i-%s.png' % (tag, n, plottag))

        ok, x, y = movie_wcs.radec2pixelxy(spec.ra, spec.dec)
        x = int(np.round(x - 1))
        y = int(np.round(y - 1))
        sz = 32

        plt.clf()
        plt.subplots_adjust(hspace=0, wspace=0)
        k = 1
        for i, img in enumerate(movie_jpegs):
            stamp = img[y - sz:y + sz + 1, x - sz:x + sz + 1]

            plt.subplot(5, 6, k)
            plt.imshow(stamp, interpolation='nearest', origin='lower')
            plt.xticks([])
            plt.yticks([])
            k += 1
        plt.suptitle('SDSS spectro object: %s at (%.4f, %.4f): DES images' %
                     (spec.label.strip(), spec.ra, spec.dec))
        plt.savefig('forced-%s%i-c.png' % (tag, n))

        n += 1
Example #11
0
def all(matched1, matched2, d, name1='ref', name2='test'):
    tt = 'Comparing %s to %s' % (name1, name2)
    plt.clf()
    plt.hist(d * 3600., 100)
    plt.xlabel('Match distance (arcsec)')
    plt.title(tt)
    plt.savefig(os.path.join(matched1.outdir, 'sep_hist.png'))
    plt.close()

    for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
        K = np.flatnonzero((matched1.t['decam_flux_ivar'][:, iband] > 0) *
                           (matched2.t['decam_flux_ivar'][:, iband] > 0))

        print('Median mw_trans', band, 'is',
              np.median(matched1.t['decam_mw_transmission'][:, iband]))

        plt.clf()
        plt.errorbar(
            matched1.t['decam_flux'][K, iband],
            matched2.t['decam_flux'][K, iband],
            fmt='.',
            color=cc,
            xerr=1. / np.sqrt(matched1.t['decam_flux_ivar'][K, iband]),
            yerr=1. / np.sqrt(matched2.t['decam_flux_ivar'][K, iband]),
            alpha=0.1,
        )
        plt.xlabel('%s flux: %s' % (name1, band))
        plt.ylabel('%s flux: %s' % (name2, band))
        plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
        plt.axis([-100, 1000, -100, 1000])
        plt.title(tt)
        plt.savefig(os.path.join(matched1.outdir, '%s_fluxerr.png' % band))
        plt.close()

    print("exiting early")
    sys.exit()
    for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
        good = ((matched1.decam_flux_ivar[:, iband] > 0) *
                (matched2.decam_flux_ivar[:, iband] > 0))
        K = np.flatnonzero(good)
        psf1 = (matched1.type == 'PSF ')
        psf2 = (matched2.type == 'PSF ')
        P = np.flatnonzero(good * psf1 * psf2)

        mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
            matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])

        iv1 = matched1.decam_flux_ivar[:, iband]
        iv2 = matched2.decam_flux_ivar[:, iband]
        std = np.sqrt(1. / iv1 + 1. / iv2)

        plt.clf()
        plt.plot(
            mag1[K],
            (matched2.decam_flux[K, iband] - matched1.decam_flux[K, iband]) /
            std[K],
            '.',
            alpha=0.1,
            color=cc)
        plt.plot(
            mag1[P],
            (matched2.decam_flux[P, iband] - matched1.decam_flux[P, iband]) /
            std[P],
            '.',
            alpha=0.1,
            color='k')
        plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' %
                   (name2, name1, band))
        plt.xlabel('%s mag: %s' % (name1, band))
        plt.axhline(0, color='k', alpha=0.5)
        plt.axis([24, 16, -10, 10])
        plt.title(tt)
        ps.savefig()

    plt.clf()
    lp, lt = [], []
    for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
        good = ((matched1.decam_flux_ivar[:, iband] > 0) *
                (matched2.decam_flux_ivar[:, iband] > 0))
        #good = True
        psf1 = (matched1.type == 'PSF ')
        psf2 = (matched2.type == 'PSF ')
        mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
            matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
        iv1 = matched1.decam_flux_ivar[:, iband]
        iv2 = matched2.decam_flux_ivar[:, iband]
        std = np.sqrt(1. / iv1 + 1. / iv2)
        #std = np.hypot(std, 0.01)
        G = np.flatnonzero(good * psf1 * psf2 * np.isfinite(mag1) *
                           (mag1 >= 20) *
                           (mag1 < dict(g=24, r=23.5, z=22.5)[band]))

        n, b, p = plt.hist(
            (matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) /
            std[G],
            range=(-4, 4),
            bins=50,
            histtype='step',
            color=cc,
            normed=True)

        sig = (matched2.decam_flux[G, iband] -
               matched1.decam_flux[G, iband]) / std[G]
        print('Raw mean and std of points:', np.mean(sig), np.std(sig))
        med = np.median(sig)
        rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.
        print('Median and percentile-based sigma:', med, rsigma)
        lp.append(p[0])
        lt.append('%s: %.2f +- %.2f' % (band, med, rsigma))

    bins = []
    gaussint = []
    for blo, bhi in zip(b, b[1:]):
        c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
        c /= (bhi - blo)
        #bins.extend([blo,bhi])
        #gaussint.extend([c,c])
        bins.append((blo + bhi) / 2.)
        gaussint.append(c)
    plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)

    plt.title(tt)
    plt.xlabel('Flux difference / error (sigma)')
    plt.axvline(0, color='k', alpha=0.1)
    plt.ylim(0, 0.45)
    plt.legend(lp, lt, loc='upper right')
    ps.savefig()

    for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]:
        plt.clf()
        mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
            matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband])
        mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
            matched2.decam_flux[:, iband], matched2.decam_flux_ivar[:, iband])

        meanmag = NanoMaggies.nanomaggiesToMag(
            (matched1.decam_flux[:, iband] + matched2.decam_flux[:, iband]) /
            2.)

        psf1 = (matched1.type == 'PSF ')
        psf2 = (matched2.type == 'PSF ')
        good = ((matched1.decam_flux_ivar[:, iband] > 0) *
                (matched2.decam_flux_ivar[:, iband] > 0) * np.isfinite(mag1) *
                np.isfinite(mag2))
        K = np.flatnonzero(good)
        P = np.flatnonzero(good * psf1 * psf2)

        plt.errorbar(mag1[K],
                     mag2[K],
                     fmt='.',
                     color=cc,
                     xerr=magerr1[K],
                     yerr=magerr2[K],
                     alpha=0.1)
        plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5)
        plt.xlabel('%s %s (mag)' % (name1, band))
        plt.ylabel('%s %s (mag)' % (name2, band))
        plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.)
        plt.axis([24, 16, 24, 16])
        plt.title(tt)
        ps.savefig()

        plt.clf()
        plt.errorbar(mag1[K],
                     mag2[K] - mag1[K],
                     fmt='.',
                     color=cc,
                     xerr=magerr1[K],
                     yerr=magerr2[K],
                     alpha=0.1)
        plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5)
        plt.xlabel('%s %s (mag)' % (name1, band))
        plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band))
        plt.axhline(0., color='k', alpha=1.)
        plt.axis([24, 16, -1, 1])
        plt.title(tt)
        ps.savefig()

        magbins = np.arange(16, 24.001, 0.5)

        plt.clf()
        plt.plot(mag1[K],
                 (mag2[K] - mag1[K]) / np.hypot(magerr1[K], magerr2[K]),
                 '.',
                 color=cc,
                 alpha=0.1)
        plt.plot(mag1[P],
                 (mag2[P] - mag1[P]) / np.hypot(magerr1[P], magerr2[P]),
                 'k.',
                 alpha=0.5)

        plt.xlabel('%s %s (mag)' % (name1, band))
        plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
                   (name2, band, name1, band))
        plt.axhline(0., color='k', alpha=1.)
        plt.axis([24, 16, -10, 10])
        plt.title(tt)
        ps.savefig()

        y = (mag2 - mag1) / np.hypot(magerr1, magerr2)

        plt.clf()
        plt.plot(meanmag[P], y[P], 'k.', alpha=0.1)

        midmag = []
        vals = np.zeros((len(magbins) - 1, 5))
        median_err1 = []

        iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)

        # FIXME -- should we do some stats after taking off the mean difference?

        for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
            I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
            midmag.append((mlo + mhi) / 2.)
            median_err1.append(np.median(magerr1[I]))
            if len(I) == 0:
                continue
            # median and +- 1 sigma quantiles
            ybin = y[I]
            vals[bini, 0] = np.percentile(ybin, 16)
            vals[bini, 1] = np.median(ybin)
            vals[bini, 2] = np.percentile(ybin, 84)
            # +- 2 sigma quantiles
            vals[bini, 3] = np.percentile(ybin, 2.3)
            vals[bini, 4] = np.percentile(ybin, 97.7)

            iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)

            print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss,
                  'vs expected for Gaussian;', len(ybin), 'points')

            # if iqd > iqd_gauss:
            #     # What error adding in quadrature would you need to make the IQD match?
            #     err = median_err1[-1]
            #     target_err = err * (iqd / iqd_gauss)
            #     sys_err = np.sqrt(target_err**2 - err**2)
            #     print('--> add systematic error', sys_err)

        # ~ Johan's cuts
        mlo = 21.
        mhi = dict(g=24., r=23.5, z=22.5)[band]
        I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
        ybin = y[I]
        iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
        print('Mag bin', mlo, mhi, 'band', band,
              ': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;',
              len(ybin), 'points')
        if iqd > iqd_gauss:
            # What error adding in quadrature would you need to make
            # the IQD match?
            err = np.median(np.hypot(magerr1[I], magerr2[I]))
            print('Median error (hypot):', err)
            target_err = err * (iqd / iqd_gauss)
            print('Target:', target_err)
            sys_err = np.sqrt((target_err**2 - err**2) / 2.)
            print('--> add systematic error', sys_err)

            # check...
            err_sys = np.hypot(np.hypot(magerr1, sys_err),
                               np.hypot(magerr2, sys_err))
            ysys = (mag2 - mag1) / err_sys
            ysys = ysys[I]
            print('Resulting median error:', np.median(err_sys[I]))
            iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
            print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian')
            # Hmmm, this doesn't work... totally overshoots.

        plt.errorbar(midmag,
                     vals[:, 1],
                     fmt='o',
                     color='b',
                     yerr=(vals[:, 1] - vals[:, 0], vals[:, 2] - vals[:, 1]),
                     capthick=3,
                     zorder=20)
        plt.errorbar(midmag,
                     vals[:, 1],
                     fmt='o',
                     color='b',
                     yerr=(vals[:, 1] - vals[:, 3], vals[:, 4] - vals[:, 1]),
                     capthick=2,
                     zorder=20)
        plt.axhline(1., color='b', alpha=0.2)
        plt.axhline(-1., color='b', alpha=0.2)
        plt.axhline(2., color='b', alpha=0.2)
        plt.axhline(-2., color='b', alpha=0.2)

        for mag, err, y in zip(midmag, median_err1, vals[:, 3]):
            if not np.isfinite(err):
                continue
            if y < -6:
                continue
            plt.text(mag,
                     y - 0.1,
                     '%.3f' % err,
                     va='top',
                     ha='center',
                     color='k',
                     fontsize=10)

        plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band))
        plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
                   (name2, band, name1, band))
        plt.axhline(0., color='k', alpha=1.)

        plt.axvline(21, color='k', alpha=0.3)
        plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3)

        plt.axis([24.1, 16, -6, 6])
        plt.title(tt)
        ps.savefig()

        #magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
        if band == 'g':
            magbins = [20, 24]
        elif band == 'r':
            magbins = [20, 23.5]
        elif band == 'z':
            magbins = [20, 22.5]

        slo, shi = -5, 5
        plt.clf()
        ha = dict(bins=25, range=(slo, shi), histtype='step', normed=True)
        y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
        midmag = []
        nn = []
        rgbs = []
        lt, lp = [], []
        for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
            I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
            if len(I) == 0:
                continue
            ybin = y[I]
            rgb = [0., 0., 0.]
            rgb[0] = float(bini) / (len(magbins) - 1)
            rgb[2] = 1. - rgb[0]
            n, b, p = plt.hist(ybin, color=rgb, **ha)
            lt.append('mag %g to %g' % (mlo, mhi))
            lp.append(p[0])
            midmag.append((mlo + mhi) / 2.)
            nn.append(n)
            rgbs.append(rgb)

        bins = []
        gaussint = []
        for blo, bhi in zip(b, b[1:]):
            #midbin.append((blo+bhi)/2.)
            #gaussint.append(scipy.stats.norm.cdf(bhi) -
            #                scipy.stats.norm.cdf(blo))
            c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
            c /= (bhi - blo)
            bins.extend([blo, bhi])
            gaussint.extend([c, c])
        plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)

        plt.legend(lp, lt)
        plt.title(tt)
        plt.xlim(slo, shi)
        ps.savefig()

        bincenters = b[:-1] + (b[1] - b[0]) / 2.
        plt.clf()
        lp = []
        for n, rgb, mlo, mhi in zip(nn, rgbs, magbins, magbins[1:]):
            p = plt.plot(bincenters, n, '-', color=rgb)
            lp.append(p[0])
        plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2)
        plt.legend(lp, lt)
        plt.title(tt)
        plt.xlim(slo, shi)
        ps.savefig()
Example #12
0
def compare_mags(TT, name, ps):
    for i,T in enumerate(TT):
        T.set('exp', np.zeros(len(T), np.uint8)+i)

    plt.clf()
    ap = 5    
    for i,T in enumerate(TT):
        cc = 'rgb'[i]
        plt.plot(T.flux, T.apflux[:, ap] / T.flux,
                 '.', color=cc, alpha=0.5)

        ff, frac = [],[]
        mags = np.arange(14, 24)
        for mlo,mhi in zip(mags, mags[1:]):
            flo = NanoMaggies.magToNanomaggies(mhi)
            fhi = NanoMaggies.magToNanomaggies(mlo)
            I = np.flatnonzero((T.flux > flo) * (T.flux <= fhi))
            ff.append(np.sqrt(flo * fhi))
            frac.append(np.median(T.apflux[I,ap] / T.flux[I]))
        plt.plot(ff, frac, 'o-', color=cc)
        
    plt.xscale('symlog')
    plt.xlim(1., 1e3)
    plt.ylim(0.9, 1.1)
    plt.xlabel('Forced-phot flux')
    plt.ylabel('Aperture / Forced-phot flux')
    plt.axhline(1, color='k', alpha=0.1)
    plt.title('%s region: Aperture %i fluxes' % (name, ap))
    ps.savefig()

    T = merge_tables(TT)

    T.bobjid = T.brickid.astype(int) * 10000 + T.objid
    bomap = {}
    for i,bo in enumerate(T.bobjid):
        try:
            bomap[bo].append(i)
        except KeyError:
            bomap[bo] = [i]

    II = []
    for bo,ii in bomap.items():
        if len(ii) != 3:
            continue
        II.append(ii)
    II = np.array(II)
    print 'II', II.shape

    exps = T.exp[II]
    print 'exposures:', exps
    assert(np.all(T.exp[II[:,0]] == 0))
    assert(np.all(T.exp[II[:,1]] == 1))
    assert(np.all(T.exp[II[:,2]] == 2))

    fluxes = T.flux[II]
    print 'fluxes', fluxes.shape
    meanflux = np.mean(fluxes, axis=1)
    print 'meanfluxes', meanflux.shape

    plt.clf()
    for i in range(3):
        plt.plot(meanflux, fluxes[:,i] / meanflux, '.',
                 color='rgb'[i], alpha=0.5)
    #plt.yscale('symlog')
    plt.xscale('symlog')
    plt.xlabel('Mean flux (nanomaggies)')
    plt.ylabel('Forced-phot flux / Mean')
    #plt.ylim(0, 2)
    plt.ylim(0.9, 1.1)
    plt.xlim(0, 1e3)
    plt.axhline(1, color='k', alpha=0.1)
    plt.title('%s region: Forced-phot fluxes' % name)
    ps.savefig()

    for ap in [4,5,6]:
        apfluxes = T.apflux[:,ap][II,]
        print 'ap fluxes', apfluxes.shape

        plt.clf()
        for i in range(3):
            plt.plot(meanflux, apfluxes[:,i] / meanflux, '.',
                     color='rgb'[i], alpha=0.5)
        plt.xscale('symlog')
        plt.xlabel('Mean flux (nanomaggies)')
        plt.ylabel('Aperture(%i) flux / Mean' % ap)
        plt.ylim(0.9, 1.1)
        plt.xlim(0, 1e3)
        plt.axhline(1, color='k', alpha=0.1)
        plt.title('%s region: Aperture %i fluxes' % (name, ap))
        ps.savefig()

        plt.clf()
        for i in range(3):
            plt.plot(fluxes[:,i], apfluxes[:,i] / fluxes[:,i], '.',
                     color='rgb'[i], alpha=0.5)
        plt.xscale('symlog')
        plt.xlim(0, 1e3)
        plt.ylim(0.9, 1.1)
        plt.xlabel('Forced-phot flux')
        plt.ylabel('Aperture / Forced-phot flux')
        plt.axhline(1, color='k', alpha=0.1)
        plt.title('%s region: Aperture %i fluxes' % (name, ap))
        ps.savefig()
Example #13
0
def compare_to_ps1(ps, ccds):
    
    decals = Decals()

    allplots = []

    for expnum,ccdname in ccds:
        ccd = decals.find_ccds(expnum=expnum, ccdname=ccdname)
        assert(len(ccd) == 1)
        ccd = ccd[0]
        im = decals.get_image_object(ccd)
        print 'Reading', im

        wcs = im.get_wcs()

        magrange = (15,20)
        ps1 = ps1cat(ccdwcs=wcs)
        ps1 = ps1.get_stars(band=im.band, magrange=magrange)
        print 'Got', len(ps1), 'PS1 stars'
        # ps1.about()

        F = fits_table('forced-%i-%s.fits' % (expnum, ccdname))
        print 'Read', len(F), 'forced-phot results'
        
        F.ra,F.dec = wcs.pixelxy2radec(F.x+1, F.y+1)

        I,J,d = match_radec(F.ra, F.dec, ps1.ra, ps1.dec, 1./3600.)
        print 'Matched', len(I), 'stars to PS1'

        F.cut(I)
        ps1.cut(J)

        F.mag = NanoMaggies.nanomaggiesToMag(F.flux)
        F.apmag = NanoMaggies.nanomaggiesToMag(F.apflux[:,5])

        iband = ps1cat.ps1band[im.band]
        ps1mag = ps1.median[:,iband]
        mags = np.arange(magrange[0], 1+magrange[1])

        psf = im.read_psf_model(0, 0, pixPsf=True)
        pixscale = 0.262
        apertures = apertures_arcsec / pixscale
        h,w = ccd.height, ccd.width
        psfimg = psf.getPointSourcePatch(w/2., h/2.).patch
        ph,pw = psfimg.shape
        cx,cy = pw/2, ph/2
        apphot = []
        for rad in apertures:
            aper = photutils.CircularAperture((cx,cy), rad)
            p = photutils.aperture_photometry(psfimg, aper)
            apphot.append(p.field('aperture_sum'))
        apphot = np.hstack(apphot)
        print 'aperture photometry:', apphot
        skyest = apphot[6] - apphot[5]
        print 'Sky estimate:', skyest
        skyest /= np.pi * (apertures[6]**2 - apertures[5]**2)
        print 'Sky estimate per pixel:', skyest
        fraction = apphot[5] - skyest * np.pi * apertures[5]**2
        print 'Fraction of flux:', fraction
        zp = 2.5 * np.log10(fraction)
        print 'ZP adjustment:', zp
        
        plt.clf()

        for cc,mag,label in [('b', F.mag, 'Forced mag'), ('r', F.apmag, 'Aper mag')]:
            plt.plot(ps1mag, mag - ps1mag, '.', color=cc, label=label, alpha=0.6)

            mm,dd = [],[]
            for mlo,mhi in zip(mags, mags[1:]):
                I = np.flatnonzero((ps1mag > mlo) * (ps1mag <= mhi))
                mm.append((mlo+mhi)/2.)
                dd.append(np.median(mag[I] - ps1mag[I]))
            plt.plot(mm, dd, 'o-', color=cc)

            mm = np.array(mm)
            dd = np.array(dd)
            plt.plot(mm, dd - zp, 'o--', lw=3, alpha=0.5, color=cc)

            allplots.append((mm, dd, zp, cc, label))
            
        plt.xlabel('PS1 %s mag' % im.band)
        plt.ylabel('Mag - PS1 (mag)')
        plt.title('PS1 - Single-epoch mag: %i-%s' % (expnum, ccdname))
        plt.ylim(-0.2, 0.2)
        mlo,mhi = magrange
        plt.xlim(mhi, mlo)
        plt.axhline(0., color='k', alpha=0.1)
        plt.legend()
        ps.savefig()

    plt.clf()
    # for mm,dd,zp,cc,label in allplots:
    #     plt.plot(mm, dd, 'o-', color=cc, label=label)
    #     plt.plot(mm, dd - zp, 'o--', lw=3, alpha=0.5, color=cc)
    for sp,add in [(1,False),(2,True)]:
        plt.subplot(2,1,sp)
        for mm,dd,zp,cc,label in allplots:
            if add:
                plt.plot(mm, dd - zp, 'o--', lw=3, alpha=0.5, color=cc)
            else:
                plt.plot(mm, dd, 'o-', color=cc, label=label)
        plt.ylabel('Mag - PS1 (mag)')
        plt.ylim(-0.2, 0.05)
        mlo,mhi = magrange
        plt.xlim(mhi, mlo)
        plt.axhline(0., color='k', alpha=0.1)
        plt.axhline(-0.05, color='k', alpha=0.1)
        plt.axhline(-0.1, color='k', alpha=0.1)
    plt.xlabel('PS1 %s mag' % im.band)
    plt.suptitle('PS1 - Single-epoch mags')
    #plt.legend()
    ps.savefig()

    plt.clf()
    for mm,dd,zp,cc,label in allplots:
        plt.plot(mm, dd, 'o-', color=cc, label=label)
        plt.plot(mm, dd - zp, 'o--', lw=3, alpha=0.5, color=cc)
    plt.ylabel('Mag - PS1 (mag)')
    plt.ylim(-0.2, 0.05)
    mlo,mhi = magrange
    plt.xlim(mhi, mlo)
    plt.axhline(0., color='k', alpha=0.1)
    plt.xlabel('PS1 %s mag' % im.band)
    plt.suptitle('PS1 - Single-epoch mags')
    ps.savefig()
Example #14
0
    import argparse
    parser = argparse.ArgumentParser(description='Produce annotated CCDs file by reading CCDs file + calibration products')
    parser.add_argument('--part', action='append', help='CCDs file to read, survey-ccds-X.fits.gz, default: ["decals","nondecals","extra"].  Can be repeated.', default=[])
    parser.add_argument('--threads', type=int, help='Run multi-threaded', default=4)
    opt = parser.parse_args()


    if False:
        #### FIX mistake in DR3 annotated CCDs: sig1
        for part in ['decals', 'nondecals', 'extra']:
            from tractor.brightness import NanoMaggies
            fn = '/global/cscratch1/sd/desiproc/dr3/ccds-annotated-%s.fits.gz' % part
            T = fits_table(fn)
            zpt = T.ccdzpt + 2.5 * np.log10(T.exptime)
            print('Median zpt:', np.median(zpt))
            zpscale = NanoMaggies.zeropointToScale(zpt)
            print('Median zpscale:', np.median(zpscale))
    
            print('Zpscale limits:', zpscale.min(), zpscale.max())
            print('Non-finite:', np.sum(np.logical_not(np.isfinite(zpscale))))
    
            T.sig1 /= zpscale
            T.sig1[np.logical_not(np.isfinite(T.sig1))] = 0.
            print('Median sig1:', np.median(T.sig1))
    
            print('Before:')
            for col in ['psfdepth', 'galdepth', 'gausspsfdepth', 'gaussgaldepth']:
                c = T.get(col)
                for band in 'grz':
                    I = np.flatnonzero((T.filter == band) * (T.sig1 > 0))
                    print(col, band, 'median', np.median(c[I]))
Example #15
0
    ps.savefig()

    print('Tim photocal:', tim.photocal)
    print('Subtim photocal:', subtim.photocal)
    
    # # Order by flux
    I = np.argsort([-src.getBrightness().getFlux(band) for src in srcs])
    for i in I:
        src = srcs[i]
        print('Source:', src)
        print('-> counts', subtim.photocal.brightnessToCounts(src.getBrightness()))

    # flux in nanomaggies
    flux = np.array([src.getBrightness().getFlux(band) for src in srcs])
    fluxiv = R.IV
    mag, magerr = NanoMaggies.fluxErrorsToMagErrors(flux, fluxiv)

    typemap = { ExpGalaxy:'E', DevGalaxy:'D', PointSource:'P' }
    
    cat.tractor_type = np.array([typemap[type(src)] for src in srcs])
    cat.set('cfht_forced_mag_%s'    % band, mag)
    cat.set('cfht_forced_magerr_%s' % band, magerr)
    cat.writeto('cfht-forced.fits')

    
    plt.clf()
    plt.plot(cat.acs_mag_auto, mag, 'b.')
    plt.xlabel('ACS I-band (mag)')
    plt.ylabel('CFHT %s-band forced phot (mag)' % band)
    plt.title('CFHT forced phot')
    ps.savefig()
Example #16
0
def gim2d_catalog(cat, band):
    '''
    http://irsa.ipac.caltech.edu/data/COSMOS/tables/morphology/cosmos_morph_zurich_colDescriptions.html
    '''
    '''
    ACS_MU_CLASS 	float 	  	Type of object.
    1 = galaxy
    2 = star
    3 = spurious
    '''
    '''
    ACS_CLEAN 	float 	  	Object useable flag.
    0 = do not use this object
    1 = use this object
    '''
    '''
    FLUX_GIM2D 	float 	counts 	GIM2D total flux
    R_GIM2D 	float 	arcseconds 	GIM2D psf-convolved half-light radius of object
    ELL_GIM2D 	float 	  	GIM2D ellipticity = 1-b/a of object
    PA_GIM2D 	float 	degrees 	GIM2D position angle of object - cw from +y-axis
    DX_GIM2D 	float 	arcseconds 	x-offset of GIM2D-model center from ACS-coordinate center
    DY_GIM2D 	float 	arcseconds 	y-offset of GIM2D-model center from ACS-coordinate center
    SERSIC_N_GIM2D 	float 	  	GIM2D Sersic index
    R_0P5_GIM2D 	float 	arcseconds 	GIM2D half-light radius of object without PSF convolution

    TYPE 	float 	  	ZEST Type CLASS
    1 = Early type
    2 = Disk
    3 = Irregular Galaxy
    9 = no classification
    '''

    '''
    BULG 	float 	  	ZEST "Bulgeness" CLASS - only for Type 2 (disk) galaxies.
    0 = bulge dominated galaxy
    1,2 = intermediate-bulge galaxies
    3 = pure disk galaxy
    9 = no classification
    '''

    '''
    STELLARITY 	float 	  	Visual Stellarity flag.

    0 if ACS_CLASS_STAR<0.6 (object is ASSUMED to be a galaxy; no visual inspection)
    0 if ACS_CLASS_STAR>=0.6 AND object visually identified as a galaxy.
    1 if ACS_CLASS_STAR>=0.6 AND visually identified as a star.
    2 if ACS_CLASS_STAR>=0.8 (object is assumed to be a star and was not visually inspected)
    3 if ACS_CLASS_STAR<0.6 but object is visually identified as a star (e.g. saturated star, etc)

    JUNKFLAG 	float 	  	
    0 = good object
    1 = spurious
    '''
    
    print('Classifications:', Counter(cat.type).most_common())
    
    cat.is_galaxy = (cat.stellarity == 0)
    srcs = []
    for t in cat:
        pos = RaDecPos(t.ra, t.dec)
        bright = NanoMaggies(**{band:NanoMaggies.magToNanomaggies(t.acs_mag_auto)})
        shape = GalaxyShape(t.r_0p5_gim2d, 1. - t.ell_gim2d, 90. + t.pa_gim2d)

        is_galaxy = (t.is_galaxy * (shape.re >= 0 ) * (shape.ab <= 1.) *
                     (shape.phi > -999))
        
        if is_galaxy and t.type == 1:
            # deV
            src = DevGalaxy(pos, bright, shape)
        elif is_galaxy and t.type == 2:
            # exp
            src = ExpGalaxy(pos, bright, shape)
        else:
            src = PointSource(pos, bright)
        srcs.append(src)
    return srcs
Example #17
0
def all(matched1, matched2, d, name1="ref", name2="test"):
    tt = "Comparing %s to %s" % (name1, name2)
    plt.clf()
    plt.hist(d * 3600.0, 100)
    plt.xlabel("Match distance (arcsec)")
    plt.title(tt)
    plt.savefig(os.path.join(matched1.outdir, "sep_hist.png"))
    plt.close()

    for iband, band, cc in [(1, "g", "g"), (2, "r", "r"), (4, "z", "m")]:
        K = np.flatnonzero(
            (matched1.t["decam_flux_ivar"][:, iband] > 0) * (matched2.t["decam_flux_ivar"][:, iband] > 0)
        )

        print("Median mw_trans", band, "is", np.median(matched1.t["decam_mw_transmission"][:, iband]))

        plt.clf()
        plt.errorbar(
            matched1.t["decam_flux"][K, iband],
            matched2.t["decam_flux"][K, iband],
            fmt=".",
            color=cc,
            xerr=1.0 / np.sqrt(matched1.t["decam_flux_ivar"][K, iband]),
            yerr=1.0 / np.sqrt(matched2.t["decam_flux_ivar"][K, iband]),
            alpha=0.1,
        )
        plt.xlabel("%s flux: %s" % (name1, band))
        plt.ylabel("%s flux: %s" % (name2, band))
        plt.plot([-1e6, 1e6], [-1e6, 1e6], "k-", alpha=1.0)
        plt.axis([-100, 1000, -100, 1000])
        plt.title(tt)
        plt.savefig(os.path.join(matched1.outdir, "%s_fluxerr.png" % band))
        plt.close()

    print("exiting early")
    sys.exit()
    for iband, band, cc in [(1, "g", "g"), (2, "r", "r"), (4, "z", "m")]:
        good = (matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0)
        K = np.flatnonzero(good)
        psf1 = matched1.type == "PSF "
        psf2 = matched2.type == "PSF "
        P = np.flatnonzero(good * psf1 * psf2)

        mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
            matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband]
        )

        iv1 = matched1.decam_flux_ivar[:, iband]
        iv2 = matched2.decam_flux_ivar[:, iband]
        std = np.sqrt(1.0 / iv1 + 1.0 / iv2)

        plt.clf()
        plt.plot(
            mag1[K], (matched2.decam_flux[K, iband] - matched1.decam_flux[K, iband]) / std[K], ".", alpha=0.1, color=cc
        )
        plt.plot(
            mag1[P], (matched2.decam_flux[P, iband] - matched1.decam_flux[P, iband]) / std[P], ".", alpha=0.1, color="k"
        )
        plt.ylabel("(%s - %s) flux / flux errors (sigma): %s" % (name2, name1, band))
        plt.xlabel("%s mag: %s" % (name1, band))
        plt.axhline(0, color="k", alpha=0.5)
        plt.axis([24, 16, -10, 10])
        plt.title(tt)
        ps.savefig()

    plt.clf()
    lp, lt = [], []
    for iband, band, cc in [(1, "g", "g"), (2, "r", "r"), (4, "z", "m")]:
        good = (matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0)
        # good = True
        psf1 = matched1.type == "PSF "
        psf2 = matched2.type == "PSF "
        mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
            matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband]
        )
        iv1 = matched1.decam_flux_ivar[:, iband]
        iv2 = matched2.decam_flux_ivar[:, iband]
        std = np.sqrt(1.0 / iv1 + 1.0 / iv2)
        # std = np.hypot(std, 0.01)
        G = np.flatnonzero(
            good * psf1 * psf2 * np.isfinite(mag1) * (mag1 >= 20) * (mag1 < dict(g=24, r=23.5, z=22.5)[band])
        )

        n, b, p = plt.hist(
            (matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) / std[G],
            range=(-4, 4),
            bins=50,
            histtype="step",
            color=cc,
            normed=True,
        )

        sig = (matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) / std[G]
        print("Raw mean and std of points:", np.mean(sig), np.std(sig))
        med = np.median(sig)
        rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.0
        print("Median and percentile-based sigma:", med, rsigma)
        lp.append(p[0])
        lt.append("%s: %.2f +- %.2f" % (band, med, rsigma))

    bins = []
    gaussint = []
    for blo, bhi in zip(b, b[1:]):
        c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
        c /= bhi - blo
        # bins.extend([blo,bhi])
        # gaussint.extend([c,c])
        bins.append((blo + bhi) / 2.0)
        gaussint.append(c)
    plt.plot(bins, gaussint, "k-", lw=2, alpha=0.5)

    plt.title(tt)
    plt.xlabel("Flux difference / error (sigma)")
    plt.axvline(0, color="k", alpha=0.1)
    plt.ylim(0, 0.45)
    plt.legend(lp, lt, loc="upper right")
    ps.savefig()

    for iband, band, cc in [(1, "g", "g"), (2, "r", "r"), (4, "z", "m")]:
        plt.clf()
        mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
            matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband]
        )
        mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
            matched2.decam_flux[:, iband], matched2.decam_flux_ivar[:, iband]
        )

        meanmag = NanoMaggies.nanomaggiesToMag((matched1.decam_flux[:, iband] + matched2.decam_flux[:, iband]) / 2.0)

        psf1 = matched1.type == "PSF "
        psf2 = matched2.type == "PSF "
        good = (
            (matched1.decam_flux_ivar[:, iband] > 0)
            * (matched2.decam_flux_ivar[:, iband] > 0)
            * np.isfinite(mag1)
            * np.isfinite(mag2)
        )
        K = np.flatnonzero(good)
        P = np.flatnonzero(good * psf1 * psf2)

        plt.errorbar(mag1[K], mag2[K], fmt=".", color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
        plt.plot(mag1[P], mag2[P], "k.", alpha=0.5)
        plt.xlabel("%s %s (mag)" % (name1, band))
        plt.ylabel("%s %s (mag)" % (name2, band))
        plt.plot([-1e6, 1e6], [-1e6, 1e6], "k-", alpha=1.0)
        plt.axis([24, 16, 24, 16])
        plt.title(tt)
        ps.savefig()

        plt.clf()
        plt.errorbar(mag1[K], mag2[K] - mag1[K], fmt=".", color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
        plt.plot(mag1[P], mag2[P] - mag1[P], "k.", alpha=0.5)
        plt.xlabel("%s %s (mag)" % (name1, band))
        plt.ylabel("%s %s - %s %s (mag)" % (name2, band, name1, band))
        plt.axhline(0.0, color="k", alpha=1.0)
        plt.axis([24, 16, -1, 1])
        plt.title(tt)
        ps.savefig()

        magbins = np.arange(16, 24.001, 0.5)

        plt.clf()
        plt.plot(mag1[K], (mag2[K] - mag1[K]) / np.hypot(magerr1[K], magerr2[K]), ".", color=cc, alpha=0.1)
        plt.plot(mag1[P], (mag2[P] - mag1[P]) / np.hypot(magerr1[P], magerr2[P]), "k.", alpha=0.5)

        plt.xlabel("%s %s (mag)" % (name1, band))
        plt.ylabel("(%s %s - %s %s) / errors (sigma)" % (name2, band, name1, band))
        plt.axhline(0.0, color="k", alpha=1.0)
        plt.axis([24, 16, -10, 10])
        plt.title(tt)
        ps.savefig()

        y = (mag2 - mag1) / np.hypot(magerr1, magerr2)

        plt.clf()
        plt.plot(meanmag[P], y[P], "k.", alpha=0.1)

        midmag = []
        vals = np.zeros((len(magbins) - 1, 5))
        median_err1 = []

        iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)

        # FIXME -- should we do some stats after taking off the mean difference?

        for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
            I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
            midmag.append((mlo + mhi) / 2.0)
            median_err1.append(np.median(magerr1[I]))
            if len(I) == 0:
                continue
            # median and +- 1 sigma quantiles
            ybin = y[I]
            vals[bini, 0] = np.percentile(ybin, 16)
            vals[bini, 1] = np.median(ybin)
            vals[bini, 2] = np.percentile(ybin, 84)
            # +- 2 sigma quantiles
            vals[bini, 3] = np.percentile(ybin, 2.3)
            vals[bini, 4] = np.percentile(ybin, 97.7)

            iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)

            print(
                "Mag bin",
                midmag[-1],
                ": IQD is factor",
                iqd / iqd_gauss,
                "vs expected for Gaussian;",
                len(ybin),
                "points",
            )

            # if iqd > iqd_gauss:
            #     # What error adding in quadrature would you need to make the IQD match?
            #     err = median_err1[-1]
            #     target_err = err * (iqd / iqd_gauss)
            #     sys_err = np.sqrt(target_err**2 - err**2)
            #     print('--> add systematic error', sys_err)

        # ~ Johan's cuts
        mlo = 21.0
        mhi = dict(g=24.0, r=23.5, z=22.5)[band]
        I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
        ybin = y[I]
        iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
        print(
            "Mag bin",
            mlo,
            mhi,
            "band",
            band,
            ": IQD is factor",
            iqd / iqd_gauss,
            "vs expected for Gaussian;",
            len(ybin),
            "points",
        )
        if iqd > iqd_gauss:
            # What error adding in quadrature would you need to make
            # the IQD match?
            err = np.median(np.hypot(magerr1[I], magerr2[I]))
            print("Median error (hypot):", err)
            target_err = err * (iqd / iqd_gauss)
            print("Target:", target_err)
            sys_err = np.sqrt((target_err ** 2 - err ** 2) / 2.0)
            print("--> add systematic error", sys_err)

            # check...
            err_sys = np.hypot(np.hypot(magerr1, sys_err), np.hypot(magerr2, sys_err))
            ysys = (mag2 - mag1) / err_sys
            ysys = ysys[I]
            print("Resulting median error:", np.median(err_sys[I]))
            iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
            print("--> IQD", iqd_sys / iqd_gauss, "vs Gaussian")
            # Hmmm, this doesn't work... totally overshoots.

        plt.errorbar(
            midmag,
            vals[:, 1],
            fmt="o",
            color="b",
            yerr=(vals[:, 1] - vals[:, 0], vals[:, 2] - vals[:, 1]),
            capthick=3,
            zorder=20,
        )
        plt.errorbar(
            midmag,
            vals[:, 1],
            fmt="o",
            color="b",
            yerr=(vals[:, 1] - vals[:, 3], vals[:, 4] - vals[:, 1]),
            capthick=2,
            zorder=20,
        )
        plt.axhline(1.0, color="b", alpha=0.2)
        plt.axhline(-1.0, color="b", alpha=0.2)
        plt.axhline(2.0, color="b", alpha=0.2)
        plt.axhline(-2.0, color="b", alpha=0.2)

        for mag, err, y in zip(midmag, median_err1, vals[:, 3]):
            if not np.isfinite(err):
                continue
            if y < -6:
                continue
            plt.text(mag, y - 0.1, "%.3f" % err, va="top", ha="center", color="k", fontsize=10)

        plt.xlabel("(%s + %s)/2 %s (mag), PSFs" % (name1, name2, band))
        plt.ylabel("(%s %s - %s %s) / errors (sigma)" % (name2, band, name1, band))
        plt.axhline(0.0, color="k", alpha=1.0)

        plt.axvline(21, color="k", alpha=0.3)
        plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color="k", alpha=0.3)

        plt.axis([24.1, 16, -6, 6])
        plt.title(tt)
        ps.savefig()

        # magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
        if band == "g":
            magbins = [20, 24]
        elif band == "r":
            magbins = [20, 23.5]
        elif band == "z":
            magbins = [20, 22.5]

        slo, shi = -5, 5
        plt.clf()
        ha = dict(bins=25, range=(slo, shi), histtype="step", normed=True)
        y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
        midmag = []
        nn = []
        rgbs = []
        lt, lp = [], []
        for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])):
            I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
            if len(I) == 0:
                continue
            ybin = y[I]
            rgb = [0.0, 0.0, 0.0]
            rgb[0] = float(bini) / (len(magbins) - 1)
            rgb[2] = 1.0 - rgb[0]
            n, b, p = plt.hist(ybin, color=rgb, **ha)
            lt.append("mag %g to %g" % (mlo, mhi))
            lp.append(p[0])
            midmag.append((mlo + mhi) / 2.0)
            nn.append(n)
            rgbs.append(rgb)

        bins = []
        gaussint = []
        for blo, bhi in zip(b, b[1:]):
            # midbin.append((blo+bhi)/2.)
            # gaussint.append(scipy.stats.norm.cdf(bhi) -
            #                scipy.stats.norm.cdf(blo))
            c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
            c /= bhi - blo
            bins.extend([blo, bhi])
            gaussint.extend([c, c])
        plt.plot(bins, gaussint, "k-", lw=2, alpha=0.5)

        plt.legend(lp, lt)
        plt.title(tt)
        plt.xlim(slo, shi)
        ps.savefig()

        bincenters = b[:-1] + (b[1] - b[0]) / 2.0
        plt.clf()
        lp = []
        for n, rgb, mlo, mhi in zip(nn, rgbs, magbins, magbins[1:]):
            p = plt.plot(bincenters, n, "-", color=rgb)
            lp.append(p[0])
        plt.plot(bincenters, gaussint[::2], "k-", alpha=0.5, lw=2)
        plt.legend(lp, lt)
        plt.title(tt)
        plt.xlim(slo, shi)
        ps.savefig()
def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--name1', help='Name for first data set')
    parser.add_argument('--name2', help='Name for second data set')
    parser.add_argument('--plot-prefix', default='compare',
                        help='Prefix for plot filenames; default "%default"')
    parser.add_argument('--match', default=1.0,
                        help='Astrometric cross-match distance in arcsec')
    parser.add_argument('dir1', help='First directory to compare')
    parser.add_argument('dir2', help='Second directory to compare')

    opt = parser.parse_args()
    
    ps = PlotSequence(opt.plot_prefix)

    name1 = opt.name1
    if name1 is None:
        name1 = os.path.basename(opt.dir1)
        if not len(name1):
            name1 = os.path.basename(os.path.dirname(opt.dir1))
    name2 = opt.name2
    if name2 is None:
        name2 = os.path.basename(opt.dir2)
        if not len(name2):
            name2 = os.path.basename(os.path.dirname(opt.dir2))
    tt = 'Comparing %s to %s' % (name1, name2)

    # regex for tractor-*.fits catalog filename
    catre = re.compile('tractor-.*.fits')
        
    cat1,cat2 = [],[]
    for basedir,cat in [(opt.dir1, cat1), (opt.dir2, cat2)]:
        for dirpath,dirnames,filenames in os.walk(basedir, followlinks=True):
            for fn in filenames:
                if not catre.match(fn):
                    print('Skipping', fn, 'due to filename')
                    continue
                fn = os.path.join(dirpath, fn)
                t = fits_table(fn)
                print(len(t), 'from', fn)
                cat.append(t)
    cat1 = merge_tables(cat1, columns='fillzero')
    cat2 = merge_tables(cat2, columns='fillzero')
    print('Total of', len(cat1), 'from', name1)
    print('Total of', len(cat2), 'from', name2)
    cat1.cut(cat1.brick_primary)
    cat2.cut(cat2.brick_primary)
    print('Total of', len(cat1), 'BRICK_PRIMARY from', name1)
    print('Total of', len(cat2), 'BRICK_PRIMARY from', name2)

    cat1.cut((cat1.decam_anymask[:,1] == 0) *
             (cat1.decam_anymask[:,2] == 0) *
             (cat1.decam_anymask[:,4] == 0))
    cat2.cut((cat2.decam_anymask[:,1] == 0) *
             (cat2.decam_anymask[:,2] == 0) *
             (cat2.decam_anymask[:,4] == 0))
    print('Total of', len(cat1), 'unmasked from', name1)
    print('Total of', len(cat2), 'unmasked from', name2)
    
    I,J,d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match/3600.,
                        nearest=True)
    print(len(I), 'matched')

    plt.clf()
    plt.hist(d * 3600., 100)
    plt.xlabel('Match distance (arcsec)')
    plt.title(tt)
    ps.savefig()

    matched1 = cat1[I]
    matched2 = cat2[J]

    for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
        K = np.flatnonzero((matched1.decam_flux_ivar[:,iband] > 0) *
                           (matched2.decam_flux_ivar[:,iband] > 0))
        
        print('Median mw_trans', band, 'is',
              np.median(matched1.decam_mw_transmission[:,iband]))
        
        plt.clf()
        plt.errorbar(matched1.decam_flux[K,iband],
                     matched2.decam_flux[K,iband],
                     fmt='.', color=cc,
                     xerr=1./np.sqrt(matched1.decam_flux_ivar[K,iband]),
                     yerr=1./np.sqrt(matched2.decam_flux_ivar[K,iband]),
                     alpha=0.1,
                     )
        plt.xlabel('%s flux: %s' % (name1, band))
        plt.ylabel('%s flux: %s' % (name2, band))
        plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.)
        plt.axis([-100, 1000, -100, 1000])
        plt.title(tt)
        ps.savefig()


    for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
        good = ((matched1.decam_flux_ivar[:,iband] > 0) *
                (matched2.decam_flux_ivar[:,iband] > 0))
        K = np.flatnonzero(good)
        psf1 = (matched1.type == 'PSF ')
        psf2 = (matched2.type == 'PSF ')
        P = np.flatnonzero(good * psf1 * psf2)

        mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
            matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
        
        iv1 = matched1.decam_flux_ivar[:, iband]
        iv2 = matched2.decam_flux_ivar[:, iband]
        std = np.sqrt(1./iv1 + 1./iv2)
        
        plt.clf()
        plt.plot(mag1[K],
                 (matched2.decam_flux[K,iband] - matched1.decam_flux[K,iband]) / std[K],
                 '.', alpha=0.1, color=cc)
        plt.plot(mag1[P],
                 (matched2.decam_flux[P,iband] - matched1.decam_flux[P,iband]) / std[P],
                 '.', alpha=0.1, color='k')
        plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' % (name2, name1, band))
        plt.xlabel('%s mag: %s' % (name1, band))
        plt.axhline(0, color='k', alpha=0.5)
        plt.axis([24, 16, -10, 10])
        plt.title(tt)
        ps.savefig()

    plt.clf()
    lp,lt = [],[]
    for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
        good = ((matched1.decam_flux_ivar[:,iband] > 0) *
                (matched2.decam_flux_ivar[:,iband] > 0))
        #good = True
        psf1 = (matched1.type == 'PSF ')
        psf2 = (matched2.type == 'PSF ')
        mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
            matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
        iv1 = matched1.decam_flux_ivar[:, iband]
        iv2 = matched2.decam_flux_ivar[:, iband]
        std = np.sqrt(1./iv1 + 1./iv2)
        #std = np.hypot(std, 0.01)
        G = np.flatnonzero(good * psf1 * psf2 *
                           np.isfinite(mag1) *
                           (mag1 >= 20) * (mag1 < dict(g=24, r=23.5, z=22.5)[band]))
        
        n,b,p = plt.hist((matched2.decam_flux[G,iband] -
                          matched1.decam_flux[G,iband]) / std[G],
                 range=(-4, 4), bins=50, histtype='step', color=cc,
                 normed=True)

        sig = (matched2.decam_flux[G,iband] -
               matched1.decam_flux[G,iband]) / std[G]
        print('Raw mean and std of points:', np.mean(sig), np.std(sig))
        med = np.median(sig)
        rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2.
        print('Median and percentile-based sigma:', med, rsigma)
        lp.append(p[0])
        lt.append('%s: %.2f +- %.2f' % (band, med, rsigma))
        
    bins = []
    gaussint = []
    for blo,bhi in zip(b, b[1:]):
        c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
        c /= (bhi - blo)
        #bins.extend([blo,bhi])
        #gaussint.extend([c,c])
        bins.append((blo+bhi)/2.)
        gaussint.append(c)
    plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
    
    plt.title(tt)
    plt.xlabel('Flux difference / error (sigma)')
    plt.axvline(0, color='k', alpha=0.1)
    plt.ylim(0, 0.45)
    plt.legend(lp, lt, loc='upper right')
    ps.savefig()
        
        
    for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]:
        plt.clf()
        mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors(
            matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband])
        mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors(
            matched2.decam_flux[:,iband], matched2.decam_flux_ivar[:,iband])

        meanmag = NanoMaggies.nanomaggiesToMag((
            matched1.decam_flux[:,iband] + matched2.decam_flux[:,iband]) / 2.)

        psf1 = (matched1.type == 'PSF ')
        psf2 = (matched2.type == 'PSF ')
        good = ((matched1.decam_flux_ivar[:,iband] > 0) *
                (matched2.decam_flux_ivar[:,iband] > 0) *
                np.isfinite(mag1) * np.isfinite(mag2))
        K = np.flatnonzero(good)
        P = np.flatnonzero(good * psf1 * psf2)
        
        plt.errorbar(mag1[K], mag2[K], fmt='.', color=cc,
                     xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
        plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5)
        plt.xlabel('%s %s (mag)' % (name1, band))
        plt.ylabel('%s %s (mag)' % (name2, band))
        plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.)
        plt.axis([24, 16, 24, 16])
        plt.title(tt)
        ps.savefig()

        plt.clf()
        plt.errorbar(mag1[K], mag2[K] - mag1[K], fmt='.', color=cc,
                     xerr=magerr1[K], yerr=magerr2[K], alpha=0.1)
        plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5)
        plt.xlabel('%s %s (mag)' % (name1, band))
        plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band))
        plt.axhline(0., color='k', alpha=1.)
        plt.axis([24, 16, -1, 1])
        plt.title(tt)
        ps.savefig()

        magbins = np.arange(16, 24.001, 0.5)
        
        plt.clf()
        plt.plot(mag1[K], (mag2[K]-mag1[K]) / np.hypot(magerr1[K], magerr2[K]),
                     '.', color=cc, alpha=0.1)
        plt.plot(mag1[P], (mag2[P]-mag1[P]) / np.hypot(magerr1[P], magerr2[P]),
                     'k.', alpha=0.5)

        plt.xlabel('%s %s (mag)' % (name1, band))
        plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
                   (name2, band, name1, band))
        plt.axhline(0., color='k', alpha=1.)
        plt.axis([24, 16, -10, 10])
        plt.title(tt)
        ps.savefig()

        y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
        
        plt.clf()
        plt.plot(meanmag[P], y[P], 'k.', alpha=0.1)

        midmag = []
        vals = np.zeros((len(magbins)-1, 5))
        median_err1 = []
        
        iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25)

        # FIXME -- should we do some stats after taking off the mean difference?
        
        for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])):
            I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
            midmag.append((mlo+mhi)/2.)
            median_err1.append(np.median(magerr1[I]))
            if len(I) == 0:
                continue
            # median and +- 1 sigma quantiles
            ybin = y[I]
            vals[bini,0] = np.percentile(ybin, 16)
            vals[bini,1] = np.median(ybin)
            vals[bini,2] = np.percentile(ybin, 84)
            # +- 2 sigma quantiles
            vals[bini,3] = np.percentile(ybin, 2.3)
            vals[bini,4] = np.percentile(ybin, 97.7)

            iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
            
            print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss,
                  'vs expected for Gaussian;', len(ybin), 'points')

            # if iqd > iqd_gauss:
            #     # What error adding in quadrature would you need to make the IQD match?
            #     err = median_err1[-1]
            #     target_err = err * (iqd / iqd_gauss)
            #     sys_err = np.sqrt(target_err**2 - err**2)
            #     print('--> add systematic error', sys_err)

        # ~ Johan's cuts
        mlo = 21.
        mhi = dict(g=24., r=23.5, z=22.5)[band]
        I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)]
        ybin = y[I]
        iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25)
        print('Mag bin', mlo, mhi, 'band', band, ': IQD is factor',
              iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points')
        if iqd > iqd_gauss:
            # What error adding in quadrature would you need to make
            # the IQD match?
            err = np.median(np.hypot(magerr1[I], magerr2[I]))
            print('Median error (hypot):', err)
            target_err = err * (iqd / iqd_gauss)
            print('Target:', target_err)
            sys_err = np.sqrt((target_err**2 - err**2) / 2.)
            print('--> add systematic error', sys_err)

            # check...
            err_sys = np.hypot(np.hypot(magerr1, sys_err),
                               np.hypot(magerr2, sys_err))
            ysys = (mag2 - mag1) / err_sys
            ysys = ysys[I]
            print('Resulting median error:', np.median(err_sys[I]))
            iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25)
            print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian')
            # Hmmm, this doesn't work... totally overshoots.
            
            
        plt.errorbar(midmag, vals[:,1], fmt='o', color='b',
                     yerr=(vals[:,1]-vals[:,0], vals[:,2]-vals[:,1]),
                     capthick=3, zorder=20)
        plt.errorbar(midmag, vals[:,1], fmt='o', color='b',
                     yerr=(vals[:,1]-vals[:,3], vals[:,4]-vals[:,1]),
                     capthick=2, zorder=20)
        plt.axhline( 1., color='b', alpha=0.2)
        plt.axhline(-1., color='b', alpha=0.2)
        plt.axhline( 2., color='b', alpha=0.2)
        plt.axhline(-2., color='b', alpha=0.2)

        for mag,err,y in zip(midmag, median_err1, vals[:,3]):
            if not np.isfinite(err):
                continue
            if y < -6:
                continue
            plt.text(mag, y-0.1, '%.3f' % err, va='top', ha='center', color='k',
                     fontsize=10)
        
        plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band))
        plt.ylabel('(%s %s - %s %s) / errors (sigma)' %
                   (name2, band, name1, band))
        plt.axhline(0., color='k', alpha=1.)

        plt.axvline(21, color='k', alpha=0.3)
        plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3)

        plt.axis([24.1, 16, -6, 6])
        plt.title(tt)
        ps.savefig()

        #magbins = np.append([16, 18], np.arange(20, 24.001, 0.5))
        if band == 'g':
            magbins = [20, 24]
        elif band == 'r':
            magbins = [20, 23.5]
        elif band == 'z':
            magbins = [20, 22.5]

        slo,shi = -5,5
        plt.clf()
        ha = dict(bins=25, range=(slo,shi), histtype='step', normed=True)
        y = (mag2 - mag1) / np.hypot(magerr1, magerr2)
        midmag = []
        nn = []
        rgbs = []
        lt,lp = [],[]
        for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])):
            I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)]
            if len(I) == 0:
                continue
            ybin = y[I]
            rgb = [0.,0.,0.]
            rgb[0] = float(bini) / (len(magbins)-1)
            rgb[2] = 1. - rgb[0]
            n,b,p = plt.hist(ybin, color=rgb, **ha)
            lt.append('mag %g to %g' % (mlo,mhi))
            lp.append(p[0])
            midmag.append((mlo+mhi)/2.)
            nn.append(n)
            rgbs.append(rgb)
            
        bins = []
        gaussint = []
        for blo,bhi in zip(b, b[1:]):
            #midbin.append((blo+bhi)/2.)
            #gaussint.append(scipy.stats.norm.cdf(bhi) -
            #                scipy.stats.norm.cdf(blo))
            c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo)
            c /= (bhi - blo)
            bins.extend([blo,bhi])
            gaussint.extend([c,c])
        plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5)
            
        plt.legend(lp, lt)
        plt.title(tt)
        plt.xlim(slo,shi)
        ps.savefig()

        bincenters = b[:-1] + (b[1]-b[0])/2.
        plt.clf()
        lp = []
        for n,rgb,mlo,mhi in zip(nn, rgbs, magbins, magbins[1:]):
            p = plt.plot(bincenters, n, '-', color=rgb)
            lp.append(p[0])
        plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2)
        plt.legend(lp, lt)
        plt.title(tt)
        plt.xlim(slo,shi)
        ps.savefig()
Example #19
0
def main():
    global survey

    init()

    ps = PlotSequence('sky')
    # export LEGACY_SURVEY_DIR=/scratch1/scratchdirs/desiproc/DRs/dr4-bootes/legacypipe-dir/
    #survey = LegacySurveyData()
    survey = get_survey('dr4v2')
    ccds = survey.get_ccds_readonly()
    print(len(ccds), 'CCDs')
    ccds = ccds[ccds.camera == 'mosaic']
    print(len(ccds), 'Mosaic CCDs')
    # plt.clf()
    # plt.hist(ccds.mjd_obs % 1.0, bins=50)
    # plt.xlabel('MJD mod 1')
    # ps.savefig()

    ccds.imjd = np.floor(ccds.mjd_obs).astype(int)

    mjds = np.unique(ccds.imjd)
    print(len(mjds), 'unique MJDs')

    mp = multiproc(nthreads=8, init=init)

    allvals = []
    medmjds = []

    args = []

    for kk, mjd in enumerate(mjds):
        I = np.flatnonzero(ccds.imjd == mjd)
        print('MJD', mjd, ' (%i of %i):' % (kk + 1, len(mjds)), len(I), 'CCDs')
        if len(I) == 0:
            continue

        # pick one near the middle
        #i = I[len(I)/2]
        for i in [I[len(I) / 4], I[len(I) / 2], I[3 * len(I) / 4]]:

            ccd = ccds[i]

            key = dict(expnum=ccd.expnum, ccdname=ccd.ccdname)
            oldvals = key

            vals = cache.find(key)
            print('Got', vals.count(), 'cache hits for', key)
            gotone = False
            for val in vals:
                #if 'median_adu' in val:
                if 'mode_adu' in val:
                    print('cache hit:', val)
                    allvals.append(val)
                    medmjds.append(mjd)
                    gotone = True
                    break
                else:
                    print('partial cache hit:', val)
                    oldvals = val
                    ###!
                    cache.delete_one(val)
            if gotone:
                continue

            print('args: key', key, 'oldvals', oldvals)
            args.append((mjd, key, oldvals, ccd))

            # plt.clf()
            # plt.hist(tim.getImage().ravel(), range=(-0.1, 0.1), bins=50,
            #          histtype='step', color='b')
            # plt.axvline(med, color='b', alpha=0.3, lw=2)
            # plt.axvline(0., color='k', alpha=0.3, lw=2)
            # plt.xlabel('Sky-subtracted pixel values')
            # plt.title('Date ' + ccd.date_obs + ': ' + str(im))
            # ps.savefig()

    if len(args):
        meds = mp.map(read_sky_val, args)
        print('Medians:', meds)
        for (mjd, key, oldvals, ccd), val in zip(args, meds):
            if val is None:
                continue
            allvals.append(val)
            medmjds.append(mjd)

    medians = []
    median_adus = []
    mode_adus = []
    skyadus = []
    keepmjds = []

    for mjd, val in zip(medmjds, allvals):
        madu = val['median_adu']
        if madu is None:
            continue

        if 'median' in val:
            medians.append(val['median'])
        else:
            from tractor.brightness import NanoMaggies
            zpscale = NanoMaggies.zeropointToScale(val['ccdzpt'])
            med = (val['median_adu'] - val['skyadu']) / zpscale
            print('Computed median diff:', med, 'nmgy')
            medians.append(med)

        keepmjds.append(mjd)
        median_adus.append(val['median_adu'])
        mode_adus.append(val['mode_adu'])
        skyadus.append(val['skyadu'])

    medmjds = keepmjds

    median_adus = np.array(median_adus)
    mode_adus = np.array(mode_adus)
    skyadus = np.array(skyadus)
    medmjds = np.array(medmjds)
    medians = np.array(medians)

    plt.clf()
    plt.plot(medmjds, median_adus - skyadus, 'b.')
    plt.xlabel('MJD')
    #plt.ylabel('Image median after sky subtraction (nmgy)')
    plt.ylabel('Image median - SKYADU (ADU)')
    #plt.ylim(-0.03, 0.03)
    #plt.ylim(-10, 10)
    plt.ylim(-1, 1)

    plt.axhline(0, color='k', lw=2, alpha=0.5)
    ps.savefig()

    plt.clf()
    plt.plot(medmjds, mode_adus - skyadus, 'b.')
    plt.xlabel('MJD')
    plt.ylabel('Image mode - SKYADU (ADU)')
    plt.ylim(-1, 1)
    plt.axhline(0, color='k', lw=2, alpha=0.5)
    ps.savefig()

    print('Median pcts:', np.percentile(medians, [0, 2, 50, 98, 100]))
    mlo, mhi = np.percentile(medians, [2, 98])
    I = np.flatnonzero((medians > mlo) * (medians < mhi))

    d0 = np.mean(medmjds)
    dmjd = medmjds[I] - d0
    A = np.zeros((len(dmjd), 2))
    A[:, 0] = 1.
    A[:, 1] = dmjd
    b = np.linalg.lstsq(A, medians[I])[0]
    print('Lstsq:', b)
    offset = b[0]
    slope = b[1]
    xx = np.array([dmjd.min(), dmjd.max()])

    print('Offset', offset)
    print('Slope', slope)

    plt.clf()
    plt.plot(medmjds, medians, 'b.')
    ax = plt.axis()
    plt.plot(xx + d0, offset + slope * xx, 'b-')
    plt.axis(ax)
    plt.xlabel('MJD')
    plt.ylabel('Image median after sky subtraction (nmgy)')
    plt.ylim(-0.03, 0.03)
    plt.axhline(0, color='k', lw=2, alpha=0.5)
    ps.savefig()

    plt.clf()
    plt.plot(median_adus, skyadus, 'b.')
    ax = plt.axis()
    lo = min(ax[0], ax[2])
    hi = max(ax[1], ax[3])
    plt.plot([lo, hi], [lo, hi], 'k-', alpha=0.5)
    plt.xlabel('Median image ADU')
    plt.ylabel('SKYADU')
    plt.axis(ax)
    ps.savefig()

    plt.clf()
    plt.plot(medmjds, skyadus / median_adus, 'b.')
    plt.xlabel('MJD')
    plt.ylim(0.98, 1.02)
    plt.axhline(1.0, color='k', alpha=0.5)
    plt.ylabel('SKYADU / median image ADU')
    ps.savefig()