def rbmain(): travis = 'travis' in sys.argv extra_args = [ '--old-calibs-ok', #'--verbose', ] if travis: extra_args.extend( ['--no-wise-ceres', '--no-gaia', '--no-large-galaxies']) if 'ceres' in sys.argv: surveydir = os.path.join(os.path.dirname(__file__), 'testcase3') main(args=[ '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815', '--no-wise', '--force-all', '--no-write', '--ceres', '--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres', '--no-depth-cut' ]) sys.exit(0) # demo RexGalaxy, with plots if False: from legacypipe.survey import RexGalaxy from tractor import NanoMaggies, PixPos from tractor import Image, GaussianMixturePSF, LinearPhotoCal from legacypipe.survey import LogRadius rex = RexGalaxy(PixPos(1., 2.), NanoMaggies(r=3.), LogRadius(0.)) print('Rex:', rex) print('Rex params:', rex.getParams()) print('Rex nparams:', rex.numberOfParams()) H, W = 100, 100 tim = Image(data=np.zeros((H, W), np.float32), inverr=np.ones((H, W), np.float32), psf=GaussianMixturePSF(1., 0., 0., 4., 4., 0.), photocal=LinearPhotoCal(1., band='r')) derivs = rex.getParamDerivatives(tim) print('Derivs:', len(derivs)) print('Rex params:', rex.getParamNames()) import pylab as plt from astrometry.util.plotutils import PlotSequence ps = PlotSequence('rex') for d, nm in zip(derivs, rex.getParamNames()): plt.clf() plt.imshow(d.patch, interpolation='nearest', origin='lower') plt.title('Derivative %s' % nm) ps.savefig() sys.exit(0) # Test RexGalaxy surveydir = os.path.join(os.path.dirname(__file__), 'testcase6') outdir = 'out-testcase6-rex' main(args=[ '--brick', '1102p240', '--zoom', '500', '600', '650', '750', '--force-all', '--no-write', '--no-wise', #'--rex', #'--plots', '--survey-dir', surveydir, '--outdir', outdir ] + extra_args) fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits') assert (os.path.exists(fn)) T = fits_table(fn) assert (len(T) == 2) print('Types:', T.type) # Since there is a Tycho-2 star in the blob, forced to be PSF. assert (T.type[0] == 'PSF ') cmd = ( '(cd %s && sha256sum -c %s)' % (outdir, os.path.join('tractor', '110', 'brick-1102p240.sha256sum'))) print(cmd) rtn = os.system(cmd) assert (rtn == 0) # Test with a Tycho-2 star in the blob. surveydir = os.path.join(os.path.dirname(__file__), 'testcase6') outdir = 'out-testcase6' main(args=[ '--brick', '1102p240', '--zoom', '500', '600', '650', '750', '--force-all', '--no-write', '--no-wise', '--survey-dir', surveydir, '--outdir', outdir ] + extra_args) fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits') assert (os.path.exists(fn)) T = fits_table(fn) assert (len(T) == 2) print('Types:', T.type) # Since there is a Tycho-2 star in the blob, forced to be PSF. assert (T.type[0] == 'PSF ') # Test that we can run splinesky calib if required... from legacypipe.decam import DecamImage DecamImage.splinesky_boxsize = 128 surveydir = os.path.join(os.path.dirname(__file__), 'testcase4') outdir = 'out-testcase4' fn = os.path.join(surveydir, 'calib', 'decam', 'splinesky', '00431', '00431608', 'decam-00431608-N3.fits') if os.path.exists(fn): os.unlink(fn) main(args=[ '--brick', '1867p255', '--zoom', '2050', '2300', '1150', '1400', '--force-all', '--no-write', '--coadd-bw', '--unwise-dir', os.path.join(surveydir, 'images', 'unwise'), '--unwise-tr-dir', os.path.join(surveydir, 'images', 'unwise-tr'), '--blob-image', '--no-hybrid-psf', '--survey-dir', surveydir, '--outdir', outdir ] + extra_args + ['-v']) print('Checking for calib file', fn) assert (os.path.exists(fn)) # Wrap-around, hybrid PSF surveydir = os.path.join(os.path.dirname(__file__), 'testcase8') outdir = 'out-testcase8' main(args=[ '--brick', '1209p050', '--zoom', '720', '1095', '3220', '3500', '--force-all', '--no-write', '--no-wise', #'--plots', '--survey-dir', surveydir, '--outdir', outdir ] + extra_args) # Test with a Tycho-2 star + another saturated star in the blob. surveydir = os.path.join(os.path.dirname(__file__), 'testcase7') outdir = 'out-testcase7' # remove --no-gaia my_extra_args = [a for a in extra_args if a != '--no-gaia'] os.environ['GAIA_CAT_DIR'] = os.path.join(surveydir, 'gaia-dr2') os.environ['GAIA_CAT_VER'] = '2' main(args=[ '--brick', '1102p240', '--zoom', '250', '350', '1550', '1650', '--force-all', '--no-write', '--no-wise', #'--plots', '--survey-dir', surveydir, '--outdir', outdir ] + my_extra_args) del os.environ['GAIA_CAT_DIR'] del os.environ['GAIA_CAT_VER'] fn = os.path.join(outdir, 'tractor', '110', 'tractor-1102p240.fits') assert (os.path.exists(fn)) T = fits_table(fn) assert (len(T) == 4) # Check skipping blobs outside the brick's unique area. # (this now doesn't detect any sources at all, reasonably) # surveydir = os.path.join(os.path.dirname(__file__), 'testcase5') # outdir = 'out-testcase5' # # fn = os.path.join(outdir, 'tractor', '186', 'tractor-1867p255.fits') # if os.path.exists(fn): # os.unlink(fn) # # main(args=['--brick', '1867p255', '--zoom', '0', '150', '0', '150', # '--force-all', '--no-write', '--coadd-bw', # '--survey-dir', surveydir, # '--early-coadds', # '--outdir', outdir] + extra_args) # # assert(os.path.exists(fn)) # T = fits_table(fn) # assert(len(T) == 1) # Custom RA,Dec; blob ra,dec. surveydir = os.path.join(os.path.dirname(__file__), 'testcase4') outdir = 'out-testcase4b' # Catalog written with one entry (--blobradec) fn = os.path.join(outdir, 'tractor', 'cus', 'tractor-custom-186743p25461.fits') if os.path.exists(fn): os.unlink(fn) main(args=[ '--radec', '186.743965', '25.461788', '--width', '250', '--height', '250', '--force-all', '--no-write', '--no-wise', '--blobradec', '186.740369', '25.453855', '--survey-dir', surveydir, '--outdir', outdir ] + extra_args) assert (os.path.exists(fn)) T = fits_table(fn) assert (len(T) == 1) surveydir = os.path.join(os.path.dirname(__file__), 'testcase3') outdir = 'out-testcase3' checkpoint_fn = os.path.join(outdir, 'checkpoint.pickle') if os.path.exists(checkpoint_fn): os.unlink(checkpoint_fn) main(args=[ '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815', '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir, '--outdir', outdir, '--checkpoint', checkpoint_fn, '--checkpoint-period', '1', '--threads', '2' ] + extra_args) # Read catalog into Tractor sources to test read_fits_catalog from legacypipe.catalog import read_fits_catalog from legacypipe.survey import LegacySurveyData, GaiaSource from tractor.galaxy import DevGalaxy from tractor import PointSource survey = LegacySurveyData(survey_dir=outdir) fn = survey.find_file('tractor', brick='2447p120') print('Checking', fn) T = fits_table(fn) cat = read_fits_catalog(T, fluxPrefix='') print('Read catalog:', cat) assert (len(cat) == 2) src = cat[0] assert (type(src) == DevGalaxy) assert (np.abs(src.pos.ra - 244.77973) < 0.00001) assert (np.abs(src.pos.dec - 12.07233) < 0.00001) src = cat[1] print('Source', src) assert (type(src) in [PointSource, GaiaSource]) assert (np.abs(src.pos.ra - 244.77830) < 0.00001) assert (np.abs(src.pos.dec - 12.07250) < 0.00001) # DevGalaxy(pos=RaDecPos[244.77975494973529, 12.072348111713127], brightness=NanoMaggies: g=19.2, r=17.9, z=17.1, shape=re=2.09234, e1=-0.198453, e2=0.023652, # PointSource(RaDecPos[244.77833280764278, 12.072521274981987], NanoMaggies: g=25, r=23, z=21.7) # Check that we can run again, using that checkpoint file. main(args=[ '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815', '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir, '--outdir', outdir, '--checkpoint', checkpoint_fn, '--checkpoint-period', '1', '--threads', '2' ] + extra_args) # Assert...... something? # Test --checkpoint without --threads main(args=[ '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815', '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir, '--outdir', outdir, '--checkpoint', checkpoint_fn, '--checkpoint-period', '1' ] + extra_args) # MzLS + BASS data # surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass') # main(args=['--brick', '3521p002', '--zoom', '2400', '2450', '1200', '1250', # '--no-wise', '--force-all', '--no-write', # '--survey-dir', surveydir2, # '--outdir', 'out-mzlsbass']) # From Kaylan's Bootes pre-DR4 run # surveydir2 = os.path.join(os.path.dirname(__file__), 'mzlsbass3') # main(args=['--brick', '2173p350', '--zoom', '100', '200', '100', '200', # '--no-wise', '--force-all', '--no-write', # '--survey-dir', surveydir2, # '--outdir', 'out-mzlsbass3'] + extra_args) # With plots! main(args=[ '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815', '--no-wise', '--force-all', '--no-write', '--survey-dir', surveydir, '--outdir', 'out-testcase3', '--plots', '--nblobs', '1' ] + extra_args) # Decals Image Simulations # Uncomment WHEN galsim build for Travis #os.environ["DECALS_SIM_DIR"]= os.path.join(os.path.dirname(__file__),'image_sims') #brick= '2447p120' #sim_main(args=['--brick', brick, '-n', '2', '-o', 'STAR', \ # '-ic', '1', '--rmag-range', '18', '26', '--threads', '1',\ # '--zoom', '1020', '1070', '2775', '2815']) # Check if correct files written out #rt_dir= os.path.join(os.getenv('DECALS_SIM_DIR'),brick,'star','001') #assert( os.path.exists(os.path.join(rt_dir,'../','metacat-'+brick+'-star.fits')) ) #for fn in ['tractor-%s-star-01.fits' % brick,'simcat-%s-star-01.fits' % brick]: # assert( os.path.exists(os.path.join(rt_dir,fn)) ) #for fn in ['image','model','resid','simscoadd']: # assert( os.path.exists(os.path.join(rt_dir,'qa-'+brick+'-star-'+fn+'-01.jpg')) ) if not travis: # With ceres main(args=[ '--brick', '2447p120', '--zoom', '1020', '1070', '2775', '2815', '--no-wise', '--force-all', '--no-write', '--ceres', '--survey-dir', surveydir, '--outdir', 'out-testcase3-ceres' ] + extra_args)
def forced2(): from bigboss_test import radecroi ps = PlotSequence('forced') basedir = os.environ.get('BIGBOSS_DATA', '/project/projectdirs/bigboss') wisedatadir = os.path.join(basedir, 'data', 'wise') l1bdir = os.path.join(wisedatadir, 'level1b') wisecat = fits_table(os.path.join( wisedatadir, 'catalogs', 'wisecat2.fits')) # CAS PhotoObjAll.resolveStatus bits sprim = 0x100 #sbad = 0x800 sedge = 0x1000 sbest = 0x200 (ra0, ra1, dec0, dec1) = radecroi ra = (ra0 + ra1) / 2. dec = (dec0 + dec1) / 2. cas = fits_table('sdss-cas-testarea-3.fits') print('Read', len(cas), 'CAS sources') cas.cut((cas.resolvestatus & sedge) == 0) print('Cut to ', len(cas), 'without SURVEY_EDGE set') # Drop "sbest" sources that have an "sprim" nearby. Ibest = (cas.resolvestatus & (sprim | sbest)) == sbest Iprim = (cas.resolvestatus & (sprim | sbest)) == sprim I, J, d = match_radec( cas.ra[Ibest], cas.dec[Ibest], cas.ra[Iprim], cas.dec[Iprim], 2. / 3600.) Ibest[np.flatnonzero(Ibest)[I]] = False #Ikeep = np.ones(len(Ibest), bool) #Ikeep[I] = False cas.cut(np.logical_or(Ibest, Iprim)) print('Cut to', len(cas), 'PRIMARY + BEST-not-near-PRIMARY') I, J, d = match_radec(cas.ra, cas.dec, cas.ra, cas.dec, 2. / 3600., notself=True) plt.clf() loghist((cas.ra[I] - cas.ra[J]) * 3600., (cas.dec[I] - cas.dec[J]) * 3600., 200) plt.title('CAS self-matches') ps.savefig() psf = pyfits.open('wise-psf-w1-500-500.fits')[0].data S = psf.shape[0] # number of Gaussian components K = 3 w, mu, sig = em_init_params(K, None, None, None) II = psf.copy() II = np.maximum(II, 0) II /= II.sum() xm, ym = -(S / 2), -(S / 2) res = em_fit_2d(II, xm, ym, w, mu, sig) if res != 0: raise RuntimeError('Failed to fit PSF') print('W1 PSF:') print(' w', w) print(' mu', mu) print(' sigma', sig) w1psf = GaussianMixturePSF(w, mu, sig) w1psf.computeRadius() print('PSF radius:', w1psf.getRadius(), 'pixels') T = fits_table('wise-roi.fits') for i in range(len(T)): basefn = os.path.join(l1bdir, '%s%i-w1' % (T.scan_id[i], T.frame_num[i])) fn = basefn + '-int-1b.fits' print('Looking for', fn) if not os.path.exists(fn): continue print(' -> Found it!') tim = read_wise_image(basefn, nanomaggies=True) tim.psf = w1psf wcs = tim.wcs.wcs r0, r1, d0, d1 = wcs.radec_bounds() print('RA,Dec bounds:', r0, r1, d0, d1) w, h = wcs.imagew, wcs.imageh rd = np.array([wcs.pixelxy2radec(x, y) for x, y in [(1, 1), (w, 1), (w, h), (1, h), (1, 1)]]) I = np.flatnonzero((cas.ra > r0) * (cas.ra < r1) * (cas.dec > d0) * (cas.dec < d1)) J = point_in_poly(cas.ra[I], cas.dec[I], rd) I = I[J] cashere = cas[I] # 10-20k sources... wbands = ['w1'] sdssband = 'i' tsrcs = get_tractor_sources_cas_dr9(cashere, nanomaggies=True, bandname=sdssband, bands=[ sdssband], extrabands=wbands) #keepsrcs = [] for src in tsrcs: for br in src.getBrightnesses(): f = br.getBand(sdssband) # if f < 0: # continue for wb in wbands: br.setBand(wb, f) # keepsrcs.append(src) #tsrcs = keepsrcs print('Created', len(tsrcs), 'tractor sources in this image') I = np.flatnonzero((wisecat.ra > r0) * (wisecat.ra < r1) * (wisecat.dec > d0) * (wisecat.dec < d1)) J = point_in_poly(wisecat.ra[I], wisecat.dec[I], rd) I = I[J] print('Found', len(I), 'WISE catalog sources in this image') wc = wisecat[I] tra = np.array([src.getPosition().ra for src in tsrcs]) tdec = np.array([src.getPosition().dec for src in tsrcs]) R = 4. I, J, d = match_radec(wc.ra, wc.dec, tra, tdec, R / 3600., nearest=True) # cashere.ra, cashere.dec, print('Found', len(I), 'SDSS-WISE matches within', R, 'arcsec') for i, j in zip(I, J): w1 = wc.w1mpro[i] w1 = NanoMaggies.magToNanomaggies(w1) bb = tsrcs[j].getBrightnesses() for b in bb: b.setBand('w1', w1 / float(len(bb))) keepsrcs = [] for src in tsrcs: # for b in src.getBrightness(): b = src.getBrightness() if b.getBand(sdssband) > 0 or b.getBand(wbands[0]) > 0: keepsrcs.append(src) tsrcs = keepsrcs print('Keeping', len(tsrcs), 'tractor sources from SDSS') unmatched = np.ones(len(wc), bool) unmatched[I] = False wun = wc[unmatched] print(len(wun), 'unmatched WISE sources') for i in range(len(wun)): pos = RaDecPos(wun.ra[i], wun.dec[i]) nm = NanoMaggies.magToNanomaggies(wun.w1mpro[i]) br = NanoMaggies(i=25., w1=nm) tsrcs.append(PointSource(pos, br)) plt.clf() plt.plot(rd[:, 0], rd[:, 1], 'k-') plt.plot(cashere.ra, cashere.dec, 'r.', alpha=0.1) plt.plot(wc.ra, wc.dec, 'bx', alpha=0.1) setRadecAxes(r0, r1, d0, d1) ps.savefig() zlo, zhi = tim.zr ima = dict(interpolation='nearest', origin='lower', vmin=zlo, vmax=zhi) imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5) plt.clf() plt.imshow(tim.getImage(), **ima) plt.hot() plt.title(tim.name + ': data') ps.savefig() wsrcs = [] for i in range(len(wc)): pos = RaDecPos(wc.ra[i], wc.dec[i]) nm = NanoMaggies.magToNanomaggies(wc.w1mpro[i]) br = NanoMaggies(i=25., w1=nm) wsrcs.append(PointSource(pos, br)) tr = Tractor([tim], wsrcs) tr.freezeParam('images') for jj in range(2): print('Rendering WISE model image...') wmod = tr.getModelImage(0) plt.clf() plt.imshow(wmod, **ima) plt.hot() plt.title(tim.name + ': WISE sources only') ps.savefig() assert(np.all(np.isfinite(wmod))) assert(np.all(np.isfinite(tim.getInvError()))) assert(np.all(np.isfinite(tim.getImage()))) wchi = tr.getChiImage(0) plt.clf() plt.imshow(wchi, **imchi) plt.title(tim.name + ': chi, WISE sources only') plt.gray() ps.savefig() if jj == 1: break tr.optimize() tr = Tractor([tim], tsrcs) print('Rendering model image...') mod = tr.getModelImage(0) plt.clf() plt.imshow(mod, **ima) plt.title(tim.name + ': SDSS + WISE sources') ps.savefig() print('tim', tim) print('tim.photocal:', tim.photocal) wsrcs = [] for i in range(len(wc)): pos = RaDecPos(wc.ra[i], wc.dec[i]) nm = NanoMaggies.magToNanomaggies(wc.w1mpro[i]) br = NanoMaggies(i=25., w1=nm) wsrcs.append(PointSource(pos, br)) tr = Tractor([tim], wsrcs) print('Rendering WISE model image...') wmod = tr.getModelImage(0) plt.clf() plt.imshow(wmod, **ima) plt.title(tim.name + ': WISE sources only') ps.savefig()
def main(decals=None, opt=None): '''Driver function for forced photometry of individual DECam images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0,x1,y0,y1) = opt.zoom zoomslice = (slice(y0,y1), slice(x0,x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.filename) opt.filename = None except: # make this 'None' for decals.find_ccds() expnum = None # Try parsing HDU number try: opt.hdu = int(opt.hdu) ccdname = None except: ccdname = opt.hdu opt.hdu = -1 if decals is None: decals = Decals() if opt.filename is not None and opt.hdu >= 0: # Read metadata from file T = exposure_metadata([opt.filename], hdus=[opt.hdu]) print('Metadata:') T.about() else: # Read metadata from decals-ccds.fits table T = decals.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if opt.hdu >= 0: T.cut(T.image_hdu == opt.hdu) print(len(T), 'with HDU', opt.hdu) if opt.filename is not None: T.cut(np.array([f.strip() == opt.filename for f in T.image_filename])) print(len(T), 'with filename', opt.filename) assert(len(T) == 1) im = decals.get_image_object(T[0]) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True) print('Got tim:', tim) if opt.catfn in ['DR1', 'DR2']: if opt.catalog_path is None: opt.catalog_path = opt.catfn.lower() margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, decals=decals) for b in bricks: # there is some overlap with this brick... read the catalog. fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3], 'tractor-%s.fits' % b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec) W,H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) * (yy >= -margin) * (yy <= (H+margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') T.cut((T.out_of_bounds == False) * (T.left_blob == False)) print('Cut to', len(T), 'on out_of_bounds and left_blob') TT.append(T) T = merge_tables(TT) T._header = TT[0]._header del TT # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE) # print('Got cat:', cat) print('Forced photom...') opti = None if opt.ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 opti = CeresOptimizer(BW=B, BH=B) tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') for src in cat: src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) F = fits_table() F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)) ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x-1).astype(np.float32) F.y = (y-1).astype(np.float32) if opt.apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() print('Apertures:', apertures, 'pixels') for rad in apertures: aper = photutils.CircularAperture(apxy, rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = ap ap = 1./(np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux_ivar = ap if opt.forced: kwa = {} if opt.plots is None: kwa.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, **kwa) if opt.plots: (data,mod,ie,chi,roi) = R.ims1[0] ima = tim.ima imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5) plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() F.flux = np.array([src.getBrightness().getFlux(tim.band) for src in cat]).astype(np.float32) F.flux_ivar = R.IV.astype(np.float32) F.fracflux = R.fitstats.profracflux.astype(np.float32) F.rchi2 = R.fitstats.prochi2 .astype(np.float32) program_name = sys.argv[0] version_hdr = get_version_header(program_name, decals.decals_dir) # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) fname = os.path.join(d2, d1, fname) print('Trimmed filename to', fname) #version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file')) version_hdr.add_record(dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file')) version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext')) version_hdr.add_record(dict(name='CAMERA', value='DECam', comment='Dark Energy Camera')) version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num')) version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name')) version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record(dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image')) keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV', 'INSTRUME'] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = {'mjd':'sec', 'exptime':'sec', 'flux':'nanomaggy', 'flux_ivar':'1/nanomaggy^2'} columns = F.get_columns() for i,col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) print('Finished forced phot:', Time()-t0) return 0
def showSipSolutions(srcs, wcs0, andDir, x0, y0, W, H, filterName, plotPrefix): ''' srcs: afw Catalog of sources wcs0: original WCS andDir: astrometry_net_data directory ''' imargs = dict(imageSize=(W, H), filterName=filterName, x0=x0, y0=y0) # Set up astrometry_net_data os.environ['ASTROMETRY_NET_DATA_DIR'] = andDir andConfig = measAstrom.AstrometryNetDataConfig() fn = os.path.join(andDir, 'andConfig.py') andConfig.load(fn) # Set up meas_astrom conf = measAstrom.ANetBasicAstrometryConfig(sipOrder=4) ast = measAstrom.ANetBasicAstrometryTask(conf, andConfig, logLevel=Log.DEBUG) # What reference sources are in the original WCS refs = ast.getReferenceSourcesForWcs(wcs0, **imargs) print('Got', len(refs), 'reference objects for initial WCS') # How does a straight TAN solution look? conf2 = measAstrom.ANetBasicAstrometryConfig(sipOrder=4, calculateSip=False) ast2 = measAstrom.ANetBasicAstrometryTask(conf2, andConfig, logLevel=Log.DEBUG) solve = ast2.determineWcs2(srcs, **imargs) tanwcs = solve.tanWcs # How about if we fit a SIP WCS using the *original* WCS? wcs2 = ast.getSipWcsFromWcs(wcs0, (W, H), x0=x0, y0=y0) # (We determineWcs() for a SIP solution below...) # Make some plots in pixel space by pushing ref sources through WCSes rx0, ry0 = [], [] rx2, ry2 = [], [] rx3, ry3 = [], [] for src in refs: xy = wcs0.skyToPixel(src.getCoord()) rx0.append(xy[0]) ry0.append(xy[1]) xy = tanwcs.skyToPixel(src.getCoord()) rx2.append(xy[0]) ry2.append(xy[1]) xy = wcs2.skyToPixel(src.getCoord()) rx3.append(xy[0]) ry3.append(xy[1]) rx0 = np.array(rx0) ry0 = np.array(ry0) rx2 = np.array(rx2) ry2 = np.array(ry2) rx3 = np.array(rx3) ry3 = np.array(ry3) x = np.array([src.getX() for src in srcs]) y = np.array([src.getY() for src in srcs]) from astrometry.libkd.spherematch import match from astrometry.util.plotutils import plothist, PlotSequence ps = PlotSequence(plotPrefix) # Match up various sources... R = 2. II, d = match(np.vstack((x, y)).T, np.vstack((rx0, ry0)).T, R) I = II[:, 0] J = II[:, 1] pa = dict(range=((-R, R), (-R, R))) plt.clf() plothist(x[I] - rx0[J], y[I] - ry0[J], 200, **pa) plt.title('Source positions - Reference positions (initial WCS)') plt.xlabel('delta-X (pixels)') plt.ylabel('delta-Y (pixels)') ps.savefig() II, d = match(np.vstack((x, y)).T, np.vstack((rx2, ry2)).T, R) I = II[:, 0] J = II[:, 1] plt.clf() plothist(x[I] - rx2[J], y[I] - ry2[J], 200, **pa) plt.title('Source positions - Reference positions (TAN WCS)') plt.xlabel('delta-X (pixels)') plt.ylabel('delta-Y (pixels)') ps.savefig() II, d = match(np.vstack((x, y)).T, np.vstack((rx3, ry3)).T, R) I = II[:, 0] J = II[:, 1] plt.clf() plothist(x[I] - rx3[J], y[I] - ry3[J], 200, **pa) plt.title('Source positions - Reference positions (SIP WCS #2)') plt.xlabel('delta-X (pixels)') plt.ylabel('delta-Y (pixels)') ps.savefig() II, d = match(np.vstack((rx0, ry0)).T, np.vstack((rx3, ry3)).T, R) I = II[:, 0] J = II[:, 1] plt.clf() plothist(rx0[I] - rx3[J], ry0[I] - ry3[J], 200, **pa) plt.title( 'Reference positions (Original WCS) - Reference positions (SIP WCS #2)' ) plt.xlabel('delta-X (pixels)') plt.ylabel('delta-Y (pixels)') ps.savefig() matches = solve.tanMatches msx, msy = [], [] mrx, mry = [], [] for m in matches: ref, src = m.first, m.second xy = tanwcs.skyToPixel(ref.getCoord()) mrx.append(xy[0]) mry.append(xy[1]) msx.append(src.getX()) msy.append(src.getY()) plt.clf() plt.plot(x, y, 'r.') plt.plot(msx, msy, 'o', mec='r') plt.plot(rx0, ry0, 'g.') plt.plot(mrx, mry, 'gx') plt.title('TAN matches') ps.savefig() # Get SIP solution (4th order) solve = ast.determineWcs2(srcs, **imargs) wcs1 = solve.sipWcs matches = solve.sipMatches msx, msy = [], [] mrx, mry = [], [] for m in matches: ref, src = m.first, m.second xy = tanwcs.skyToPixel(ref.getCoord()) mrx.append(xy[0]) mry.append(xy[1]) msx.append(src.getX()) msy.append(src.getY()) plt.clf() plt.plot(x, y, 'r.') plt.plot(msx, msy, 'o', mec='r') plt.plot(rx0, ry0, 'g.') plt.plot(mrx, mry, 'gx') plt.title('SIP matches') ps.savefig() rx1, ry1 = [], [] for src in refs: xy = wcs1.skyToPixel(src.getCoord()) rx1.append(xy[0]) ry1.append(xy[1]) rx1 = np.array(rx1) ry1 = np.array(ry1) plt.clf() plt.plot(x, y, 'o', mec='r', mfc='none') plt.plot(rx0, ry0, 'bx') plt.plot(rx1, ry1, 'g+') plt.plot(rx2, ry2, 'mx') plt.plot(rx3, ry3, 'r+') ps.savefig() plt.axis([x0, x0 + 500, y0, y0 + 500]) ps.savefig() II, d = match(np.vstack((x, y)).T, np.vstack((rx1, ry1)).T, R) I = II[:, 0] J = II[:, 1] plt.clf() plothist(x[I] - rx1[J], y[I] - ry1[J], 200, **pa) plt.title('Source positions - Reference positions (SIP WCS)') plt.xlabel('delta-X (pixels)') plt.ylabel('delta-Y (pixels)') ps.savefig()
[-0.21878855, -0.0432496 ], [-0.83365747, -0.13039277]]) sigma = np.array([[[ 7.72925584e-01, 5.23305564e-02], [ 5.23305564e-02, 8.89078473e-01]], [[ 9.84585869e+00, 7.79378820e-01], [ 7.79378820e-01, 8.84764455e+00]], [[ 2.02664489e+02, -8.16667434e-01], [ -8.16667434e-01, 1.87881670e+02]]]) psf = GaussianMixturePSF(w, mu, sigma) data = np.zeros((200, 200)) invvar = np.zeros_like(data) tim = Image(data=data, invvar=invvar, psf=psf) tractor = Tractor([tim], [s]) nn = np.linspace(0.5, 5.5, 12) cols = int(np.ceil(np.sqrt(len(nn)))) rows = int(np.ceil(len(nn) / float(cols))) plt.clf() for i,n in enumerate(nn): s.sersicindex.setValue(n) mod = tractor.getModelImage(0) plt.subplot(rows, cols, i+1) plt.imshow(np.log10(np.maximum(1e-16, mod)), interpolation='nearest', origin='lower') plt.title('index %.2f' % n) ps.savefig()
def main(): ps = PlotSequence('cov') survey = LegacySurveyData() ra,dec = 242.0, 10.2 fn = 'coverage-ccds.fits' if not os.path.exists(fn): ccds = survey.get_ccds() ccds.cut(ccds.filter == 'r') ccds.cut(ccds.propid == '2014B-0404') ccds.cut(np.hypot(ccds.ra_bore - ra, ccds.dec_bore - dec) < 2.5) print(np.unique(ccds.expnum), 'unique exposures') print('propids', np.unique(ccds.propid)) ccds.writeto(fn) else: ccds = fits_table(fn) plt.clf() for e in np.unique(ccds.expnum): I = np.flatnonzero(ccds.expnum == e) plt.plot(ccds.ra[I], ccds.dec[I], '.') ps.savefig() degw = 3.0 pixscale = 10. W = degw * 3600 / 10. H = W hi = 6 cmap = cmap_discretize('jet', hi+1) wcs = Tan(ra, dec, W/2.+0.5, H/2.+0.5, -pixscale/3600., 0., 0., pixscale/3600., float(W), float(H)) r0,d0 = wcs.pixelxy2radec(1,1) r1,d1 = wcs.pixelxy2radec(W,H) extent = [min(r0,r1),max(r0,r1), min(d0,d1),max(d0,d1)] for expnums in [ [348666], [348666,348710, 348686], [348659, 348667, 348658, 348666, 348665, 348669, 348668], None, [348683, 348687, 347333, 348686, 348685, 348692, 348694, 348659, 348667, 348658, 348666, 348665, 348669, 348668, 348707, 348709, 348708, 348710, 348711, 348716, 348717], ]: nexp = np.zeros((H,W), np.uint8) for ccd in ccds: if expnums is not None and not ccd.expnum in expnums: continue ccdwcs = survey.get_approx_wcs(ccd) r,d = ccdwcs.pixelxy2radec(1, 1) ok,x0,y0 = wcs.radec2pixelxy(r, d) r,d = ccdwcs.pixelxy2radec(ccd.width, ccd.height) ok,x1,y1 = wcs.radec2pixelxy(r, d) xlo = np.clip(int(np.round(min(x0,x1))) - 1, 0, W-1) xhi = np.clip(int(np.round(max(x0,x1))) - 1, 0, W-1) ylo = np.clip(int(np.round(min(y0,y1))) - 1, 0, H-1) yhi = np.clip(int(np.round(max(y0,y1))) - 1, 0, H-1) nexp[ylo:yhi+1, xlo:xhi+1] += 1 plt.clf() plt.imshow(nexp, interpolation='nearest', origin='lower', vmin=-0.5, vmax=hi+0.5, cmap=cmap, extent=extent) plt.colorbar(ticks=np.arange(hi+1)) ps.savefig() O = fits_table('obstatus/decam-tiles_obstatus.fits') O.cut(np.hypot(O.ra - ra, O.dec - dec) < 2.5) for p in [1,2,3]: print('Pass', p, 'exposures:', O.r_expnum[O.get('pass') == p]) O.cut(O.get('pass') == 2) print(len(O), 'pass 2 nearby') d = np.hypot(O.ra - ra, O.dec - dec) print('Dists:', d) I = np.flatnonzero(d < 0.5) assert(len(I) == 1) ocenter = O[I[0]] print('Center expnum', ocenter.r_expnum) I = np.flatnonzero(d >= 0.5) O.cut(I) #center = ccds[ccds.expnum == ocenter.r_expnum] #p2 = ccds[ccds. ok,xc,yc = wcs.radec2pixelxy(ocenter.ra, ocenter.dec) xx,yy = np.meshgrid(np.arange(W)+1, np.arange(H)+1) c_d2 = (xc - xx)**2 + (yc - yy)**2 best = np.ones((H,W), bool) for o in O: ok,x,y = wcs.radec2pixelxy(o.ra, o.dec) d2 = (x - xx)**2 + (y - yy)**2 best[d2 < c_d2] = False del d2 del c_d2,xx,yy # plt.clf() # plt.imshow(best, interpolation='nearest', origin='lower', cmap='gray', # vmin=0, vmax=1) # ps.savefig() plt.clf() plt.imshow(nexp * best, interpolation='nearest', origin='lower', vmin=-0.5, vmax=hi+0.5, cmap=cmap, extent=extent) plt.colorbar(ticks=np.arange(hi+1)) ps.savefig() plt.clf() n,b,p = plt.hist(np.clip(nexp[best], 0, hi), range=(-0.5,hi+0.5), bins=hi+1) plt.xlim(-0.5, hi+0.5) ps.savefig() print('b', b) print('n', n) print('fracs', np.array(n) / np.sum(n)) print('pcts', ', '.join(['%.1f' % f for f in 100. * np.array(n)/np.sum(n)]))
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('--name1', help='Name for first data set') parser.add_argument('--name2', help='Name for second data set') parser.add_argument('--plot-prefix', default='compare', help='Prefix for plot filenames; default "%default"') parser.add_argument('--match', default=1.0, help='Astrometric cross-match distance in arcsec') parser.add_argument('dir1', help='First directory to compare') parser.add_argument('dir2', help='Second directory to compare') opt = parser.parse_args() ps = PlotSequence(opt.plot_prefix) name1 = opt.name1 if name1 is None: name1 = os.path.basename(opt.dir1) if not len(name1): name1 = os.path.basename(os.path.dirname(opt.dir1)) name2 = opt.name2 if name2 is None: name2 = os.path.basename(opt.dir2) if not len(name2): name2 = os.path.basename(os.path.dirname(opt.dir2)) tt = 'Comparing %s to %s' % (name1, name2) # regex for tractor-*.fits catalog filename catre = re.compile('tractor-.*.fits') cat1, cat2 = [], [] for basedir, cat in [(opt.dir1, cat1), (opt.dir2, cat2)]: for dirpath, dirnames, filenames in os.walk(basedir, followlinks=True): for fn in filenames: if not catre.match(fn): print('Skipping', fn, 'due to filename') continue fn = os.path.join(dirpath, fn) t = fits_table(fn) print(len(t), 'from', fn) cat.append(t) cat1 = merge_tables(cat1, columns='fillzero') cat2 = merge_tables(cat2, columns='fillzero') print('Total of', len(cat1), 'from', name1) print('Total of', len(cat2), 'from', name2) cat1.cut(cat1.brick_primary) cat2.cut(cat2.brick_primary) print('Total of', len(cat1), 'BRICK_PRIMARY from', name1) print('Total of', len(cat2), 'BRICK_PRIMARY from', name2) cat1.cut((cat1.decam_anymask[:, 1] == 0) * (cat1.decam_anymask[:, 2] == 0) * (cat1.decam_anymask[:, 4] == 0)) cat2.cut((cat2.decam_anymask[:, 1] == 0) * (cat2.decam_anymask[:, 2] == 0) * (cat2.decam_anymask[:, 4] == 0)) print('Total of', len(cat1), 'unmasked from', name1) print('Total of', len(cat2), 'unmasked from', name2) I, J, d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match / 3600., nearest=True) print(len(I), 'matched') plt.clf() plt.hist(d * 3600., 100) plt.xlabel('Match distance (arcsec)') plt.title(tt) ps.savefig() matched1 = cat1[I] matched2 = cat2[J] for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]: K = np.flatnonzero((matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0)) print('Median mw_trans', band, 'is', np.median(matched1.decam_mw_transmission[:, iband])) plt.clf() plt.errorbar( matched1.decam_flux[K, iband], matched2.decam_flux[K, iband], fmt='.', color=cc, xerr=1. / np.sqrt(matched1.decam_flux_ivar[K, iband]), yerr=1. / np.sqrt(matched2.decam_flux_ivar[K, iband]), alpha=0.1, ) plt.xlabel('%s flux: %s' % (name1, band)) plt.ylabel('%s flux: %s' % (name2, band)) plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.) plt.axis([-100, 1000, -100, 1000]) plt.title(tt) ps.savefig() for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]: good = ((matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0)) K = np.flatnonzero(good) psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') P = np.flatnonzero(good * psf1 * psf2) mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband]) iv1 = matched1.decam_flux_ivar[:, iband] iv2 = matched2.decam_flux_ivar[:, iband] std = np.sqrt(1. / iv1 + 1. / iv2) plt.clf() plt.plot( mag1[K], (matched2.decam_flux[K, iband] - matched1.decam_flux[K, iband]) / std[K], '.', alpha=0.1, color=cc) plt.plot( mag1[P], (matched2.decam_flux[P, iband] - matched1.decam_flux[P, iband]) / std[P], '.', alpha=0.1, color='k') plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' % (name2, name1, band)) plt.xlabel('%s mag: %s' % (name1, band)) plt.axhline(0, color='k', alpha=0.5) plt.axis([24, 16, -10, 10]) plt.title(tt) ps.savefig() plt.clf() lp, lt = [], [] for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]: good = ((matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0)) #good = True psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband]) iv1 = matched1.decam_flux_ivar[:, iband] iv2 = matched2.decam_flux_ivar[:, iband] std = np.sqrt(1. / iv1 + 1. / iv2) #std = np.hypot(std, 0.01) G = np.flatnonzero(good * psf1 * psf2 * np.isfinite(mag1) * (mag1 >= 20) * (mag1 < dict(g=24, r=23.5, z=22.5)[band])) n, b, p = plt.hist( (matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) / std[G], range=(-4, 4), bins=50, histtype='step', color=cc, normed=True) sig = (matched2.decam_flux[G, iband] - matched1.decam_flux[G, iband]) / std[G] print('Raw mean and std of points:', np.mean(sig), np.std(sig)) med = np.median(sig) rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2. print('Median and percentile-based sigma:', med, rsigma) lp.append(p[0]) lt.append('%s: %.2f +- %.2f' % (band, med, rsigma)) bins = [] gaussint = [] for blo, bhi in zip(b, b[1:]): c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo) c /= (bhi - blo) #bins.extend([blo,bhi]) #gaussint.extend([c,c]) bins.append((blo + bhi) / 2.) gaussint.append(c) plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5) plt.title(tt) plt.xlabel('Flux difference / error (sigma)') plt.axvline(0, color='k', alpha=0.1) plt.ylim(0, 0.45) plt.legend(lp, lt, loc='upper right') ps.savefig() for iband, band, cc in [(1, 'g', 'g'), (2, 'r', 'r'), (4, 'z', 'm')]: plt.clf() mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:, iband], matched1.decam_flux_ivar[:, iband]) mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors( matched2.decam_flux[:, iband], matched2.decam_flux_ivar[:, iband]) meanmag = NanoMaggies.nanomaggiesToMag( (matched1.decam_flux[:, iband] + matched2.decam_flux[:, iband]) / 2.) psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') good = ((matched1.decam_flux_ivar[:, iband] > 0) * (matched2.decam_flux_ivar[:, iband] > 0) * np.isfinite(mag1) * np.isfinite(mag2)) K = np.flatnonzero(good) P = np.flatnonzero(good * psf1 * psf2) plt.errorbar(mag1[K], mag2[K], fmt='.', color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1) plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('%s %s (mag)' % (name2, band)) plt.plot([-1e6, 1e6], [-1e6, 1e6], 'k-', alpha=1.) plt.axis([24, 16, 24, 16]) plt.title(tt) ps.savefig() plt.clf() plt.errorbar(mag1[K], mag2[K] - mag1[K], fmt='.', color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1) plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axis([24, 16, -1, 1]) plt.title(tt) ps.savefig() magbins = np.arange(16, 24.001, 0.5) plt.clf() plt.plot(mag1[K], (mag2[K] - mag1[K]) / np.hypot(magerr1[K], magerr2[K]), '.', color=cc, alpha=0.1) plt.plot(mag1[P], (mag2[P] - mag1[P]) / np.hypot(magerr1[P], magerr2[P]), 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('(%s %s - %s %s) / errors (sigma)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axis([24, 16, -10, 10]) plt.title(tt) ps.savefig() y = (mag2 - mag1) / np.hypot(magerr1, magerr2) plt.clf() plt.plot(meanmag[P], y[P], 'k.', alpha=0.1) midmag = [] vals = np.zeros((len(magbins) - 1, 5)) median_err1 = [] iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25) # FIXME -- should we do some stats after taking off the mean difference? for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])): I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)] midmag.append((mlo + mhi) / 2.) median_err1.append(np.median(magerr1[I])) if len(I) == 0: continue # median and +- 1 sigma quantiles ybin = y[I] vals[bini, 0] = np.percentile(ybin, 16) vals[bini, 1] = np.median(ybin) vals[bini, 2] = np.percentile(ybin, 84) # +- 2 sigma quantiles vals[bini, 3] = np.percentile(ybin, 2.3) vals[bini, 4] = np.percentile(ybin, 97.7) iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25) print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points') # if iqd > iqd_gauss: # # What error adding in quadrature would you need to make the IQD match? # err = median_err1[-1] # target_err = err * (iqd / iqd_gauss) # sys_err = np.sqrt(target_err**2 - err**2) # print('--> add systematic error', sys_err) # ~ Johan's cuts mlo = 21. mhi = dict(g=24., r=23.5, z=22.5)[band] I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)] ybin = y[I] iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25) print('Mag bin', mlo, mhi, 'band', band, ': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points') if iqd > iqd_gauss: # What error adding in quadrature would you need to make # the IQD match? err = np.median(np.hypot(magerr1[I], magerr2[I])) print('Median error (hypot):', err) target_err = err * (iqd / iqd_gauss) print('Target:', target_err) sys_err = np.sqrt((target_err**2 - err**2) / 2.) print('--> add systematic error', sys_err) # check... err_sys = np.hypot(np.hypot(magerr1, sys_err), np.hypot(magerr2, sys_err)) ysys = (mag2 - mag1) / err_sys ysys = ysys[I] print('Resulting median error:', np.median(err_sys[I])) iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25) print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian') # Hmmm, this doesn't work... totally overshoots. plt.errorbar(midmag, vals[:, 1], fmt='o', color='b', yerr=(vals[:, 1] - vals[:, 0], vals[:, 2] - vals[:, 1]), capthick=3, zorder=20) plt.errorbar(midmag, vals[:, 1], fmt='o', color='b', yerr=(vals[:, 1] - vals[:, 3], vals[:, 4] - vals[:, 1]), capthick=2, zorder=20) plt.axhline(1., color='b', alpha=0.2) plt.axhline(-1., color='b', alpha=0.2) plt.axhline(2., color='b', alpha=0.2) plt.axhline(-2., color='b', alpha=0.2) for mag, err, y in zip(midmag, median_err1, vals[:, 3]): if not np.isfinite(err): continue if y < -6: continue plt.text(mag, y - 0.1, '%.3f' % err, va='top', ha='center', color='k', fontsize=10) plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band)) plt.ylabel('(%s %s - %s %s) / errors (sigma)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axvline(21, color='k', alpha=0.3) plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3) plt.axis([24.1, 16, -6, 6]) plt.title(tt) ps.savefig() #magbins = np.append([16, 18], np.arange(20, 24.001, 0.5)) if band == 'g': magbins = [20, 24] elif band == 'r': magbins = [20, 23.5] elif band == 'z': magbins = [20, 22.5] slo, shi = -5, 5 plt.clf() ha = dict(bins=25, range=(slo, shi), histtype='step', normed=True) y = (mag2 - mag1) / np.hypot(magerr1, magerr2) midmag = [] nn = [] rgbs = [] lt, lp = [], [] for bini, (mlo, mhi) in enumerate(zip(magbins, magbins[1:])): I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)] if len(I) == 0: continue ybin = y[I] rgb = [0., 0., 0.] rgb[0] = float(bini) / (len(magbins) - 1) rgb[2] = 1. - rgb[0] n, b, p = plt.hist(ybin, color=rgb, **ha) lt.append('mag %g to %g' % (mlo, mhi)) lp.append(p[0]) midmag.append((mlo + mhi) / 2.) nn.append(n) rgbs.append(rgb) bins = [] gaussint = [] for blo, bhi in zip(b, b[1:]): #midbin.append((blo+bhi)/2.) #gaussint.append(scipy.stats.norm.cdf(bhi) - # scipy.stats.norm.cdf(blo)) c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo) c /= (bhi - blo) bins.extend([blo, bhi]) gaussint.extend([c, c]) plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5) plt.legend(lp, lt) plt.title(tt) plt.xlim(slo, shi) ps.savefig() bincenters = b[:-1] + (b[1] - b[0]) / 2. plt.clf() lp = [] for n, rgb, mlo, mhi in zip(nn, rgbs, magbins, magbins[1:]): p = plt.plot(bincenters, n, '-', color=rgb) lp.append(p[0]) plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2) plt.legend(lp, lt) plt.title(tt) plt.xlim(slo, shi) ps.savefig()
def galex_forcedphot(galex_dir, cat, tiles, band, roiradecbox, pixelized_psf=False, ps=None): ''' Given a list of tractor sources *cat* and a list of GALEX tiles *tiles* (a fits_table with RA,Dec,tilename) runs forced photometry, returning a FITS table the same length as *cat*. ''' from tractor import Tractor from astrometry.util.ttime import Time if False: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('wise-forced-w%i' % band) plots = (ps is not None) if plots: import pylab as plt use_ceres = True wantims = True get_models = True gband = 'galex' phot = fits_table() tims = [] for tile in tiles: info('Reading GALEX tile', tile.visitname.strip(), 'band', band) tim = galex_tractor_image(tile, band, galex_dir, roiradecbox, gband) if tim is None: debug('Actually, no overlap with tile', tile.tilename) continue # if plots: # sig1 = tim.sig1 # plt.clf() # plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', # cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) # plt.colorbar() # tag = '%s W%i' % (tile.tilename, band) # plt.title('%s: tim data' % tag) # ps.savefig() if pixelized_psf: psfimg = galex_psf(band, galex_dir) tim.psf = PixelizedPSF(psfimg) # if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): # mjd[I] = (tim.mjdmin + tim.mjdmax) / 2. # # PSF norm for depth # psf = tim.getPsf() # h,w = tim.shape # patch = psf.getPointSourcePatch(h//2, w//2).patch # psfnorm = np.sqrt(np.sum(patch**2)) # # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags. # psfdepth = 1. / (tim.sig1 / psfnorm)**2 # phot.get(wband + '_psfdepth')[I] = psfdepth tim.tile = tile tims.append(tim) tractor = Tractor(tims, cat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer ceres_block = 8 tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(gband) t0 = Time() R = tractor.optimize_forced_photometry( fitstats=True, variance=True, shared_params=False, wantims=wantims) info('GALEX forced photometry took', Time() - t0) #info('Result:', R) if use_ceres: term = R.ceres_status['termination'] # Running out of memory can cause failure to converge and term # status = 2. Fail completely in this case. if term != 0: info('Ceres termination status:', term) raise RuntimeError('Ceres terminated with status %i' % term) if wantims: ims1 = R.ims1 flux_invvars = R.IV # if plots: # # Create models for just the brightest sources # bright_cat = [src for src in cat # if src.getBrightness().getBand(wanyband) > 1000] # debug('Bright soures:', len(bright_cat)) # btr = Tractor(tims, bright_cat) # for tim in tims: # mod = btr.getModelImage(tim) # tile = tim.tile # tag = '%s W%i' % (tile.tilename, band) # sig1 = tim.sig1 # plt.clf() # plt.imshow(mod, interpolation='nearest', origin='lower', # cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) # plt.colorbar() # plt.title('%s: bright-star models' % tag) # ps.savefig() if get_models: models = [] for i,tim in enumerate(tims): tile = tim.tile (dat, mod, ie, _, _) = ims1[i] models.append((tile.visitname, band, tim.wcs.wcs, dat, mod, ie)) if plots: for i,tim in enumerate(tims): tile = tim.tile tag = '%s %s' % (tile.tilename, band) (dat, mod, _, chi, _) = ims1[i] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() nm = np.array([src.getBrightness().getBand(gband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their default # (1-sigma or whatever) initial fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. niceband = band + 'uv' phot.set('flux_' + niceband, nm.astype(np.float32)) phot.set('flux_ivar_' + niceband, nm_ivar.astype(np.float32)) #phot.set(band + '_mjd', mjd) rtn = gphotduck() rtn.phot = phot rtn.models = None if get_models: rtn.models = models return rtn
def stage_fit_on_coadds(survey=None, targetwcs=None, pixscale=None, bands=None, tims=None, brickname=None, version_header=None, coadd_tiers=None, apodize=True, subsky=True, ubercal_sky=False, subsky_radii=None, nsatur=None, fitoncoadds_reweight_ivar=True, plots=False, plots2=False, ps=None, coadd_bw=False, W=None, H=None, brick=None, blobs=None, lanczos=True, ccds=None, write_metrics=True, mp=None, record_event=None, **kwargs): from legacypipe.coadds import make_coadds from legacypipe.bits import DQ_BITS from legacypipe.survey import LegacySurveyWcs from legacypipe.coadds import get_coadd_headers from tractor.image import Image from tractor.basics import LinearPhotoCal from tractor.sky import ConstantSky from tractor.psf import PixelizedPSF from tractor.tractortime import TAITime import astropy.time import fitsio if plots or plots2: import pylab as plt from legacypipe.survey import get_rgb # Custom sky-subtraction for large galaxies. skydict = {} if not subsky: if ubercal_sky: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('fitoncoadds-{}'.format(brickname)) tims, skydict = ubercal_skysub(tims, targetwcs, survey, brickname, bands, mp, subsky_radii=subsky_radii, plots=True, plots2=False, ps=ps, verbose=True) else: print('Skipping sky-subtraction entirely.') # Create coadds and then build custom tims from them. for tim in tims: ie = tim.inverr if np.any(ie < 0): print('Negative inverse error in image {}'.format(tim.name)) CC = [] if coadd_tiers: # Sort by band and sort them into tiers. tiers = [[] for i in range(coadd_tiers)] for b in bands: btims = [] seeing = [] for tim in tims: if tim.band != b: continue btims.append(tim) seeing.append(tim.psf_fwhm * tim.imobj.pixscale) I = np.argsort(seeing) btims = [btims[i] for i in I] seeing = [seeing[i] for i in I] N = min(coadd_tiers, len(btims)) splits = np.round(np.arange(N + 1) * float(len(btims)) / N).astype(int) print('Splitting', len(btims), 'images into', N, 'tiers: splits:', splits) print('Seeing limits:', [seeing[min(s, len(seeing) - 1)] for s in splits]) for s0, s1, tt in zip(splits, splits[1:], tiers): tt.extend(btims[s0:s1]) for itier, tier in enumerate(tiers): print('Producing coadds for tier', (itier + 1)) C = make_coadds( tier, bands, targetwcs, detmaps=True, ngood=True, lanczos=lanczos, allmasks=True, anymasks=True, psf_images=True, nsatur=2, mp=mp, plots=plots2, ps=ps, # note plots2 here! callback=None) if plots: plt.clf() for iband, (band, psf) in enumerate(zip(bands, C.psf_imgs)): plt.subplot(1, len(bands), iband + 1) plt.imshow(psf, interpolation='nearest', origin='lower') plt.title('Coadd PSF image: band %s' % band) plt.suptitle('Tier %i' % (itier + 1)) ps.savefig() # for band,img in zip(bands, C.coimgs): # plt.clf() # plt.imshow(img, plt.clf() plt.imshow(get_rgb(C.coimgs, bands), origin='lower') plt.title('Tier %i' % (itier + 1)) ps.savefig() CC.append(C) else: C = make_coadds( tims, bands, targetwcs, detmaps=True, ngood=True, lanczos=lanczos, allmasks=True, anymasks=True, psf_images=True, mp=mp, plots=plots2, ps=ps, # note plots2 here! callback=None) CC.append(C) cotims = [] for C in CC: if plots2: for band, iv in zip(bands, C.cowimgs): pass # plt.clf() # plt.imshow(np.sqrt(iv), interpolation='nearest', origin='lower') # plt.title('Coadd Inverr: band %s' % band) # ps.savefig() for band, psf in zip(bands, C.psf_imgs): plt.clf() plt.imshow(psf, interpolation='nearest', origin='lower') plt.title('Coadd PSF image: band %s' % band) ps.savefig() for band, img, iv in zip(bands, C.coimgs, C.cowimgs): from scipy.ndimage.filters import gaussian_filter # plt.clf() # plt.hist((img * np.sqrt(iv))[iv>0], bins=50, range=(-5,8), log=True) # plt.title('Coadd pixel values (sigmas): band %s' % band) # ps.savefig() psf_sigma = np.mean([ (tim.psf_sigma * tim.imobj.pixscale / pixscale) for tim in tims if tim.band == band ]) gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma) psfnorm = gnorm #np.sqrt(np.sum(psfimg**2)) detim = gaussian_filter(img, psf_sigma) / psfnorm**2 cosig1 = 1. / np.sqrt(np.median(iv[iv > 0])) detsig1 = cosig1 / psfnorm # plt.clf() # plt.subplot(2,1,1) # plt.hist(detim.ravel() / detsig1, bins=50, range=(-5,8), log=True) # plt.title('Coadd detection map values / sig1 (sigmas): band %s' % band) # plt.subplot(2,1,2) # plt.hist(detim.ravel() / detsig1, bins=50, range=(-5,8)) # ps.savefig() # # as in detection.py # detiv = np.zeros_like(detim) + (1. / detsig1**2) # detiv[iv == 0] = 0. # detiv = gaussian_filter(detiv, psf_sigma) # # plt.clf() # plt.hist((detim * np.sqrt(detiv)).ravel(), bins=50, range=(-5,8), log=True) # plt.title('Coadd detection map values / detie (sigmas): band %s' % band) # ps.savefig() for iband, (band, img, iv, allmask, anymask, psfimg) in enumerate( zip(bands, C.coimgs, C.cowimgs, C.allmasks, C.anymasks, C.psf_imgs)): mjd = np.mean( [tim.imobj.mjdobs for tim in tims if tim.band == band]) mjd_tai = astropy.time.Time(mjd, format='mjd', scale='utc').tai.mjd tai = TAITime(None, mjd=mjd_tai) twcs = LegacySurveyWcs(targetwcs, tai) #print('PSF sigmas (in pixels) for band', band, ':', # ['%.2f' % tim.psf_sigma for tim in tims if tim.band == band]) print( 'PSF sigmas in coadd pixels:', ', '.join([ '%.2f' % (tim.psf_sigma * tim.imobj.pixscale / pixscale) for tim in tims if tim.band == band ])) psf_sigma = np.mean([ (tim.psf_sigma * tim.imobj.pixscale / pixscale) for tim in tims if tim.band == band ]) print('Using average PSF sigma', psf_sigma) psf = PixelizedPSF(psfimg) gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma) psfnorm = np.sqrt(np.sum(psfimg**2)) print('Gaussian PSF norm', gnorm, 'vs pixelized', psfnorm) # if plots: # from collections import Counter # plt.clf() # plt.imshow(mask, interpolation='nearest', origin='lower') # plt.colorbar() # plt.title('allmask') # ps.savefig() # print('allmask for band', band, ': values:', Counter(mask.ravel())) # Scale invvar to take into account that we have resampled (~double-counted) pixels tim_pixscale = np.mean( [tim.imobj.pixscale for tim in tims if tim.band == band]) cscale = tim_pixscale / pixscale print('average tim pixel scale / coadd scale:', cscale) iv /= cscale**2 if fitoncoadds_reweight_ivar: # We first tried setting the invvars constant per tim -- this # makes things worse, since we *remove* the lowered invvars at # the cores of galaxies. # # Here we're hacking the relative weights -- squaring the # weights but then making the median the same, ie, squaring # the dynamic range or relative weights -- ie, downweighting # the cores even more than they already are from source # Poisson terms. median_iv = np.median(iv[iv > 0]) assert (median_iv > 0) iv = iv * np.sqrt(iv) / np.sqrt(median_iv) assert (np.all(np.isfinite(iv))) assert (np.all(iv >= 0)) cotim = Image(img, invvar=iv, wcs=twcs, psf=psf, photocal=LinearPhotoCal(1., band=band), sky=ConstantSky(0.), name='coadd-' + band) cotim.band = band cotim.subwcs = targetwcs cotim.psf_sigma = psf_sigma cotim.sig1 = 1. / np.sqrt(np.median(iv[iv > 0])) # Often, SATUR masks on galaxies / stars are surrounded by BLEED pixels. Soak these into # the SATUR mask. from scipy.ndimage.morphology import binary_dilation anymask |= np.logical_and(((anymask & DQ_BITS['bleed']) > 0), binary_dilation( ((anymask & DQ_BITS['satur']) > 0), iterations=10)) * DQ_BITS['satur'] # Saturated in any image -> treat as saturated in coadd # (otherwise you get weird systematics in the weighted coadds, and weird source detection!) mask = allmask mask[(anymask & DQ_BITS['satur'] > 0)] |= DQ_BITS['satur'] if coadd_tiers: # nsatur -- reset SATUR bit mask &= ~DQ_BITS['satur'] mask |= DQ_BITS['satur'] * C.satmaps[iband] cotim.dq = mask cotim.dq_saturation_bits = DQ_BITS['satur'] cotim.psfnorm = gnorm cotim.galnorm = 1.0 # bogus! cotim.imobj = Duck() cotim.imobj.fwhm = 2.35 * psf_sigma cotim.imobj.pixscale = pixscale cotim.time = tai cotim.primhdr = fitsio.FITSHDR() get_coadd_headers(cotim.primhdr, tims, band, coadd_headers=skydict) cotims.append(cotim) if plots: plt.clf() bitmap = dict([(v, k) for k, v in DQ_BITS.items()]) k = 1 for i in range(12): bitval = 1 << i if not bitval in bitmap: continue # only 9 bits are actually used plt.subplot(3, 3, k) k += 1 plt.imshow((cotim.dq & bitval) > 0, vmin=0, vmax=1.5, cmap='hot', origin='lower') plt.title(bitmap[bitval]) plt.suptitle('Coadd mask planes %s band' % band) ps.savefig() plt.clf() h, w = cotim.shape rgb = np.zeros((h, w, 3), np.uint8) rgb[:, :, 0] = (cotim.dq & DQ_BITS['satur'] > 0) * 255 rgb[:, :, 1] = (cotim.dq & DQ_BITS['bleed'] > 0) * 255 plt.imshow(rgb, origin='lower') plt.suptitle('Coadd DQ band %s: red = SATUR, green = BLEED' % band) ps.savefig() # Save an image of the coadd PSF # copy version_header before modifying it. hdr = fitsio.FITSHDR() for r in version_header.records(): hdr.add_record(r) hdr.add_record( dict(name='IMTYPE', value='coaddpsf', comment='LegacySurveys image type')) hdr.add_record( dict(name='BAND', value=band, comment='Band of this coadd/PSF')) hdr.add_record( dict(name='PSF_SIG', value=psf_sigma, comment='Average PSF sigma (coadd pixels)')) hdr.add_record( dict(name='PIXSCAL', value=pixscale, comment='Pixel scale of this PSF (arcsec)')) hdr.add_record( dict(name='INPIXSC', value=tim_pixscale, comment='Native image pixscale scale (average, arcsec)')) hdr.add_record( dict(name='MJD', value=mjd, comment='Average MJD for coadd')) hdr.add_record( dict(name='MJD_TAI', value=mjd_tai, comment='Average MJD (in TAI) for coadd')) with survey.write_output('copsf', brick=brickname, band=band) as out: out.fits.write(psfimg, header=hdr) # EVIL return dict(tims=cotims, coadd_headers=skydict)
def main(): ps = PlotSequence('conv') S = 51 center = S/2 print('Center', center) #for psf_sigma in [2., 1.5, 1.]: for psf_sigma in [2.]: rms2 = [] x = np.arange(S) y = np.arange(S) xx,yy = np.meshgrid(x, y) scale = 1.5 / psf_sigma pixpsf = render_airy((scale, center), x, y) psf = (scale,center) eval_psf = render_airy plt.clf() plt.subplot(2,1,1) plt.plot(x, pixpsf[center,:], 'b-') plt.plot(x, pixpsf[:,center], 'r-') plt.subplot(2,1,2) plt.plot(x, np.maximum(1e-16, pixpsf[center,:]), 'b-') plt.plot(x, np.maximum(1e-16, pixpsf[:,center]), 'r-') plt.yscale('log') ps.savefig() plt.clf() plt.imshow(pixpsf, interpolation='nearest', origin='lower') ps.savefig() plt.clf() plt.imshow(np.log10(np.maximum(1e-16, pixpsf)), interpolation='nearest', origin='lower') plt.colorbar() plt.title('log PSF') ps.savefig() # psf #psf = scipy.stats.norm(loc=center + 0.5, scale=psf_sigma) # plt.clf() # plt.imshow(Pcdf, interpolation='nearest', origin='lower') # ps.savefig() # #Pcdf = psf.cdf(xx) * psf.cdf(yy) # #pixpsf = integrate_gaussian(psf, xx, yy) # # padpsf = np.zeros((S*2-1, S*2-1)) # ph,pw = pixpsf.shape # padpsf[S/2:S/2+ph, S/2:S/2+pw] = pixpsf # Fpsf = np.fft.rfft2(padpsf) # # padh,padw = padpsf.shape # v = np.fft.rfftfreq(padw) # w = np.fft.fftfreq(padh) # fmax = max(max(np.abs(v)), max(np.abs(w))) # cut = fmax / 2. * 1.000001 # #print('Frequence cut:', cut) # Ffiltpsf = Fpsf.copy() # #print('Ffiltpsf', Ffiltpsf.shape) # #print((np.abs(w) < cut).shape) # #print((np.abs(v) < cut).shape) # Ffiltpsf[np.abs(w) > cut, :] = 0. # Ffiltpsf[:, np.abs(v) > cut] = 0. # #print('pad v', v) # #print('pad w', w) # # filtpsf = np.fft.irfft2(Ffiltpsf, s=(padh,padw)) # # print('filtered PSF real', np.max(np.abs(filtpsf.real))) # print('filtered PSF imag', np.max(np.abs(filtpsf.imag))) # # plt.clf() # plt.subplot(2,3,1) # dimshow(Fpsf.real) # plt.colorbar() # plt.title('Padded PSF real') # plt.subplot(2,3,4) # dimshow(Fpsf.imag) # plt.colorbar() # plt.title('Padded PSF imag') # # plt.subplot(2,3,2) # dimshow(Ffiltpsf.real) # plt.colorbar() # plt.title('Filt PSF real') # plt.subplot(2,3,5) # dimshow(Ffiltpsf.imag) # plt.colorbar() # plt.title('Filt PSF imag') # # plt.subplot(2,3,3) # dimshow(filtpsf.real) # plt.title('PSF real') # plt.colorbar() # # plt.subplot(2,3,6) # dimshow(filtpsf.imag) # plt.title('PSF imag') # plt.colorbar() # # ps.savefig() # # # pixpsf = filtpsf gal_sigmas = [2, 1, 0.5, 0.25] for gal_sigma in gal_sigmas: # plt.clf() # plt.imshow(Gcdf, interpolation='nearest', origin='lower') # plt.savefig('dcdf.png') # plt.clf() # plt.imshow(np.exp(-0.5 * ((xx-center)**2 + (yy-center)**2)/2.**2), # interpolation='nearest', origin='lower') # plt.savefig('g.png') # my convolution pixscale = 1. cd = pixscale * np.eye(2) / 3600. P,FG,Gmine,v,w = galaxy_psf_convolution( gal_sigma, 0., 0., GaussianGalaxy, cd, 0., 0., pixpsf, debug=True) #print('v:', v) #print('w:', w) #print('P:', P.shape) print() print('PSF %g, Gal %g' % (psf_sigma, gal_sigma)) rmax = np.argmax(np.abs(w)) cmax = np.argmax(np.abs(v)) l2_rmax = np.sqrt(np.sum(P[rmax,:].real**2 + P[rmax,:].imag**2)) l2_cmax = np.sqrt(np.sum(P[:,cmax].real**2 + P[:,cmax].imag**2)) print('PSF L_2 in highest-frequency rows & cols:', l2_rmax, l2_cmax) l2_rmax = np.sqrt(np.sum(FG[rmax,:].real**2 + FG[rmax,:].imag**2)) l2_cmax = np.sqrt(np.sum(FG[:,cmax].real**2 + FG[:,cmax].imag**2)) print('Gal L_2 in highest-frequency rows & cols:', l2_rmax, l2_cmax) C = P * FG l2_rmax = np.sqrt(np.sum(C[rmax,:].real**2 + C[rmax,:].imag**2)) l2_cmax = np.sqrt(np.sum(C[:,cmax].real**2 + C[:,cmax].imag**2)) print('PSF*Gal L_2 in highest-frequency rows & cols:', l2_rmax, l2_cmax) print() Fpsf, Fgal = compare_subsampled( S, 1, ps, psf, pixpsf, Gmine,v,w, gal_sigma, psf_sigma, cd, get_ffts=True, eval_psf=eval_psf) plt.clf() plt.subplot(2,4,1) dimshow(P.real) plt.colorbar() plt.title('PSF real') plt.subplot(2,4,5) dimshow(P.imag) plt.colorbar() plt.title('PSF imag') plt.subplot(2,4,2) dimshow(FG.real) plt.colorbar() plt.title('Gal real') plt.subplot(2,4,6) dimshow(FG.imag) plt.colorbar() plt.title('Gal imag') plt.subplot(2,4,3) dimshow((P * FG).real) plt.colorbar() plt.title('P*Gal real') plt.subplot(2,4,7) dimshow((P * FG).imag) plt.colorbar() plt.title('P*Gal imag') plt.subplot(2,4,4) dimshow((Fgal).real) plt.colorbar() plt.title('pixGal real') plt.subplot(2,4,8) dimshow((Fgal).imag) plt.colorbar() plt.title('pixGal imag') plt.suptitle('PSF %g, Gal %g' % (psf_sigma, gal_sigma)) ps.savefig() subsample = [1,2,4] rms1 = [] for s in subsample: rms = compare_subsampled(S, s, ps, psf, pixpsf, Gmine,v,w, gal_sigma, psf_sigma, cd, eval_psf=eval_psf) rms1.append(rms) rms2.append(rms1) print() print('PSF sigma =', psf_sigma) print('RMSes:') for rms1,gal_sigma in zip(rms2, gal_sigmas): print('Gal sigma', gal_sigma, 'rms:', ', '.join(['%.3g' % r for r in rms1]))
def main(decals=None, opt=None): '''Driver function for forced photometry of individual DECam images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0, x1, y0, y1) = opt.zoom zoomslice = (slice(y0, y1), slice(x0, x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.filename) opt.filename = None except: # make this 'None' for decals.find_ccds() expnum = None # Try parsing HDU number try: opt.hdu = int(opt.hdu) ccdname = None except: ccdname = opt.hdu opt.hdu = -1 if decals is None: decals = Decals() if opt.filename is not None and opt.hdu >= 0: # Read metadata from file T = exposure_metadata([opt.filename], hdus=[opt.hdu]) print('Metadata:') T.about() else: # Read metadata from decals-ccds.fits table T = decals.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if opt.hdu >= 0: T.cut(T.image_hdu == opt.hdu) print(len(T), 'with HDU', opt.hdu) if opt.filename is not None: T.cut( np.array([f.strip() == opt.filename for f in T.image_filename])) print(len(T), 'with filename', opt.filename) assert (len(T) == 1) im = decals.get_image_object(T[0]) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True) print('Got tim:', tim) if opt.catfn in ['DR1', 'DR2']: if opt.catalog_path is None: opt.catalog_path = opt.catfn.lower() margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, decals=decals) for b in bricks: # there is some overlap with this brick... read the catalog. fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3], 'tractor-%s.fits' % b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec) W, H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) * (yy >= -margin) * (yy <= (H + margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') T.cut((T.out_of_bounds == False) * (T.left_blob == False)) print('Cut to', len(T), 'on out_of_bounds and left_blob') TT.append(T) T = merge_tables(TT) T._header = TT[0]._header del TT # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE) # print('Got cat:', cat) print('Forced photom...') opti = None if opt.ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 opti = CeresOptimizer(BW=B, BH=B) tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') for src in cat: src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) F = fits_table() F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)) ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x - 1).astype(np.float32) F.y = (y - 1).astype(np.float32) if opt.apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack( [tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() print('Apertures:', apertures, 'pixels') for rad in apertures: aper = photutils.CircularAperture(apxy, rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = ap ap = 1. / (np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux_ivar = ap if opt.forced: kwa = {} if opt.plots is None: kwa.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, **kwa) if opt.plots: (data, mod, ie, chi, roi) = R.ims1[0] ima = tim.ima imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5) plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() F.flux = np.array([ src.getBrightness().getFlux(tim.band) for src in cat ]).astype(np.float32) F.flux_ivar = R.IV.astype(np.float32) F.fracflux = R.fitstats.profracflux.astype(np.float32) F.rchi2 = R.fitstats.prochi2.astype(np.float32) program_name = sys.argv[0] version_hdr = get_version_header(program_name, decals.decals_dir) # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) fname = os.path.join(d2, d1, fname) print('Trimmed filename to', fname) #version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file')) version_hdr.add_record( dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file')) version_hdr.add_record( dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext')) version_hdr.add_record( dict(name='CAMERA', value='DECam', comment='Dark Energy Camera')) version_hdr.add_record( dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num')) version_hdr.add_record( dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name')) version_hdr.add_record( dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record( dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image')) keys = [ 'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME' ] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = { 'mjd': 'sec', 'exptime': 'sec', 'flux': 'nanomaggy', 'flux_ivar': '1/nanomaggy^2' } columns = F.get_columns() for i, col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) print('Finished forced phot:', Time() - t0) return 0
def main(): W = H = 50 pixscale = 0.262 / 3600. band = 'r' truewcs = Tan(0., 0., W / 2., H / 2., -pixscale, 0., 0., pixscale, float(W), float(H)) src = PointSource(RaDecPos( 0., 0., ), NanoMaggies(**{band: 1.})) src.symmetric_derivs = True forced_cat = [src] sig1 = 0.25 flux = 100. psf_sigma = 2.0 psfnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma) nsigma = flux * psfnorm / sig1 print('S/N:', nsigma) v = psf_sigma**2 # Create a pixelized PSF model by rendering the Gaussian on a stamp xx, yy = np.meshgrid(np.arange(-12, 13), np.arange(-12, 13)) pp = np.exp(-0.5 * (xx**2 + yy**2) / psf_sigma**2) pp /= np.sum(pp) psf = PixelizedPSF(pp) tim = Image(data=np.zeros((H, W), np.float32), inverr=np.ones((H, W), np.float32) * 1. / sig1, wcs=ConstantFitsWcs(truewcs), photocal=LinearPhotoCal(1., band=band), sky=ConstantSky(0.), psf=psf) tim.band = band tim.sig1 = sig1 #dra_pix = np.linspace(-5, 5, 21) dra_pix = np.linspace(-2, 2, 21) ddec_pix = np.zeros_like(dra_pix) dra2_pix = np.linspace(-5, 5, 21) ddec2_pix = -0.5 * dra2_pix ps = PlotSequence('pm') for dras, ddecs, srcflux in [ (dra_pix * pixscale, ddec_pix * pixscale, flux), (dra_pix * pixscale, ddec_pix * pixscale, flux / 2), (dra2_pix * pixscale, ddec2_pix * pixscale, flux), ]: FF = [] slicex = [] slicey = [] residx = [] residy = [] for dra, ddec in zip(dras, ddecs): src = PointSource(RaDecPos(0. + dra, 0. + ddec), NanoMaggies(**{band: srcflux})) tr = Tractor([tim], [src]) truemod = tr.getModelImage(0) noise = np.random.normal(size=truemod.shape) * sig1 tim.data = truemod + noise F = run_forced_phot(forced_cat, tim, ceres=False, derivs=True, do_apphot=False, fixed_also=True) #, ps=ps) #print('Src:', forced_cat) t = Tractor([tim], forced_cat) m = t.getModelImage(0) mh, mw = m.shape slicex.append(m[mh // 2, :]) slicey.append(m[:, mw // 2]) residx.append((m - truemod)[mh // 2, :]) residy.append((m - truemod)[:, mw // 2]) #F.about() F.true_dra = dra + np.zeros(len(F)) F.true_ddec = ddec + np.zeros(len(F)) FF.append(F) F = merge_tables(FF) plt.clf() plt.plot(F.true_dra * 3600., F.flux_dra / F.flux * 3600., 'b.', label='RA') plt.plot(F.true_ddec * 3600., F.flux_ddec / F.flux * 3600., 'g.', label='Dec') mx = max(max(np.abs(F.true_dra)), max(np.abs(F.true_ddec))) mx *= 3600. plt.plot([-mx, mx], [-mx, mx], 'k-', alpha=0.1) plt.xlabel('True offset (arcsec)') plt.ylabel('Flux deriv / Flux * 3600 (arcsec)') plt.legend() ps.savefig() plt.clf() plt.plot(np.hypot(F.true_dra, F.true_ddec) * 3600., F.flux, 'b.', label='Flux (dra/dec)') plt.plot(np.hypot(F.true_dra, F.true_ddec) * 3600., F.flux_fixed, 'g.', label='Flux (fixed)') plt.xlabel('True offset (arcsec)') plt.ylabel('Flux') ps.savefig() plt.clf() N = len(slicex) cc = float(N - 1) for i, s in enumerate(slicex): #plt.plot(s + i*10, 'b-') rgb = (0, i / cc, 1. - i / cc) #print('rgb', rgb) plt.plot(s, '-', color=rgb, alpha=0.5) ps.savefig() plt.clf() N = len(residx) cc = float(N - 1) for i, s in enumerate(residx): rgb = (0, i / cc, 1. - i / cc) plt.plot(s, '-', color=rgb, alpha=0.5) ps.savefig()
def plot_unmatched(): from bigboss_test import radecroi ''' select run, rerun, camcol, field, nChild, probPSF, psfFlux_u, psfFlux_g, psfFlux_r, psfFlux_i, psfFlux_z, deVRad_u, deVRad_g, deVRad_r, deVRad_i, deVRad_z, deVAB_u, deVAB_g, deVAB_r, deVAB_i, deVAB_z, deVPhi_u, deVPhi_g, deVPhi_r, deVPhi_i, deVPhi_z, deVFlux_u, deVFlux_g, deVFlux_r, deVFlux_i, deVFlux_z, expRad_u, expRad_g, expRad_r, expRad_i, expRad_z, expAB_u, expAB_g, expAB_r, expAB_i, expAB_z, expPhi_u, expPhi_g, expPhi_r, expPhi_i, expPhi_z, expFlux_u, expFlux_g, expFlux_r, expFlux_i, expFlux_z, fracDeV_u, fracDeV_g, fracDeV_r, fracDeV_i, fracDeV_z, flags_u, flags_g, flags_r, flags_i, flags_z, probPSF_u, probPSF_g, probPSF_r, probPSF_i, probPSF_z, ra, dec from PhotoPrimary where ra between 333.5 and 335.5 and dec between -0.5 and 1.5 ''' ''' -> 124k rows. (sdss-cas-testarea.fits) Distinct runs: 2585, 2728, 7712, 4203, 2583, 4192, 4207, 4184, 2662, 7717 Run #sources 2583 663 2585 675 2662 4 2728 156 4184 762 4192 36135 4203 5 4207 44078 7712 12047 7717 29911 ''' ''' select run, rerun, camcol, field, nChild, probPSF, psfFlux_u, psfFlux_g, psfFlux_r, psfFlux_i, psfFlux_z, deVRad_u, deVRad_g, deVRad_r, deVRad_i, deVRad_z, deVAB_u, deVAB_g, deVAB_r, deVAB_i, deVAB_z, deVPhi_u, deVPhi_g, deVPhi_r, deVPhi_i, deVPhi_z, deVFlux_u, deVFlux_g, deVFlux_r, deVFlux_i, deVFlux_z, expRad_u, expRad_g, expRad_r, expRad_i, expRad_z, expAB_u, expAB_g, expAB_r, expAB_i, expAB_z, expPhi_u, expPhi_g, expPhi_r, expPhi_i, expPhi_z, expFlux_u, expFlux_g, expFlux_r, expFlux_i, expFlux_z, fracDeV_u, fracDeV_g, fracDeV_r, fracDeV_i, fracDeV_z, flags_u, flags_g, flags_r, flags_i, flags_z, probPSF_u, probPSF_g, probPSF_r, probPSF_i, probPSF_z, ra, dec, resolveStatus, score into mydb.wisetest from PhotoObjAll where ra between 333.5 and 335.5 and dec between -0.5 and 1.5 and (resolveStatus & ( dbo.fResolveStatus('SURVEY_PRIMARY') | dbo.fResolveStatus('SURVEY_BADFIELD') | dbo.fResolveStatus('SURVEY_EDGE'))) != 0 --> sdss-cas-testarea-2.fits select run, rerun, camcol, field, nChild, probPSF, psfFlux_u, psfFlux_g, psfFlux_r, psfFlux_i, psfFlux_z, deVRad_u, deVRad_g, deVRad_r, deVRad_i, deVRad_z, deVAB_u, deVAB_g, deVAB_r, deVAB_i, deVAB_z, deVPhi_u, deVPhi_g, deVPhi_r, deVPhi_i, deVPhi_z, deVFlux_u, deVFlux_g, deVFlux_r, deVFlux_i, deVFlux_z, expRad_u, expRad_g, expRad_r, expRad_i, expRad_z, expAB_u, expAB_g, expAB_r, expAB_i, expAB_z, expPhi_u, expPhi_g, expPhi_r, expPhi_i, expPhi_z, expFlux_u, expFlux_g, expFlux_r, expFlux_i, expFlux_z, fracDeV_u, fracDeV_g, fracDeV_r, fracDeV_i, fracDeV_z, flags_u, flags_g, flags_r, flags_i, flags_z, probPSF_u, probPSF_g, probPSF_r, probPSF_i, probPSF_z, ra, dec, resolveStatus, score into mydb.wisetest from PhotoObjAll where ra between 333.5 and 335.5 and dec between -0.5 and 1.5 and (resolveStatus & ( dbo.fResolveStatus('SURVEY_PRIMARY') | dbo.fResolveStatus('SURVEY_BADFIELD') | dbo.fResolveStatus('SURVEY_EDGE') | dbo.fResolveStatus('SURVEY_BEST') )) != 0 --> sdss-cas-testarea-3.fits ''' ''' Check it out: spatial source density looks fine. No overlap between runs. ''' rng = ((333.5, 335.5), (-0.5, 1.5)) from astrometry.util.plotutils import PlotSequence ps = PlotSequence('sdss') if False: r, d = np.mean(rng[0]), np.mean(rng[1]) RCF = radec_to_sdss_rcf(r, d, radius=2. * 60., tablefn='window_flist-DR9.fits') print('Found', len(RCF), 'fields in range') for run, c, f, ra, dec in RCF: print(' ', run, c, f, 'at', ra, dec) from astrometry.blind.plotstuff import PlotSequence plot = Plotstuff(rdw=(r, d, 10), size=(1000, 1000), outformat='png') plot.color = 'white' plot.alpha = 0.5 plot.apply_settings() T = fits_table('window_flist-DR9.fits') I, J, d = match_radec(T.ra, T.dec, r, d, 10) T.cut(I) print('Plotting', len(T)) for i, (m0, m1, n0, n1, node, incl) in enumerate(zip(T.mu_start, T.mu_end, T.nu_start, T.nu_end, T.node, T.incl)): #rr,dd = [],[] for j, (m, n) in enumerate([(m0, n0), (m0, n1), (m1, n1), (m1, n0), (m0, n0)]): ri, di = munu_to_radec_deg(m, n, node, incl) # rr.append(ri) # dd.append(di) if j == 0: plot.move_to_radec(ri, di) else: plot.line_to_radec(ri, di) plot.stroke() plot.plot_grid(2, 2, 5, 5) plot.write('fields.png') # CAS PhotoObjAll.resolveStatus bits sprim = 0x100 sbad = 0x800 sedge = 0x1000 sbest = 0x200 if False: T = fits_table('sdss-cas-testarea.fits') plt.clf() plothist(T.ra, T.dec, 200, range=rng) plt.title('PhotoPrimary') ps.savefig() T = fits_table('sdss-cas-testarea-2.fits') plt.clf() plothist(T.ra, T.dec, 200, range=rng) plt.title('PhotoObjAll: SURVEY_PRIMARY | SURVEY_BADFIELD | SURVEY_EDGE') ps.savefig() T = fits_table('sdss-cas-testarea-3.fits') plt.clf() plothist(T.ra, T.dec, 200, range=rng) plt.title( 'PhotoObjAll: SURVEY_PRIMARY | SURVEY_BADFIELD | SURVEY_EDGE | SURVEY_BEST') ps.savefig() for j, (flags, txt) in enumerate([(sprim, 'PRIM'), (sbad, 'BAD'), (sedge, 'EDGE'), (sbest, 'BEST')]): I = np.flatnonzero( (T.resolvestatus & (sprim | sbad | sedge | sbest)) == flags) print(len(I), 'with', txt) if len(I) == 0: continue plt.clf() plothist(T.ra[I], T.dec[I], 200, range=rng) plt.title('%i with %s' % (len(I), txt)) ps.savefig() for j, (flags, txt) in enumerate([(sprim | sbad, 'PRIM + BAD'), (sprim | sedge, 'PRIM + EDGE'), (sprim | sbest, 'PRIM + BEST')]): I = np.flatnonzero((T.resolvestatus & flags) > 0) print(len(I), 'with', txt) if len(I) == 0: continue plt.clf() plothist(T.ra[I], T.dec[I], 200, range=rng) plt.title('%i with %s' % (len(I), txt)) ps.savefig() # for run in np.unique(T.run): # I = (T.run == run) # plt.clf() # plothist(T.ra[I], T.dec[I], 200, range=rng) # plt.title('Run %i' % run) # ps.savefig() R = 1. / 3600. I, J, d = match_radec(T.ra, T.dec, T.ra, T.dec, R, notself=True) print(len(I), 'matches') plt.clf() loghist((T.ra[I] - T.ra[J]) * 3600., (T.dec[I] - T.dec[J]) * 3600., 200, range=((-1, 1), (-1, 1))) ps.savefig() ps = PlotSequence('forced') basedir = os.environ.get('BIGBOSS_DATA', '/project/projectdirs/bigboss') wisedatadir = os.path.join(basedir, 'data', 'wise') wisecat = fits_table(os.path.join( wisedatadir, 'catalogs', 'wisecat2.fits')) # plt.clf() #plothist(wisecat.ra, wisecat.dec, 200, range=rng) # plt.savefig('wisecat.png') (ra0, ra1, dec0, dec1) = radecroi ra = (ra0 + ra1) / 2. dec = (dec0 + dec1) / 2. #cas = fits_table('sdss-cas-testarea.fits') #cas = fits_table('sdss-cas-testarea-2.fits') cas = fits_table('sdss-cas-testarea-3.fits') print('Read', len(cas), 'CAS sources') cas.cut((cas.resolvestatus & sedge) == 0) print('Cut to ', len(cas), 'without SURVEY_EDGE set') # Check out WISE / SDSS matches. wise = wisecat sdss = cas print(len(sdss), 'SDSS sources') print(len(wise), 'WISE sources') R = 10. I, J, d = match_radec(wise.ra, wise.dec, sdss.ra, sdss.dec, R / 3600., nearest=True) print(len(I), 'matches') print('max dist:', d.max()) plt.clf() plt.hist(d * 3600., 100, range=(0, R), log=True) plt.xlabel('Match distance (arcsec)') plt.ylabel('Number of matches') plt.title('SDSS-WISE astrometric matches') ps.savefig() plt.clf() loghist((wise.ra[I] - sdss.ra[J]) * 3600., (wise.dec[I] - sdss.dec[J]) * 3600., 200, range=((-R, R), (-R, R))) plt.title('SDSS-WISE astrometric matches') plt.xlabel('dRA (arcsec)') plt.ylabel('dDec (arcsec)') ps.savefig() R = 4. I, J, d = match_radec(wise.ra, wise.dec, sdss.ra, sdss.dec, R / 3600., nearest=True) print(len(I), 'matches') unmatched = np.ones(len(wise), bool) unmatched[I] = False wun = wise[unmatched] plt.clf() plothist(sdss.ra, sdss.dec, 200, range=rng) plt.title('SDSS source density') ps.savefig() plt.clf() plothist(wise.ra, wise.dec, 200, range=rng) plt.title('WISE source density') ps.savefig() plt.clf() #plt.plot(wun.ra, wun.dec, 'r.') #plt.axis(rng[0] + rng[1]) plothist(wun.ra, wun.dec, 200, range=rng) plt.title('Unmatched WISE sources') ps.savefig() for band in 'ugriz': sdss.set('psfmag_' + band, NanoMaggies.nanomaggiesToMag(sdss.get('psfflux_' + band))) # plt.clf() # loghist(wise.w1mpro[I], sdss.psfmag_r[J], 200) # plt.xlabel('WISE w1mpro') # plt.ylabel('SDSS psfflux_r') # ps.savefig() for band in 'riz': ax = [0, 10, 25, 5] plt.clf() mag = sdss.get('psfmag_' + band)[J] loghist(mag - wise.w1mpro[I], mag, 200, range=((ax[0], ax[1]), (ax[3], ax[2]))) plt.xlabel('SDSS %s - WISE w1' % band) plt.ylabel('SDSS ' + band) plt.axis(ax) ps.savefig() for w, t in [(wise[I], 'Matched'), (wun, 'Unmatched')]: plt.clf() w1 = w.get('w1mpro') w2 = w.get('w2mpro') ax = [-1, 3, 18, 6] loghist(w1 - w2, w1, 200, range=((ax[0], ax[1]), (ax[3], ax[2]))) plt.xlabel('W1 - W2') plt.ylabel('W1') plt.title('WISE CMD for %s sources' % t) plt.axis(ax) ps.savefig() sdssobj = DR9() band = 'r' RCF = np.unique(zip(sdss.run, sdss.camcol, sdss.field)) wcses = [] fns = [] pfn = 'wcses.pickle' if os.path.exists(pfn): print('Reading', pfn) wcses, fns = unpickle_from_file(pfn) else: for r, c, f in RCF: fn = sdssobj.retrieve('frame', r, c, f, band) print('got', fn) fns.append(fn) wcs = Tan(fn, 0) print('got wcs', wcs) wcses.append(wcs) pickle_to_file((wcses, fns), pfn) print('Wrote to', pfn) wisefns = glob(os.path.join(wisedatadir, 'level3', '*w1-int-3.fits')) wisewcs = [] for fn in wisefns: print('Reading', fn) wcs = anwcs(fn, 0) print('Got', wcs) wisewcs.append(wcs) I = np.argsort(wun.w1mpro) wun.cut(I) for i in range(len(wun)): ra, dec = wun.ra[i], wun.dec[i] insdss = -1 for j, wcs in enumerate(wcses): if wcs.is_inside(ra, dec): insdss = j break inwise = -1 for j, wcs in enumerate(wisewcs): if wcs.is_inside(ra, dec): inwise = j break N = 0 if insdss != -1: N += 1 if inwise != -1: N += 1 if N == 0: continue if N != 2: continue plt.clf() ss = 1 plt.subplot(2, N, ss) ss += 1 M = 0.02 I = np.flatnonzero((sdss.ra > (ra - M)) * (sdss.ra < (ra + M)) * (sdss.dec > (dec - M)) * (sdss.dec < (dec + M))) sdssnear = sdss[I] plt.plot(sdss.ra[I], sdss.dec[I], 'b.', alpha=0.7) I = np.flatnonzero((wise.ra > (ra - M)) * (wise.ra < (ra + M)) * (wise.dec > (dec - M)) * (wise.dec < (dec + M))) wisenear = wise[I] plt.plot(wise.ra[I], wise.dec[I], 'rx', alpha=0.7) if insdss: wcs = wcses[j] w, h = wcs.imagew, wcs.imageh rd = np.array([wcs.pixelxy2radec(x, y) for x, y in [(1, 1), (w, 1), (w, h), (1, h), (1, 1)]]) plt.plot(rd[:, 0], rd[:, 1], 'b-', alpha=0.5) if inwise: wcs = wisewcs[j] w, h = wcs.imagew, wcs.imageh rd = np.array([wcs.pixelxy2radec(x, y) for x, y in [(1, 1), (w, 1), (w, h), (1, h), (1, 1)]]) plt.plot(rd[:, 0], rd[:, 1], 'r-', alpha=0.5) plt.plot([ra], [dec], 'o', mec='k', mfc='none', mew=3, ms=20, alpha=0.5) #plt.axis([ra+M, ra-M, dec-M, dec+M]) plt.axis([ra - M, ra + M, dec - M, dec + M]) plt.xticks([ra], ['RA = %0.3f' % ra]) plt.yticks([dec], ['Dec = %0.3f' % dec]) SW = 20 ss = N + 1 plt.subplot(2, N, ss) if insdss != -1: ss += 1 j = insdss wcs = wcses[j] xc, yc = wcs.radec2pixelxy(ra, dec) r, c, f = RCF[j] frame = sdssobj.readFrame(r, c, f, band) #S = 50 S = SW * 3.472 im = frame.image H, W = im.shape y0, x0 = max(0, yc - S), max(0, xc - S) y1, x1 = min(H, yc + S), min(W, xc + S) subim = im[y0:y1, x0:x1] # plt.imshow(subim, interpolation='nearest', origin='lower', # vmax=0.3, extent=[x0,x1,y0,y1]) plt.imshow(subim.T, interpolation='nearest', origin='lower', vmax=0.3, extent=[y0, y1, x0, x1]) # vmax=subim.max()*1.01) ax = plt.axis() sx, sy = wcs.radec2pixelxy(sdssnear.ra, sdssnear.dec) #plt.plot(x, y, 'o', mec='b', mfc='none', ms=15) plt.plot(sy, sx, 'o', mec='b', mfc='none', ms=15) x, y = wcs.radec2pixelxy(wisenear.ra, wisenear.dec) #plt.plot(x, y, 'rx', ms=10) plt.plot(y, x, 'rx', ms=10) # Which way up? x, y = wcs.radec2pixelxy(np.array([ra, ra]), np.array( [0.5, 2.0]) * S * 0.396 / 3600. + dec) #plt.plot(x, y, 'b-', alpha=0.5, lw=2) plt.plot(y, x, 'b-', alpha=0.5, lw=2) plt.axis(ax) plt.gray() plt.title('SDSS %s (%i/%i/%i)' % (band, r, c, f)) # Try to guess the PRIMARY run from the nearest object. for I in np.argsort((sx - xc)**2 + (sy - yc)**2): r, c, f = sdssnear.run[I], sdssnear.camcol[I], sdssnear.field[I] jj = None for j, (ri, ci, fi) in enumerate(RCF): if ri == r and ci == c and fi == f: jj = j break assert(jj is not None) wcs = wcses[jj] xc, yc = wcs.radec2pixelxy(ra, dec) frame = sdssobj.readFrame(r, c, f, band) S = SW * 3.472 im = frame.image H, W = im.shape y0, x0 = max(0, yc - S), max(0, xc - S) y1, x1 = min(H, yc + S), min(W, xc + S) subim = im[y0:y1, x0:x1] if np.prod(subim.shape) == 0: continue print('subim shape', subim.shape) plt.subplot(2, N, 2) plt.imshow(subim.T, interpolation='nearest', origin='lower', vmax=0.3, extent=[y0, y1, x0, x1]) plt.gray() plt.title('SDSS %s (%i/%i/%i)' % (band, r, c, f)) break if inwise != -1: plt.subplot(2, N, ss) ss += 1 j = inwise wcs = wisewcs[j] ok, x, y = wcs.radec2pixelxy(ra, dec) im = pyfits.open(wisefns[j])[0].data S = SW H, W = im.shape y0, x0 = max(0, y - S), max(0, x - S) subim = im[y0: min(H, y + S), x0: min(W, x + S)] plt.imshow(subim, interpolation='nearest', origin='lower', vmax=subim.max() * 1.01) ax = plt.axis() x, y = [], [] for r, d in zip(wisenear.ra, wisenear.dec): ok, xi, yi = wcs.radec2pixelxy(r, d) x.append(xi) y.append(yi) x = np.array(x) y = np.array(y) plt.plot(x - x0, y - y0, 'rx', ms=15) x, y = [], [] for r, d in zip(sdssnear.ra, sdssnear.dec): ok, xi, yi = wcs.radec2pixelxy(r, d) x.append(xi) y.append(yi) x = np.array(x) y = np.array(y) plt.plot(x - x0, y - y0, 'o', mec='b', mfc='none', ms=10) # Which way up? pixscale = 1.375 / 3600. ok, x1, y1 = wcs.radec2pixelxy(ra, dec + 0.5 * S * pixscale) ok, x2, y2 = wcs.radec2pixelxy(ra, dec + 2.0 * S * pixscale) plt.plot([x1 - x0, x2 - x0], [y1 - y0, y2 - y0], 'r-', alpha=0.5, lw=2) plt.axis([ax[1], ax[0], ax[2], ax[3]]) # plt.axis(ax) plt.gray() plt.title('WISE W1 (coadd)') plt.suptitle('WISE unmatched source: w1=%.1f, RA,Dec = (%.3f, %.3f)' % (wun.w1mpro[i], ra, dec)) ps.savefig() rcfs = zip(sdssnear.run, sdssnear.camcol, sdssnear.field) print('Nearby SDSS sources are from:', np.unique(rcfs)) return
def forcedphot(): T1 = fits_table('cs82data/cas-primary-DR8.fits') print(len(T1), 'primary') T1.cut(T1.nchild == 0) print(len(T1), 'children') rl, rh = T1.ra.min(), T1.ra.max() dl, dh = T1.dec.min(), T1.dec.max() tims = [] # Coadd basedir = os.path.join('cs82data', 'wise', 'level3') basefn = os.path.join(basedir, '3342p000_ab41-w1') tim = read_wise_coadd(basefn, radecroi=[rl, rh, dl, dh], nanomaggies=True) tims.append(tim) # Individuals basedir = os.path.join('cs82data', 'wise', 'level1b') for fn in ['04933b137-w1', '04937b137-w1', '04941b137-w1', '04945b137-w1', '04948a112-w1', '04949b137-w1', '04952a112-w1', '04953b137-w1', '04956a112-w1', '04960a112-w1', '04964a112-w1', '04968a112-w1', '05204a106-w1']: basefn = os.path.join(basedir, fn) tim = read_wise_image( basefn, radecroi=[rl, rh, dl, dh], nanomaggies=True) tims.append(tim) # tractor.Image's setMask() does a binary dilation on bad pixels! for tim in tims: # tim.mask = np.zeros(tim.shape, dtype=bool) tim.invvar = tim.origInvvar tim.mask = np.zeros(tim.shape, dtype=bool) print('tim:', tim) ps = PlotSequence('forced') plt.clf() plt.plot(T1.ra, T1.dec, 'r.') for tim in tims: wcs = tim.getWcs() H, W = tim.shape rr, dd = [], [] for x, y in zip([1, 1, W, W, 1], [1, H, H, 1, 1]): rd = wcs.pixelToPosition(x, y) rr.append(rd.ra) dd.append(rd.dec) plt.plot(rr, dd, 'k-', alpha=0.5) # setRadecAxes(rl,rh,dl,dh) ps.savefig() T2 = fits_table('wise-cut.fits') T2.w1 = T2.w1mpro R = 1. / 3600. I, J, d = match_radec(T1.ra, T1.dec, T2.ra, T2.dec, R) print(len(I), 'matches') refband = 'r' #bandnum = band_index('r') Lstar = (T1.probpsf == 1) * 1.0 Lgal = (T1.probpsf == 0) fracdev = T1.get('fracdev_%s' % refband) Ldev = Lgal * fracdev Lexp = Lgal * (1. - fracdev) ndev, nexp, ncomp, nstar = 0, 0, 0, 0 cat = Catalog() # for i,t in enumerate(T1): jmatch = np.zeros(len(T1)) jmatch[:] = -1 jmatch[I] = J for i in range(len(T1)): j = jmatch[i] if j >= 0: # match source: grab WISE catalog mag w1 = T2.w1[j] else: # unmatched: set it faint w1 = 18. bright = NanoMaggies(w1=NanoMaggies.magToNanomaggies(w1)) pos = RaDecPos(T1.ra[i], T1.dec[i]) if Lstar[i] > 0: # Star star = PointSource(pos, bright) cat.append(star) nstar += 1 continue hasdev = (Ldev[i] > 0) hasexp = (Lexp[i] > 0) iscomp = (hasdev and hasexp) if iscomp: dbright = bright * Ldev[i] ebright = bright * Lexp[i] elif hasdev: dbright = bright elif hasexp: ebright = bright else: assert(False) if hasdev: re = T1.get('devrad_%s' % refband)[i] ab = T1.get('devab_%s' % refband)[i] phi = T1.get('devphi_%s' % refband)[i] dshape = GalaxyShape(re, ab, phi) if hasexp: re = T1.get('exprad_%s' % refband)[i] ab = T1.get('expab_%s' % refband)[i] phi = T1.get('expphi_%s' % refband)[i] eshape = GalaxyShape(re, ab, phi) if iscomp: gal = CompositeGalaxy(pos, ebright, eshape, dbright, dshape) ncomp += 1 elif hasdev: gal = DevGalaxy(pos, dbright, dshape) ndev += 1 elif hasexp: gal = ExpGalaxy(pos, ebright, eshape) nexp += 1 cat.append(gal) print('Created', ndev, 'pure deV', nexp, 'pure exp and', end=' ') print(ncomp, 'composite galaxies', end=' ') print('and', nstar, 'stars') tractor = Tractor(tims, cat) for i, tim in enumerate(tims): ima = dict(interpolation='nearest', origin='lower', vmin=tim.zr[0], vmax=tim.zr[1]) mod = tractor.getModelImage(i) plt.clf() plt.imshow(mod, **ima) plt.gray() plt.title('model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(tim.getImage(), **ima) plt.gray() plt.title('data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(tim.getInvvar(), interpolation='nearest', origin='lower') plt.gray() plt.title('invvar') ps.savefig() # for tim in tims: wcs = tim.getWcs() H, W = tim.shape poly = [] for r, d in zip([rl, rl, rh, rh, rl], [dl, dh, dh, dl, dl]): x, y = wcs.positionToPixel(RaDecPos(r, d)) poly.append((x, y)) xx, yy = np.meshgrid(np.arange(W), np.arange(H)) xy = np.vstack((xx.flat, yy.flat)).T grid = points_inside_poly(xy, poly) grid = grid.reshape((H, W)) tim.setInvvar(tim.getInvvar() * grid) # plt.clf() # plt.imshow(grid, interpolation='nearest', origin='lower') # plt.gray() # ps.savefig() plt.clf() plt.imshow(tim.getInvvar(), interpolation='nearest', origin='lower') plt.gray() plt.title('invvar') ps.savefig() if i == 1: plt.clf() plt.imshow(tim.goodmask, interpolation='nearest', origin='lower') plt.gray() plt.title('goodmask') ps.savefig() miv = (1. / (tim.uncplane)**2) for bit in range(-1, 32): if bit >= 0: miv[(tim.maskplane & (1 << bit)) != 0] = 0. if bit == 31: plt.clf() plt.imshow(miv, interpolation='nearest', origin='lower') plt.gray() plt.title( 'invvar with mask bits up to %i blanked out' % bit) ps.savefig()
def main(): # ps = PlotSequence('pro') # star_profiles(ps) # sys.exit(0) #survey_dir = '/project/projectdirs/desiproc/dr3' #survey = LegacySurveyData(survey_dir=survey_dir) survey = LegacySurveyData() ralo,rahi = 240,245 declo,dechi = 5, 12 ps = PlotSequence('comp') bands = 'grz' ccdfn = 'ccds-forced.fits' if not os.path.exists(ccdfn): ccds = survey.get_annotated_ccds() ccds.cut((ccds.ra > ralo) * (ccds.ra < rahi) * (ccds.dec > declo) * (ccds.dec < dechi)) print(len(ccds), 'CCDs') ccds.path = np.array([os.path.join(#'dr3', 'forced', ('%08i' % e)[:5], '%08i' % e, 'decam-%08i-%s-forced.fits' % (e, n.strip())) for e,n in zip(ccds.expnum, ccds.ccdname)]) I, = np.nonzero([os.path.exists(fn) for fn in ccds.path]) print(len(I), 'CCDs with forced photometry') ccds.cut(I) #ccds = ccds[:500] #e,I = np.unique(ccds.expnum, return_index=True) #print(len(I), 'unique exposures') #ccds.cut(I) FF = read_forcedphot_ccds(ccds, survey) FF.writeto('forced-all-matches.fits') # - z band -- no trend w/ PS1 mag (brighter-fatter) ccds.writeto(ccdfn) ccdfn2 = 'ccds-forced-2.fits' if not os.path.exists(ccdfn2): ccds = fits_table(ccdfn) # Split into brighter/fainter halves FF = fits_table('forced-all-matches.fits') print(len(FF), 'forced measurements') FF.cut(FF.masked == False) print(len(FF), 'forced measurements not masked') ccds.brightest_mdiff = np.zeros(len(ccds)) ccds.brightest_mscatter = np.zeros(len(ccds)) ccds.bright_mdiff = np.zeros(len(ccds)) ccds.bright_mscatter = np.zeros(len(ccds)) ccds.faint_mdiff = np.zeros(len(ccds)) ccds.faint_mscatter = np.zeros(len(ccds)) for iccd in range(len(ccds)): I = np.flatnonzero(FF.iforced == iccd) if len(I) == 0: continue if len(I) < 10: continue F = FF[I] b = np.percentile(F.psmag, 10) m = np.median(F.psmag) print(len(F), 'matches for CCD', iccd, 'median mag', m, '10th pct', b) J = np.flatnonzero(F.psmag < b) diff = F.mag[J] - F.psmag[J] ccds.brightest_mdiff[iccd] = np.median(diff) ccds.brightest_mscatter[iccd] = (np.percentile(diff, 84) - np.percentile(diff, 16))/2. J = np.flatnonzero(F.psmag < m) diff = F.mag[J] - F.psmag[J] ccds.bright_mdiff[iccd] = np.median(diff) ccds.bright_mscatter[iccd] = (np.percentile(diff, 84) - np.percentile(diff, 16))/2. J = np.flatnonzero(F.psmag > m) diff = F.mag[J] - F.psmag[J] ccds.faint_mdiff[iccd] = np.median(diff) ccds.faint_mscatter[iccd] = (np.percentile(diff, 84) - np.percentile(diff, 16))/2. ccds.writeto(ccdfn2) ccds = fits_table(ccdfn2) plt.clf() plt.hist(ccds.nforced, bins=100) plt.title('nforced') ps.savefig() plt.clf() plt.hist(ccds.nmatched, bins=100) plt.title('nmatched') ps.savefig() #ccds.cut(ccds.nmatched >= 150) ccds.cut(ccds.nmatched >= 50) print('Cut to', len(ccds), 'with >50 matched') ccds.cut(ccds.photometric) print('Cut to', len(ccds), 'photometric') neff = 1. / ccds.psfnorm_mean**2 # Narcsec is in arcsec**2 narcsec = neff * ccds.pixscale_mean**2 # to arcsec narcsec = np.sqrt(narcsec) # Correction factor to get back to equivalent of Gaussian sigma narcsec /= (2. * np.sqrt(np.pi)) # Conversion factor to FWHM (2.35) narcsec *= 2. * np.sqrt(2. * np.log(2.)) ccds.psfsize = narcsec for band in bands: I = np.flatnonzero((ccds.filter == band) * (ccds.photometric) * (ccds.blacklist_ok)) mlo,mhi = -0.01, 0.05 plt.clf() plt.plot(ccds.ccdzpt[I], ccds.exptime[I], 'k.', alpha=0.1) J = np.flatnonzero((ccds.filter == band) * (ccds.photometric == False)) plt.plot(ccds.ccdzpt[J], ccds.exptime[J], 'r.', alpha=0.1) plt.xlabel('Zeropoint (mag)') plt.ylabel('exptime') plt.title('DR3: EDR region, Forced phot: %s band' % band) ps.savefig() plt.clf() plt.plot(ccds.ccdzpt[I], np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1) plt.xlabel('Zeropoint (mag)') plt.ylabel('DECaLS PSF - PS1 (mag)') plt.axhline(0, color='k', alpha=0.2) #plt.axis([0, mxsee, mlo,mhi]) plt.title('DR3: EDR region, Forced phot: %s band' % band) ps.savefig() plt.clf() plt.plot(ccds.ccdzpt[I], ccds.psfsize[I], 'k.', alpha=0.1) plt.xlabel('Zeropoint (mag)') plt.ylabel('PSF size (arcsec)') plt.title('DR3: EDR region, Forced phot: %s band' % band) ps.savefig() # plt.clf() # plt.plot(ccds.avsky[I], # np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1) # plt.xlabel('avsky') # plt.ylabel('DECaLS PSF - PS1 (mag)') # plt.axhline(0, color='k', alpha=0.2) # plt.title('DR3: EDR region, Forced phot: %s band' % band) # ps.savefig() # # plt.clf() # plt.plot(ccds.meansky[I], # np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1) # plt.xlabel('meansky') # plt.ylabel('DECaLS PSF - PS1 (mag)') # plt.axhline(0, color='k', alpha=0.2) # plt.title('DR3: EDR region, Forced phot: %s band' % band) # ps.savefig() plt.clf() lo,hi = (-0.02, 0.05) ha = dict(bins=50, histtype='step', range=(lo,hi)) n,b,p1 = plt.hist(ccds.brightest_mdiff[I], color='r', **ha) n,b,p2 = plt.hist(ccds.bright_mdiff[I], color='g', **ha) n,b,p3 = plt.hist(ccds.faint_mdiff[I], color='b', **ha) plt.legend((p1[0],p2[0],p3[0]), ('Brightest 10%', 'Brightest 50%', 'Faintest 50%')) plt.xlabel('DECaLS PSF - PS1 (mag)') plt.ylabel('Number of CCDs') plt.title('DR3: EDR region, Forced phot: %s band' % band) plt.xlim(lo,hi) ps.savefig() for band in bands: I = np.flatnonzero(ccds.filter == band) mxsee = 4. mlo,mhi = -0.01, 0.05 plt.clf() plt.plot(np.clip(ccds.psfsize[I], 0, mxsee), np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1) # for p in [1,2,3]: # J = np.flatnonzero(ccds.tilepass[I] == p) # if len(J): # plt.plot(np.clip(ccds.psfsize[I[J]], 0, mxsee), # np.clip(ccds.mdiff[I[J]], mlo,mhi), '.', color='rgb'[p-1], alpha=0.2) #plt.plot(ccds.seeing[I], ccds.mdiff[I], 'b.') plt.xlabel('PSF size (arcsec)') plt.ylabel('DECaLS PSF - PS1 (mag)') plt.axhline(0, color='k', alpha=0.2) plt.axis([0, mxsee, mlo,mhi]) plt.title('DR3: EDR region, Forced phot: %s band' % band) ps.savefig() # Group by exposure for band in bands: I = np.flatnonzero((ccds.filter == band) * (ccds.photometric) * (ccds.blacklist_ok)) E,J = np.unique(ccds.expnum[I], return_index=True) print(len(E), 'unique exposures in', band) exps = ccds[I[J]] print(len(exps), 'unique exposures in', band) assert(len(np.unique(exps.expnum)) == len(exps)) exps.ddiff = np.zeros(len(exps)) exps.dsize = np.zeros(len(exps)) exps.nccds = np.zeros(len(exps), int) exps.brightest_ddiff = np.zeros(len(exps)) exps.bright_ddiff = np.zeros(len(exps)) exps.faint_ddiff = np.zeros(len(exps)) for iexp,exp in enumerate(exps): J = np.flatnonzero(ccds.expnum[I] == exp.expnum) J = I[J] print(len(J), 'CCDs in exposure', exp.expnum) exps.brightest_mdiff[iexp] = np.median(ccds.brightest_mdiff[J]) exps.bright_mdiff[iexp] = np.median(ccds.bright_mdiff[J]) exps.faint_mdiff[iexp] = np.median(ccds.faint_mdiff[J]) exps.brightest_ddiff[iexp] = ( np.percentile(ccds.brightest_mdiff[J], 84) - np.percentile(ccds.brightest_mdiff[J], 16))/2. exps.bright_ddiff[iexp] = ( np.percentile(ccds.bright_mdiff[J], 84) - np.percentile(ccds.bright_mdiff[J], 16))/2. exps.faint_ddiff[iexp] = ( np.percentile(ccds.faint_mdiff[J], 84) - np.percentile(ccds.faint_mdiff[J], 16))/2. exps.mdiff[iexp] = np.median(ccds.mdiff[J]) exps.ddiff[iexp] = (np.percentile(ccds.mdiff[J], 84) - np.percentile(ccds.mdiff[J], 16))/2. exps.psfsize[iexp] = np.median(ccds.psfsize[J]) exps.dsize[iexp] = (np.percentile(ccds.psfsize[J], 84) - np.percentile(ccds.psfsize[J], 16))/2. exps.nccds[iexp] = len(J) mxsee = 4. mlo,mhi = -0.01, 0.05 exps.cut(exps.nccds >= 10) plt.clf() plt.errorbar(np.clip(exps.psfsize, 0, mxsee), np.clip(exps.mdiff, mlo,mhi), yerr=exps.ddiff, #xerr=exps.dsize, fmt='.', color='k') # plt.errorbar(np.clip(exps.psfsize, 0, mxsee), # np.clip(exps.brightest_mdiff, mlo,mhi), # yerr=exps.brightest_ddiff, fmt='r.') # plt.errorbar(np.clip(exps.psfsize, 0, mxsee), # np.clip(exps.bright_mdiff, mlo,mhi), # yerr=exps.bright_ddiff, fmt='g.') # plt.errorbar(np.clip(exps.psfsize, 0, mxsee), # np.clip(exps.faint_mdiff, mlo,mhi), # yerr=exps.faint_ddiff, fmt='b.') # plt.plot(np.clip(exps.psfsize, 0, mxsee), # np.clip(exps.brightest_mdiff, mlo,mhi), 'r.') # plt.plot(np.clip(exps.psfsize, 0, mxsee), # np.clip(exps.bright_mdiff, mlo,mhi), 'g.') # plt.plot(np.clip(exps.psfsize, 0, mxsee), # np.clip(exps.faint_mdiff, mlo,mhi), 'b.') #plt.plot(ccds.seeing[I], ccds.mdiff[I], 'b.') plt.xlabel('PSF size (arcsec)') plt.ylabel('DECaLS PSF - PS1 (mag)') plt.axhline(0, color='k', alpha=0.2) plt.axis([0, mxsee, mlo,mhi]) plt.title('DR3: EDR region, Forced phot: %s band' % band) ps.savefig() plt.clf() p1 = plt.plot(np.clip(exps.psfsize, 0, mxsee), np.clip(exps.brightest_mdiff, mlo,mhi), 'r.', alpha=0.5) p2 = plt.plot(np.clip(exps.psfsize, 0, mxsee), np.clip(exps.bright_mdiff, mlo,mhi), 'g.', alpha=0.5) p3 = plt.plot(np.clip(exps.psfsize, 0, mxsee), np.clip(exps.faint_mdiff, mlo,mhi), 'b.', alpha=0.5) plt.legend((p1[0],p2[0],p3[0]), ('Brightest 10%', 'Brightest 50%', 'Faintest 50%')) plt.xlabel('PSF size (arcsec)') plt.ylabel('DECaLS PSF - PS1 (mag)') plt.axhline(0, color='k', alpha=0.2) plt.axis([0, mxsee, mlo,mhi]) plt.title('DR3: EDR region, Forced phot: %s band' % band) ps.savefig() J = np.argsort(-exps.mdiff) for j in J: print(' Photometric diff', exps.mdiff[j], 'PSF size', exps.psfsize[j], 'expnum', exps.expnum[j]) sys.exit(0)
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None, use_ceres=True, ceres_block=8, save_fits=False, get_models=False, ps=None, psf_broadening=None, pixelized_psf=False, get_masks=None, move_crpix=False, modelsky_dir=None, tag=None): ''' Given a list of tractor sources *cat* and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id) runs forced photometry, returning a FITS table the same length as *cat*. *get_masks*: the WCS to resample mask bits into. ''' from tractor import PointSource, Tractor, ExpGalaxy, DevGalaxy from tractor.sersic import SersicGalaxy if tag is None: tag = '' else: tag = tag + ': ' if not pixelized_psf and psf_broadening is None: # PSF broadening in post-reactivation data, by band. # Newer version from Aaron's email to decam-chatter, 2018-06-14. broadening = {1: 1.0405, 2: 1.0346, 3: None, 4: None} psf_broadening = broadening[band] if False: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('wise-forced-w%i' % band) plots = (ps is not None) if plots: import pylab as plt wantims = (plots or save_fits or get_models) wanyband = 'w' if get_models: models = [] wband = 'w%i' % band Nsrcs = len(cat) phot = fits_table() # Filled in based on unique tile overlap phot.wise_coadd_id = np.array([' '] * Nsrcs, dtype='U8') phot.wise_x = np.zeros(Nsrcs, np.float32) phot.wise_y = np.zeros(Nsrcs, np.float32) phot.set('psfdepth_%s' % wband, np.zeros(Nsrcs, np.float32)) nexp = np.zeros(Nsrcs, np.int16) mjd = np.zeros(Nsrcs, np.float64) central_flux = np.zeros(Nsrcs, np.float32) ra = np.array([src.getPosition().ra for src in cat]) dec = np.array([src.getPosition().dec for src in cat]) fskeys = ['prochi2', 'profracflux'] fitstats = {} if get_masks: mh, mw = get_masks.shape maskmap = np.zeros((mh, mw), np.uint32) tims = [] for tile in tiles: info(tag + 'Reading WISE tile', tile.coadd_id, 'band', band) tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradecbox) if tim is None: debug('Actually, no overlap with WISE coadd tile', tile.coadd_id) continue if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data' % tag) ps.savefig() plt.clf() plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), range=(-5, 10), bins=100) plt.xlabel('Per-pixel intensity (Sigma)') plt.title(tag) ps.savefig() if move_crpix and band in [1, 2]: realwcs = tim.wcs.wcs x, y = realwcs.crpix tile_crpix = tile.get('crpix_w%i' % band) dx = tile_crpix[0] - 1024.5 dy = tile_crpix[1] - 1024.5 realwcs.set_crpix(x + dx, y + dy) debug('unWISE', tile.coadd_id, 'band', band, 'CRPIX', x, y, 'shift by', dx, dy, 'to', realwcs.crpix) if modelsky_dir and band in [1, 2]: fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band)) if not os.path.exists(fn): raise RuntimeError('WARNING: does not exist:', fn) x0, x1, y0, y1 = tim.roi bg = fitsio.FITS(fn)[2][y0:y1, x0:x1] assert (bg.shape == tim.shape) if plots: plt.clf() plt.subplot(1, 2, 1) plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) plt.subplot(1, 2, 2) plt.imshow(bg, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) tag = '%s W%i' % (tile.coadd_id, band) plt.suptitle(tag) ps.savefig() plt.clf() ha = dict(range=(-5, 10), bins=100, histtype='step') plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), color='b', label='Original', **ha) plt.hist(((tim.getImage() - bg) * tim.inverr)[tim.inverr > 0].ravel(), color='g', label='Minus Background', **ha) plt.axvline(0, color='k', alpha=0.5) plt.xlabel('Per-pixel intensity (Sigma)') plt.legend() plt.title(tag + ': background') ps.savefig() # Actually subtract the background! tim.data -= bg # Floor the per-pixel variances, # and add Poisson contribution from sources if band in [1, 2]: # in Vega nanomaggies per pixel floor_sigma = {1: 0.5, 2: 2.0} poissons = {1: 0.15, 2: 0.3} with np.errstate(divide='ignore'): new_ie = 1. / np.sqrt( (1. / tim.inverr)**2 + floor_sigma[band] + poissons[band]**2 * np.maximum(0., tim.data)) new_ie[tim.inverr == 0] = 0. if plots: plt.clf() plt.plot((1. / tim.inverr[tim.inverr > 0]).ravel(), (1. / new_ie[tim.inverr > 0]).ravel(), 'b.') plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band)) plt.xlabel('original') plt.ylabel('floored') ps.savefig() assert (np.all(np.isfinite(new_ie))) assert (np.all(new_ie >= 0.)) tim.inverr = new_ie # Expand a 3-pixel radius around weight=0 (saturated) pixels # from Eddie via crowdsource # https://github.com/schlafly/crowdsource/blob/7069da3e7d9d3124be1cbbe1d21ffeb63fc36dcc/python/wise_proc.py#L74 ## FIXME -- W3/W4 ?? satlimit = 85000 msat = ((tim.data > satlimit) | ((tim.nims == 0) & (tim.nuims > 1))) from scipy.ndimage.morphology import binary_dilation xx, yy = np.mgrid[-3:3 + 1, -3:3 + 1] dilate = xx**2 + yy**2 <= 3**2 msat = binary_dilation(msat, dilate) nbefore = np.sum(tim.inverr == 0) tim.inverr[msat] = 0 nafter = np.sum(tim.inverr == 0) debug('Masking an additional', (nafter - nbefore), 'near-saturated pixels in unWISE', tile.coadd_id, 'band', band) # Read mask file? if get_masks: from astrometry.util.resample import resample_with_wcs, OverlapError # unwise_dir can be a colon-separated list of paths tilemask = None for d in tile.unwise_dir.split(':'): fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id, 'unwise-%s-msk.fits.gz' % tile.coadd_id) if os.path.exists(fn): debug('Reading unWISE mask file', fn) x0, x1, y0, y1 = tim.roi tilemask = fitsio.FITS(fn)[0][y0:y1, x0:x1] break if tilemask is None: info('unWISE mask file for tile', tile.coadd_id, 'does not exist') else: try: tanwcs = tim.wcs.wcs assert (tanwcs.shape == tilemask.shape) Yo, Xo, Yi, Xi, _ = resample_with_wcs(get_masks, tanwcs, intType=np.int16) # Only deal with mask pixels that are set. I, = np.nonzero(tilemask[Yi, Xi] > 0) # Trim to unique area for this tile rr, dd = get_masks.pixelxy2radec(Xo[I] + 1, Yo[I] + 1) good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I = I[good] maskmap[Yo[I], Xo[I]] = tilemask[Yi[I], Xi[I]] except OverlapError: # Shouldn't happen by this point print('Warning: no overlap between WISE tile', tile.coadd_id, 'and brick') if plots: plt.clf() plt.imshow(tilemask, interpolation='nearest', origin='lower') plt.title('Tile %s: mask' % tile.coadd_id) ps.savefig() plt.clf() plt.imshow(maskmap, interpolation='nearest', origin='lower') plt.title('Tile %s: accumulated maskmap' % tile.coadd_id) ps.savefig() # The tiles have some overlap, so zero out pixels outside the # tile's unique area. th, tw = tim.shape xx, yy = np.meshgrid(np.arange(tw), np.arange(th)) rr, dd = tim.wcs.wcs.pixelxy2radec(xx + 1, yy + 1) unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) debug('Tile', tile.coadd_id, '- total of', np.sum(unique), 'unique pixels out of', len(unique.flat), 'total pixels') if get_models: # Save the inverr before blanking out non-unique pixels, for making coadds with no gaps! # (actually, slightly more subtly, expand unique area by 1 pixel) from scipy.ndimage.morphology import binary_dilation du = binary_dilation(unique) tim.coadd_inverr = tim.inverr * du tim.inverr[unique == False] = 0. del xx, yy, rr, dd, unique if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage() * (tim.inverr > 0), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data (unique)' % tag) ps.savefig() if pixelized_psf: from unwise_psf import unwise_psf if (band == 1) or (band == 2): # we only have updated PSFs for W1 and W2 psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id, modelname='neo6_unwisecat') else: psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id) if band == 4: # oversample (the unwise_psf models are at native W4 5.5"/pix, # while the unWISE coadds are made at 2.75"/pix. ph, pw = psfimg.shape subpsf = np.zeros((ph * 2 - 1, pw * 2 - 1), np.float32) from astrometry.util.util import lanczos3_interpolate xx, yy = np.meshgrid( np.arange(0., pw - 0.51, 0.5, dtype=np.float32), np.arange(0., ph - 0.51, 0.5, dtype=np.float32)) xx = xx.ravel() yy = yy.ravel() ix = xx.astype(np.int32) iy = yy.astype(np.int32) dx = (xx - ix).astype(np.float32) dy = (yy - iy).astype(np.float32) psfimg = psfimg.astype(np.float32) rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg]) if plots: plt.clf() plt.imshow(psfimg, interpolation='nearest', origin='lower') plt.title('Original PSF model') ps.savefig() plt.clf() plt.imshow(subpsf, interpolation='nearest', origin='lower') plt.title('Subsampled PSF model') ps.savefig() psfimg = subpsf del xx, yy, ix, iy, dx, dy from tractor.psf import PixelizedPSF psfimg /= psfimg.sum() fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0} psfimg *= fluxrescales[band] tim.psf = PixelizedPSF(psfimg) if psf_broadening is not None and not pixelized_psf: # psf_broadening is a factor by which the PSF FWHMs # should be scaled; the PSF is a little wider # post-reactivation. psf = tim.getPsf() from tractor import GaussianMixturePSF if isinstance(psf, GaussianMixturePSF): debug('Broadening PSF: from', psf) p0 = psf.getParams() pnames = psf.getParamNames() p1 = [ p * psf_broadening**2 if 'var' in name else p for (p, name) in zip(p0, pnames) ] psf.setParams(p1) debug('Broadened PSF:', psf) else: print( 'WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf)) wcs = tim.wcs.wcs _, fx, fy = wcs.radec2pixelxy(ra, dec) x = np.round(fx - 1.).astype(int) y = np.round(fy - 1.).astype(int) good = (x >= 0) * (x < tw) * (y >= 0) * (y < th) # Which sources are in this brick's unique area? usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I, = np.nonzero(good * usrc) nexp[I] = tim.nuims[y[I], x[I]] if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): mjd[I] = (tim.mjdmin + tim.mjdmax) / 2. phot.wise_coadd_id[I] = tile.coadd_id phot.wise_x[I] = fx[I] - 1. phot.wise_y[I] = fy[I] - 1. central_flux[I] = tim.getImage()[y[I], x[I]] del x, y, good, usrc # PSF norm for depth psf = tim.getPsf() h, w = tim.shape patch = psf.getPointSourcePatch(h // 2, w // 2).patch psfnorm = np.sqrt(np.sum(patch**2)) # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags. # In the small empty patches of the sky (eg W4 in 0922p702), we get sig1 = NaN if np.isfinite(tim.sig1): phot.get('psfdepth_%s' % wband)[I] = 1. / (tim.sig1 / psfnorm)**2 tim.tile = tile tims.append(tim) if plots: plt.clf() mn, mx = 0.1, 20000 plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100, range=(np.log10(mn), np.log10(mx))) logt = np.arange(0, 5) plt.xticks(logt, ['%i' % i for i in 10.**logt]) plt.title('Central fluxes (W%i)' % band) plt.axvline(np.log10(20000), color='k') plt.axvline(np.log10(1000), color='k') ps.savefig() # Eddie's non-secret recipe: #- central pixel <= 1000: 19x19 pix box size #- central pixel in 1000 - 20000: 59x59 box size #- central pixel > 20000 or saturated: 149x149 box size #- object near "bright star": 299x299 box size nbig = nmedium = nsmall = 0 for src, cflux in zip(cat, central_flux): if cflux > 20000: R = 100 nbig += 1 elif cflux > 1000: R = 30 nmedium += 1 else: R = 15 nsmall += 1 if isinstance(src, PointSource): src.fixedRadius = R else: ### FIXME -- sizes for galaxies..... can we set PSF size separately? galrad = 0 # RexGalaxy is a subclass of ExpGalaxy if isinstance(src, (ExpGalaxy, DevGalaxy, SersicGalaxy)): galrad = src.shape.re pixscale = 2.75 src.halfsize = int(np.hypot(R, galrad * 5 / pixscale)) debug('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small') tractor = Tractor(tims, cat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(wanyband) t0 = Time() R = tractor.optimize_forced_photometry(fitstats=True, variance=True, shared_params=False, wantims=wantims) info(tag + 'unWISE forced photometry took', Time() - t0) if use_ceres: term = R.ceres_status['termination'] # Running out of memory can cause failure to converge and term # status = 2. Fail completely in this case. if term != 0: info(tag + 'Ceres termination status:', term) raise RuntimeError('Ceres terminated with status %i' % term) if wantims: ims1 = R.ims1 # can happen if empty source list (we still want to generate coadds) if ims1 is None: ims1 = R.ims0 flux_invvars = R.IV if R.fitstats is not None: for k in fskeys: x = getattr(R.fitstats, k) fitstats[k] = np.array(x).astype(np.float32) if save_fits: for i, tim in enumerate(tims): tile = tim.tile (dat, mod, _, chi, _) = ims1[i] wcshdr = fitsio.FITSHDR() tim.wcs.wcs.add_to_header(wcshdr) tag = 'fit-%s-w%i' % (tile.coadd_id, band) fitsio.write('%s-data.fits' % tag, dat, clobber=True, header=wcshdr) fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr) fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr) if plots: # Create models for just the brightest sources bright_cat = [ src for src in cat if src.getBrightness().getBand(wanyband) > 1000 ] debug('Bright soures:', len(bright_cat)) btr = Tractor(tims, bright_cat) for tim in tims: mod = btr.getModelImage(tim) tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) sig1 = tim.sig1 plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: bright-star models' % tag) ps.savefig() if get_models: for i, tim in enumerate(tims): tile = tim.tile (dat, mod, _, _, _) = ims1[i] models.append( (tile.coadd_id, band, tim.wcs.wcs, dat, mod, tim.coadd_inverr)) if plots: for i, tim in enumerate(tims): tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) (dat, mod, _, chi, _) = ims1[i] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() nm = np.array([src.getBrightness().getBand(wanyband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their initial # fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. phot.set('flux_%s' % wband, nm.astype(np.float32)) phot.set('flux_ivar_%s' % wband, nm_ivar.astype(np.float32)) for k in fskeys: phot.set(k + '_' + wband, fitstats.get(k, np.zeros(len(phot), np.float32))) phot.set('nobs_%s' % wband, nexp) phot.set('mjd_%s' % wband, mjd) rtn = wphotduck() rtn.phot = phot rtn.models = None rtn.maskmap = None if get_models: rtn.models = models if get_masks: rtn.maskmap = maskmap return rtn
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('--plots', action='store_true') parser.add_argument('--brick', help='Brick name to run') parser.add_argument( '--input-dir', default='/global/projecta/projectdirs/cosmo/work/legacysurvey/dr7') #/global/cscratch1/sd/desiproc/dr7out') parser.add_argument('--survey-dir', default='/global/cscratch1/sd/dstn/dr7-depthcut') parser.add_argument('--output-dir', default='/global/cscratch1/sd/dstn/bright') opt = parser.parse_args() plots = opt.plots ps = PlotSequence('bright') brickname = opt.brick insurvey = LegacySurveyData(opt.input_dir, cache_dir=opt.survey_dir) outsurvey = LegacySurveyData(opt.output_dir, output_dir=opt.output_dir) bfn = insurvey.find_file('blobmap', brick=brickname) print('Found blob map', bfn) blobs = fitsio.read(bfn) h, w = blobs.shape brick = insurvey.get_brick_by_name(brickname) brickwcs = wcs_for_brick(brick) radius = np.sqrt(2.) * 0.25 * 1.01 neighbors = insurvey.get_bricks_near(brick.ra, brick.dec, radius) print(len(neighbors), 'bricks nearby') def showbool(X): d = downsample_max(X, 8) h, w = X.shape plt.imshow(d, interpolation='nearest', origin='lower', vmin=0, vmax=1, extent=[0, w, 0, h], cmap='gray') brightblobs = set() for nb in neighbors: if nb.brickname == brickname: # ignore myself! continue print('Neighbor:', nb.brickname) mfn = insurvey.find_file('maskbits', brick=nb.brickname) if not os.path.exists(mfn): print('No maskbits file:', mfn) continue maskbits = fitsio.read(mfn) bright = ((maskbits & MASKBITS['BRIGHT']) > 0) print(np.sum(bright > 0), 'BRIGHT pixels set') primary = (maskbits & MASKBITS['NPRIMARY'] == 0) print(np.sum(primary), 'PRIMARY pixels set') edge = binary_dilation(primary, structure=np.ones((3, 3), bool)) edge = edge * np.logical_not(primary) brightedge = edge & bright if plots: plt.clf() showbool(bright) plt.title('bright: brick %s' % nb.brickname) ps.savefig() # plt.clf() # showbool(primary) # plt.title('PRIMARY, brick %s' % nb.brickname) # ps.savefig() # # plt.clf() # showbool(edge) # plt.title('boundary, brick %s' % nb.brickname) # ps.savefig() plt.clf() showbool(brightedge) plt.title('bright at edge, brick %s' % nb.brickname) ps.savefig() nwcs = wcs_for_brick(nb) yy, xx = np.nonzero(brightedge) print(len(yy), 'bright edge pixels') if len(yy) == 0: continue rr, dd = nwcs.pixelxy2radec(xx + 1, yy + 1) print('RA range', rr.min(), rr.max(), 'vs brick', brick.ra1, brick.ra2) print('Dec range', dd.min(), dd.max(), 'vs brick', brick.dec1, brick.dec2) # Find pixels that are within this brick's unique area I, = np.nonzero((rr >= brick.ra1) * (rr <= brick.ra2) * (dd >= brick.dec1) * (dd <= brick.dec2)) if plots: plt.clf() plt.plot( [brick.ra1, brick.ra1, brick.ra2, brick.ra2, brick.ra1], [brick.dec1, brick.dec2, brick.dec2, brick.dec1, brick.dec1], 'b-') plt.plot(rr, dd, 'k.') plt.plot(rr[I], dd[I], 'r.') plt.title('Bright pixels from %s' % nb.brickname) ps.savefig() if len(I) == 0: print('No edge pixels touch') #plt.plot(br,bd, 'b-') continue #print('Edge pixels touch!') #plt.plot(br,bd, 'r-', zorder=20) ok, x, y = brickwcs.radec2pixelxy(rr[I], dd[I]) x = np.round(x).astype(int) - 1 y = np.round(y).astype(int) - 1 print('Pixel ranges X', x.min(), x.max(), 'Y', y.min(), y.max()) assert (np.all((x >= 0) * (x < w) * (y >= 0) * (y < h))) print('Adding blobs:', np.unique(blobs[y, x])) brightblobs.update(blobs[y, x]) print('Blobs touching bright pixels:', brightblobs) print() brightblobs.discard(-1) if len(brightblobs) == 0: print('No neighboring bright blobs to update!') return print('Updating', len(brightblobs), 'blobs:', brightblobs) tmap = np.zeros(blobs.max() + 2, bool) for b in brightblobs: tmap[b + 1] = True touching = tmap[blobs + 1] if plots: plt.clf() showbool(touching) plt.title('Blobs touching bright, brick %s' % brickname) ps.savefig() mfn = insurvey.find_file('maskbits', brick=brickname) maskbits, hdr = fitsio.read(mfn, header=True) updated = maskbits | (MASKBITS['BRIGHT'] * touching) if np.all(maskbits == updated): print('No bits updated! (Bright stars were already masked)') return maskbits = updated if plots: plt.clf() showbool((maskbits & MASKBITS['BRIGHT']) > 0) plt.title('New maskbits map for BRIGHT, brick %s' % brickname) ps.savefig() with outsurvey.write_output('maskbits', brick=brickname) as out: out.fits.write(maskbits, hdr=hdr) tfn = insurvey.find_file('tractor', brick=brickname) phdr = fitsio.read_header(tfn, ext=0) hdr = fitsio.read_header(tfn, ext=1) T = fits_table(tfn) print('Read', len(T), 'sources') print('Bright:', Counter(T.brightstarinblob)) iby = np.clip(np.round(T.by), 0, h - 1).astype(int) ibx = np.clip(np.round(T.bx), 0, w - 1).astype(int) if plots: before = np.flatnonzero(T.brightstarinblob) T.brightstarinblob |= touching[iby, ibx] print('Bright:', Counter(T.brightstarinblob)) # yuck -- copy the TUNIT headers from input to output. units = [ hdr.get('TUNIT%i' % (i + 1), '') for i in range(len(T.get_columns())) ] if plots: plt.clf() showbool((maskbits & MASKBITS['BRIGHT']) > 0) ax = plt.axis() after = np.flatnonzero(T.brightstarinblob) plt.plot(T.bx[before], T.by[before], 'gx') plt.plot(T.bx[after], T.by[after], 'r.') plt.axis(ax) plt.title('sources with brightstarinblob, brick %s' % brickname) ps.savefig() with outsurvey.write_output('tractor', brick=brickname) as out: T.writeto(None, fits_object=out.fits, primheader=phdr, header=hdr, units=units)
def main(): ps = PlotSequence('conv') S = 51 center = S / 2 print('Center', center) #for psf_sigma in [2., 1.5, 1.]: for psf_sigma in [2.]: rms2 = [] x = np.arange(S) y = np.arange(S) xx, yy = np.meshgrid(x, y) scale = 1.5 / psf_sigma pixpsf = render_airy((scale, center), x, y) psf = (scale, center) eval_psf = render_airy plt.clf() plt.subplot(2, 1, 1) plt.plot(x, pixpsf[center, :], 'b-') plt.plot(x, pixpsf[:, center], 'r-') plt.subplot(2, 1, 2) plt.plot(x, np.maximum(1e-16, pixpsf[center, :]), 'b-') plt.plot(x, np.maximum(1e-16, pixpsf[:, center]), 'r-') plt.yscale('log') ps.savefig() plt.clf() plt.imshow(pixpsf, interpolation='nearest', origin='lower') ps.savefig() plt.clf() plt.imshow(np.log10(np.maximum(1e-16, pixpsf)), interpolation='nearest', origin='lower') plt.colorbar() plt.title('log PSF') ps.savefig() # psf #psf = scipy.stats.norm(loc=center + 0.5, scale=psf_sigma) # plt.clf() # plt.imshow(Pcdf, interpolation='nearest', origin='lower') # ps.savefig() # #Pcdf = psf.cdf(xx) * psf.cdf(yy) # #pixpsf = integrate_gaussian(psf, xx, yy) # # padpsf = np.zeros((S*2-1, S*2-1)) # ph,pw = pixpsf.shape # padpsf[S/2:S/2+ph, S/2:S/2+pw] = pixpsf # Fpsf = np.fft.rfft2(padpsf) # # padh,padw = padpsf.shape # v = np.fft.rfftfreq(padw) # w = np.fft.fftfreq(padh) # fmax = max(max(np.abs(v)), max(np.abs(w))) # cut = fmax / 2. * 1.000001 # #print('Frequence cut:', cut) # Ffiltpsf = Fpsf.copy() # #print('Ffiltpsf', Ffiltpsf.shape) # #print((np.abs(w) < cut).shape) # #print((np.abs(v) < cut).shape) # Ffiltpsf[np.abs(w) > cut, :] = 0. # Ffiltpsf[:, np.abs(v) > cut] = 0. # #print('pad v', v) # #print('pad w', w) # # filtpsf = np.fft.irfft2(Ffiltpsf, s=(padh,padw)) # # print('filtered PSF real', np.max(np.abs(filtpsf.real))) # print('filtered PSF imag', np.max(np.abs(filtpsf.imag))) # # plt.clf() # plt.subplot(2,3,1) # dimshow(Fpsf.real) # plt.colorbar() # plt.title('Padded PSF real') # plt.subplot(2,3,4) # dimshow(Fpsf.imag) # plt.colorbar() # plt.title('Padded PSF imag') # # plt.subplot(2,3,2) # dimshow(Ffiltpsf.real) # plt.colorbar() # plt.title('Filt PSF real') # plt.subplot(2,3,5) # dimshow(Ffiltpsf.imag) # plt.colorbar() # plt.title('Filt PSF imag') # # plt.subplot(2,3,3) # dimshow(filtpsf.real) # plt.title('PSF real') # plt.colorbar() # # plt.subplot(2,3,6) # dimshow(filtpsf.imag) # plt.title('PSF imag') # plt.colorbar() # # ps.savefig() # # # pixpsf = filtpsf gal_sigmas = [2, 1, 0.5, 0.25] for gal_sigma in gal_sigmas: # plt.clf() # plt.imshow(Gcdf, interpolation='nearest', origin='lower') # plt.savefig('dcdf.png') # plt.clf() # plt.imshow(np.exp(-0.5 * ((xx-center)**2 + (yy-center)**2)/2.**2), # interpolation='nearest', origin='lower') # plt.savefig('g.png') # my convolution pixscale = 1. cd = pixscale * np.eye(2) / 3600. P, FG, Gmine, v, w = galaxy_psf_convolution(gal_sigma, 0., 0., GaussianGalaxy, cd, 0., 0., pixpsf, debug=True) #print('v:', v) #print('w:', w) #print('P:', P.shape) print() print('PSF %g, Gal %g' % (psf_sigma, gal_sigma)) rmax = np.argmax(np.abs(w)) cmax = np.argmax(np.abs(v)) l2_rmax = np.sqrt(np.sum(P[rmax, :].real**2 + P[rmax, :].imag**2)) l2_cmax = np.sqrt(np.sum(P[:, cmax].real**2 + P[:, cmax].imag**2)) print('PSF L_2 in highest-frequency rows & cols:', l2_rmax, l2_cmax) l2_rmax = np.sqrt(np.sum(FG[rmax, :].real**2 + FG[rmax, :].imag**2)) l2_cmax = np.sqrt(np.sum(FG[:, cmax].real**2 + FG[:, cmax].imag**2)) print('Gal L_2 in highest-frequency rows & cols:', l2_rmax, l2_cmax) C = P * FG l2_rmax = np.sqrt(np.sum(C[rmax, :].real**2 + C[rmax, :].imag**2)) l2_cmax = np.sqrt(np.sum(C[:, cmax].real**2 + C[:, cmax].imag**2)) print('PSF*Gal L_2 in highest-frequency rows & cols:', l2_rmax, l2_cmax) print() Fpsf, Fgal = compare_subsampled(S, 1, ps, psf, pixpsf, Gmine, v, w, gal_sigma, psf_sigma, cd, get_ffts=True, eval_psf=eval_psf) plt.clf() plt.subplot(2, 4, 1) dimshow(P.real) plt.colorbar() plt.title('PSF real') plt.subplot(2, 4, 5) dimshow(P.imag) plt.colorbar() plt.title('PSF imag') plt.subplot(2, 4, 2) dimshow(FG.real) plt.colorbar() plt.title('Gal real') plt.subplot(2, 4, 6) dimshow(FG.imag) plt.colorbar() plt.title('Gal imag') plt.subplot(2, 4, 3) dimshow((P * FG).real) plt.colorbar() plt.title('P*Gal real') plt.subplot(2, 4, 7) dimshow((P * FG).imag) plt.colorbar() plt.title('P*Gal imag') plt.subplot(2, 4, 4) dimshow((Fgal).real) plt.colorbar() plt.title('pixGal real') plt.subplot(2, 4, 8) dimshow((Fgal).imag) plt.colorbar() plt.title('pixGal imag') plt.suptitle('PSF %g, Gal %g' % (psf_sigma, gal_sigma)) ps.savefig() subsample = [1, 2, 4] rms1 = [] for s in subsample: rms = compare_subsampled(S, s, ps, psf, pixpsf, Gmine, v, w, gal_sigma, psf_sigma, cd, eval_psf=eval_psf) rms1.append(rms) rms2.append(rms1) print() print('PSF sigma =', psf_sigma) print('RMSes:') for rms1, gal_sigma in zip(rms2, gal_sigmas): print('Gal sigma', gal_sigma, 'rms:', ', '.join(['%.3g' % r for r in rms1]))
def main(): # ps = PlotSequence('pro') # star_profiles(ps) # sys.exit(0) #survey_dir = '/project/projectdirs/desiproc/dr3' #survey = LegacySurveyData(survey_dir=survey_dir) survey = LegacySurveyData() ralo, rahi = 240, 245 declo, dechi = 5, 12 ps = PlotSequence('comp') bands = 'grz' ccdfn = 'ccds-forced.fits' if not os.path.exists(ccdfn): ccds = survey.get_annotated_ccds() ccds.cut((ccds.ra > ralo) * (ccds.ra < rahi) * (ccds.dec > declo) * (ccds.dec < dechi)) print(len(ccds), 'CCDs') ccds.path = np.array([ os.path.join( #'dr3', 'forced', ('%08i' % e)[:5], '%08i' % e, 'decam-%08i-%s-forced.fits' % (e, n.strip())) for e, n in zip(ccds.expnum, ccds.ccdname) ]) I, = np.nonzero([os.path.exists(fn) for fn in ccds.path]) print(len(I), 'CCDs with forced photometry') ccds.cut(I) #ccds = ccds[:500] #e,I = np.unique(ccds.expnum, return_index=True) #print(len(I), 'unique exposures') #ccds.cut(I) FF = read_forcedphot_ccds(ccds, survey) FF.writeto('forced-all-matches.fits') # - z band -- no trend w/ PS1 mag (brighter-fatter) ccds.writeto(ccdfn) ccdfn2 = 'ccds-forced-2.fits' if not os.path.exists(ccdfn2): ccds = fits_table(ccdfn) # Split into brighter/fainter halves FF = fits_table('forced-all-matches.fits') print(len(FF), 'forced measurements') FF.cut(FF.masked == False) print(len(FF), 'forced measurements not masked') ccds.brightest_mdiff = np.zeros(len(ccds)) ccds.brightest_mscatter = np.zeros(len(ccds)) ccds.bright_mdiff = np.zeros(len(ccds)) ccds.bright_mscatter = np.zeros(len(ccds)) ccds.faint_mdiff = np.zeros(len(ccds)) ccds.faint_mscatter = np.zeros(len(ccds)) for iccd in range(len(ccds)): I = np.flatnonzero(FF.iforced == iccd) if len(I) == 0: continue if len(I) < 10: continue F = FF[I] b = np.percentile(F.psmag, 10) m = np.median(F.psmag) print(len(F), 'matches for CCD', iccd, 'median mag', m, '10th pct', b) J = np.flatnonzero(F.psmag < b) diff = F.mag[J] - F.psmag[J] ccds.brightest_mdiff[iccd] = np.median(diff) ccds.brightest_mscatter[iccd] = (np.percentile(diff, 84) - np.percentile(diff, 16)) / 2. J = np.flatnonzero(F.psmag < m) diff = F.mag[J] - F.psmag[J] ccds.bright_mdiff[iccd] = np.median(diff) ccds.bright_mscatter[iccd] = (np.percentile(diff, 84) - np.percentile(diff, 16)) / 2. J = np.flatnonzero(F.psmag > m) diff = F.mag[J] - F.psmag[J] ccds.faint_mdiff[iccd] = np.median(diff) ccds.faint_mscatter[iccd] = (np.percentile(diff, 84) - np.percentile(diff, 16)) / 2. ccds.writeto(ccdfn2) ccds = fits_table(ccdfn2) plt.clf() plt.hist(ccds.nforced, bins=100) plt.title('nforced') ps.savefig() plt.clf() plt.hist(ccds.nmatched, bins=100) plt.title('nmatched') ps.savefig() #ccds.cut(ccds.nmatched >= 150) ccds.cut(ccds.nmatched >= 50) print('Cut to', len(ccds), 'with >50 matched') ccds.cut(ccds.photometric) print('Cut to', len(ccds), 'photometric') neff = 1. / ccds.psfnorm_mean**2 # Narcsec is in arcsec**2 narcsec = neff * ccds.pixscale_mean**2 # to arcsec narcsec = np.sqrt(narcsec) # Correction factor to get back to equivalent of Gaussian sigma narcsec /= (2. * np.sqrt(np.pi)) # Conversion factor to FWHM (2.35) narcsec *= 2. * np.sqrt(2. * np.log(2.)) ccds.psfsize = narcsec for band in bands: I = np.flatnonzero( (ccds.filter == band) * (ccds.photometric) * (ccds.blacklist_ok)) mlo, mhi = -0.01, 0.05 plt.clf() plt.plot(ccds.ccdzpt[I], ccds.exptime[I], 'k.', alpha=0.1) J = np.flatnonzero((ccds.filter == band) * (ccds.photometric == False)) plt.plot(ccds.ccdzpt[J], ccds.exptime[J], 'r.', alpha=0.1) plt.xlabel('Zeropoint (mag)') plt.ylabel('exptime') plt.title('DR3: EDR region, Forced phot: %s band' % band) ps.savefig() plt.clf() plt.plot(ccds.ccdzpt[I], np.clip(ccds.mdiff[I], mlo, mhi), 'k.', alpha=0.1) plt.xlabel('Zeropoint (mag)') plt.ylabel('DECaLS PSF - PS1 (mag)') plt.axhline(0, color='k', alpha=0.2) #plt.axis([0, mxsee, mlo,mhi]) plt.title('DR3: EDR region, Forced phot: %s band' % band) ps.savefig() plt.clf() plt.plot(ccds.ccdzpt[I], ccds.psfsize[I], 'k.', alpha=0.1) plt.xlabel('Zeropoint (mag)') plt.ylabel('PSF size (arcsec)') plt.title('DR3: EDR region, Forced phot: %s band' % band) ps.savefig() # plt.clf() # plt.plot(ccds.avsky[I], # np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1) # plt.xlabel('avsky') # plt.ylabel('DECaLS PSF - PS1 (mag)') # plt.axhline(0, color='k', alpha=0.2) # plt.title('DR3: EDR region, Forced phot: %s band' % band) # ps.savefig() # # plt.clf() # plt.plot(ccds.meansky[I], # np.clip(ccds.mdiff[I], mlo,mhi), 'k.', alpha=0.1) # plt.xlabel('meansky') # plt.ylabel('DECaLS PSF - PS1 (mag)') # plt.axhline(0, color='k', alpha=0.2) # plt.title('DR3: EDR region, Forced phot: %s band' % band) # ps.savefig() plt.clf() lo, hi = (-0.02, 0.05) ha = dict(bins=50, histtype='step', range=(lo, hi)) n, b, p1 = plt.hist(ccds.brightest_mdiff[I], color='r', **ha) n, b, p2 = plt.hist(ccds.bright_mdiff[I], color='g', **ha) n, b, p3 = plt.hist(ccds.faint_mdiff[I], color='b', **ha) plt.legend((p1[0], p2[0], p3[0]), ('Brightest 10%', 'Brightest 50%', 'Faintest 50%')) plt.xlabel('DECaLS PSF - PS1 (mag)') plt.ylabel('Number of CCDs') plt.title('DR3: EDR region, Forced phot: %s band' % band) plt.xlim(lo, hi) ps.savefig() for band in bands: I = np.flatnonzero(ccds.filter == band) mxsee = 4. mlo, mhi = -0.01, 0.05 plt.clf() plt.plot(np.clip(ccds.psfsize[I], 0, mxsee), np.clip(ccds.mdiff[I], mlo, mhi), 'k.', alpha=0.1) # for p in [1,2,3]: # J = np.flatnonzero(ccds.tilepass[I] == p) # if len(J): # plt.plot(np.clip(ccds.psfsize[I[J]], 0, mxsee), # np.clip(ccds.mdiff[I[J]], mlo,mhi), '.', color='rgb'[p-1], alpha=0.2) #plt.plot(ccds.seeing[I], ccds.mdiff[I], 'b.') plt.xlabel('PSF size (arcsec)') plt.ylabel('DECaLS PSF - PS1 (mag)') plt.axhline(0, color='k', alpha=0.2) plt.axis([0, mxsee, mlo, mhi]) plt.title('DR3: EDR region, Forced phot: %s band' % band) ps.savefig() # Group by exposure for band in bands: I = np.flatnonzero( (ccds.filter == band) * (ccds.photometric) * (ccds.blacklist_ok)) E, J = np.unique(ccds.expnum[I], return_index=True) print(len(E), 'unique exposures in', band) exps = ccds[I[J]] print(len(exps), 'unique exposures in', band) assert (len(np.unique(exps.expnum)) == len(exps)) exps.ddiff = np.zeros(len(exps)) exps.dsize = np.zeros(len(exps)) exps.nccds = np.zeros(len(exps), int) exps.brightest_ddiff = np.zeros(len(exps)) exps.bright_ddiff = np.zeros(len(exps)) exps.faint_ddiff = np.zeros(len(exps)) for iexp, exp in enumerate(exps): J = np.flatnonzero(ccds.expnum[I] == exp.expnum) J = I[J] print(len(J), 'CCDs in exposure', exp.expnum) exps.brightest_mdiff[iexp] = np.median(ccds.brightest_mdiff[J]) exps.bright_mdiff[iexp] = np.median(ccds.bright_mdiff[J]) exps.faint_mdiff[iexp] = np.median(ccds.faint_mdiff[J]) exps.brightest_ddiff[iexp] = ( np.percentile(ccds.brightest_mdiff[J], 84) - np.percentile(ccds.brightest_mdiff[J], 16)) / 2. exps.bright_ddiff[iexp] = ( np.percentile(ccds.bright_mdiff[J], 84) - np.percentile(ccds.bright_mdiff[J], 16)) / 2. exps.faint_ddiff[iexp] = ( np.percentile(ccds.faint_mdiff[J], 84) - np.percentile(ccds.faint_mdiff[J], 16)) / 2. exps.mdiff[iexp] = np.median(ccds.mdiff[J]) exps.ddiff[iexp] = (np.percentile(ccds.mdiff[J], 84) - np.percentile(ccds.mdiff[J], 16)) / 2. exps.psfsize[iexp] = np.median(ccds.psfsize[J]) exps.dsize[iexp] = (np.percentile(ccds.psfsize[J], 84) - np.percentile(ccds.psfsize[J], 16)) / 2. exps.nccds[iexp] = len(J) mxsee = 4. mlo, mhi = -0.01, 0.05 exps.cut(exps.nccds >= 10) plt.clf() plt.errorbar( np.clip(exps.psfsize, 0, mxsee), np.clip(exps.mdiff, mlo, mhi), yerr=exps.ddiff, #xerr=exps.dsize, fmt='.', color='k') # plt.errorbar(np.clip(exps.psfsize, 0, mxsee), # np.clip(exps.brightest_mdiff, mlo,mhi), # yerr=exps.brightest_ddiff, fmt='r.') # plt.errorbar(np.clip(exps.psfsize, 0, mxsee), # np.clip(exps.bright_mdiff, mlo,mhi), # yerr=exps.bright_ddiff, fmt='g.') # plt.errorbar(np.clip(exps.psfsize, 0, mxsee), # np.clip(exps.faint_mdiff, mlo,mhi), # yerr=exps.faint_ddiff, fmt='b.') # plt.plot(np.clip(exps.psfsize, 0, mxsee), # np.clip(exps.brightest_mdiff, mlo,mhi), 'r.') # plt.plot(np.clip(exps.psfsize, 0, mxsee), # np.clip(exps.bright_mdiff, mlo,mhi), 'g.') # plt.plot(np.clip(exps.psfsize, 0, mxsee), # np.clip(exps.faint_mdiff, mlo,mhi), 'b.') #plt.plot(ccds.seeing[I], ccds.mdiff[I], 'b.') plt.xlabel('PSF size (arcsec)') plt.ylabel('DECaLS PSF - PS1 (mag)') plt.axhline(0, color='k', alpha=0.2) plt.axis([0, mxsee, mlo, mhi]) plt.title('DR3: EDR region, Forced phot: %s band' % band) ps.savefig() plt.clf() p1 = plt.plot(np.clip(exps.psfsize, 0, mxsee), np.clip(exps.brightest_mdiff, mlo, mhi), 'r.', alpha=0.5) p2 = plt.plot(np.clip(exps.psfsize, 0, mxsee), np.clip(exps.bright_mdiff, mlo, mhi), 'g.', alpha=0.5) p3 = plt.plot(np.clip(exps.psfsize, 0, mxsee), np.clip(exps.faint_mdiff, mlo, mhi), 'b.', alpha=0.5) plt.legend((p1[0], p2[0], p3[0]), ('Brightest 10%', 'Brightest 50%', 'Faintest 50%')) plt.xlabel('PSF size (arcsec)') plt.ylabel('DECaLS PSF - PS1 (mag)') plt.axhline(0, color='k', alpha=0.2) plt.axis([0, mxsee, mlo, mhi]) plt.title('DR3: EDR region, Forced phot: %s band' % band) ps.savefig() J = np.argsort(-exps.mdiff) for j in J: print(' Photometric diff', exps.mdiff[j], 'PSF size', exps.psfsize[j], 'expnum', exps.expnum[j]) sys.exit(0)
def main(passnum, threads): global udecs global P3 global T global tilewcs global exps global bad_expids global tileid_to_depth ps = PlotSequence('covfill-p%i' % passnum) retirablefn = 'retirable-p%i.fits' % passnum depthsfn = 'all-depths-p%i.fits' % passnum if os.path.exists(retirablefn): R = fits_table(retirablefn) pcts = np.arange(0, 101) target = 22.5 req_pcts = [0, 2, 2, 5, 5, 10, 10, 100] req_depths = [0, 0, target-0.6, target-0.6, target-0.3, target-0.3, target, target] maglo, maghi = 21,23 plt.clf() for depths in R.depths: plt.plot(pcts, np.clip(depths, maglo, maghi), 'b-', alpha=0.1) plt.plot(req_pcts, np.clip(req_depths, maglo, maghi), 'k-', lw=2, alpha=0.5) plt.ylim(maglo, maghi) plt.xlim(0, 100) plt.xlabel('Coverage fraction') plt.ylabel('Existing depth') plt.suptitle('MzLS: retirable pass-%i tiles: %i' % (passnum,len(R))) ps.savefig() # Where are they on the sky? T = fits_table('obstatus/mosaic-tiles_obstatus.fits') T.cut(T.in_desi == 1) T.cut(T.get('pass') <= 3) tileid_to_index = np.zeros(T.tileid.max()+1, int) tileid_to_index[T.tileid] = np.arange(len(T)) R.ra = T.ra [tileid_to_index[R.tileid]] R.dec = T.dec[tileid_to_index[R.tileid]] plt.clf() plt.plot(T.ra, T.dec, 'k.', alpha=0.02) I = (T.z_done == 1) plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.1) plt.plot(R.ra, R.dec, 'b.') ax = [310,80,30,85] #xl,xh = plt.xlim() #plt.xlim(xh,xl) plt.xlabel('RA (deg)') plt.ylabel('Dec (deg)') plt.title('MzLS: retirable pass-%i tiles' % passnum) plt.axis(ax) ps.savefig() for p in [1,2,3]: plt.clf() plt.plot(T.ra, T.dec, 'k.', alpha=0.02) #plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.1) I = np.flatnonzero((T.get('pass') == p) * (T.z_done == 1) * (T.z_depth > 1) * (T.z_depth < 30)) plt.scatter(T.ra[I], T.dec[I], c=T.z_depth[I], vmin=20, vmax=23, s=4) I = np.flatnonzero((T.get('pass') == p) * (T.z_done == 1) * (T.z_depth == 30)) plt.plot(T.ra[I], T.dec[I], 'k.', alpha=0.5) plt.colorbar() plt.title('MzLS: Finished tiles in pass %i' % p) plt.axis(ax) ps.savefig() sys.exit(0) # NERSC: export LEGACY_SURVEY_DIR=/global/cscratch1/sd/dstn/dr4plus # (dstn laptop: export LEGACY_SURVEY_DIR=~/legacypipe-dir-mzls/) survey = LegacySurveyData() ccds = survey.get_annotated_ccds() print('Annotated CCDs:', len(ccds)) ccds.cut(ccds.camera == 'mosaic') print(len(ccds), 'Mosaic') print('Unique exposures:', len(np.unique(ccds.expnum))) ccds.cut(ccds.exptime > 60) print('Exptime > 60 sec:', len(ccds)) nccds = Counter(ccds.expnum) for k,v in nccds.most_common(): if v <= 4: break print('Expnum', k, 'appears', v, 'times') print('Tile pass numbers:', Counter(ccds.tilepass).most_common()) # Fix parsing of OBJECT field to tileid... from obsbot import get_tile_id_from_name tileids = [] for o in ccds.object: tid = get_tile_id_from_name(o.strip()) if tid is None: tid = 0 tileids.append(tid) tileids = np.array(tileids) print(len(np.unique(tileids)), 'unique tile ids in annotated file, from OBJECT') print(len(np.unique(ccds.tileid)), 'unique tile ids in ann file from TILEID') D = np.flatnonzero(tileids != ccds.tileid) print(len(D), 'different tileids') print('From OBJECT:', tileids[D]) print('From TILEID:', ccds.tileid[D]) ccds.tileid = tileids T = fits_table('obstatus/mosaic-tiles_obstatus.fits') f = open('obstatus/bad_expid.txt') bad_expids = [] for line in f: line = line.strip() if len(line) == 0: continue if line[0] == '#': continue words = line.split() try: expnum = int(words[0]) except: print('Skipping line:', line) continue bad_expids.append(expnum) print('Read', len(bad_expids), 'bad exposure numbers') # Update ccds.tilepass from ccds.tileid tileidtopass = dict(zip(T.tileid, T.get('pass'))) tileidtoebv = dict(zip(T.tileid, T.ebv_med)) ccds.tilepass = np.array([tileidtopass.get(tileid, 0) for tileid in ccds.tileid]) ccds.tileebv = np.array([tileidtoebv.get(tileid, 0) for tileid in ccds.tileid]) print('Tile pass numbers after update:', Counter(ccds.tilepass).most_common()) e,I = np.unique(ccds.expnum, return_index=True) exps = ccds[I] #print('Object names,exptimes for tilepass==0:', zip(exps.object[exps.tilepass == 0], exps.exptime[exps.tilepass == 0])) # Average the depth per exposure for j,expnum in enumerate(exps.expnum): I = np.flatnonzero(ccds.expnum == expnum) if len(I) != 4: print('Exposure', expnum, 'has', len(I), 'CCD entries') continue # Don't include zeros in computing average depths! Igood = I[(ccds.galdepth[I] > 0) * (ccds.ccdzpt[I] < 30)] if len(Igood) > 0: exps.galdepth[j] = np.mean(ccds.galdepth[Igood]) else: exps.galdepth[j] = 0. # CCDs-table-based mapping from tileid to depth. I = np.flatnonzero((exps.tilepass > 0) * (exps.galdepth > 0) * (exps.tileid > 0)) tileid_to_depth = dict(zip(exps.tileid[I], exps.galdepth[I])) T.cut(T.in_desi == 1) T.cut(T.get('pass') <= 3) # The tiles we'll examine P3 = T[T.get('pass') == passnum] print(len(P3), 'pass', passnum, 'and in DESI') todo = P3[P3.z_done == 0] print(len(todo), 'pass', passnum, 'tiles to do (Z_DONE=0)') # Tiles with measured depths T.cut((T.z_depth > 15) * (T.z_depth < 30)) print(len(T), 'tiles with measured depths') # Passes other than 3... they ~ only barely overlap, so don't # contribute significant depth. #T.cut(T.get('pass') < 3) udecs = np.unique(P3.dec) print(len(udecs), 'unique Dec values in pass', passnum) # Grab an arbitrary weight-map image and use that as a proxy! wtfn = 'k4m_170501_112501_oow_zd_v1.fits.fz' F = fitsio.FITS(wtfn) tilewcs = [] # Read the WCS headers for each chip. # They make the CRVAL be the boresight for all 4 chips... perfect! # (because this means we can just set CRVAL = RA,Dec to shift the WCSes) for i in range(1, len(F)): hdr = F[i].read_header() wcs = wcs_pv2sip_hdr(hdr) tilewcs.append(wcs) mp = multiproc(threads) #args = [(i,t,udecs,P3,T,tilewcs,exps,bad_expids,tileid_to_depth) args = [(i,t) for i,t in enumerate(todo)] thedepths = mp.map(one_tile, args) alldepths = [] retirable = [] for arg,depths in zip(args, thedepths): if depths is None: continue itile,tile = arg[:2] target = 22.5 req_pcts = [0, 2, 2, 5, 5, 10, 10, 100] req_depths = [0, 0, target-0.6, target-0.6, target-0.3, target-0.3, target, target] print(' Depths at 2, 5, and 10th percentile vs target:', '%.2f' % (depths[2] - (target - 0.6)), '%.2f' % (depths[5] - (target - 0.3)), '%.2f' % (depths[10] - target)) alldepths.append((tile.tileid, depths)) if not ((depths[2] > target - 0.6) and (depths[5] > target - 0.3) and (depths[10] > target)): continue retirable.append((tile.tileid, depths)) #if len(retirable) == 10: # break # if ps.ploti >= 100: # continue # # maglo, maghi = 21,23 # # plt.clf() # #plt.subplot(1,2,1) # plt.subplot2grid((2,2), (0,0)) # plt.imshow(depth, interpolation='nearest', origin='lower', # vmin=maglo, vmax=maghi) # plt.colorbar(ticks=[np.arange(maglo, maghi+0.01, 0.5)]) # plt.xticks([]); plt.yticks([]) # #plt.subplot(1,2,2) # plt.subplot2grid((2,2), (1,0)) # plt.imshow(nexp, interpolation='nearest', origin='lower', # vmin=0, vmax=4) # plt.colorbar(ticks=[0,1,2,3,4]) # plt.xticks([]); plt.yticks([]) # # ax = plt.subplot2grid((2,2), (0,1), rowspan=2) # plt.plot(req_pcts, np.clip(req_depths, maglo, maghi), 'k-', # lw=2, alpha=0.5) # plt.plot(pcts, np.clip(depths, maglo, maghi), 'b-') # ax.yaxis.tick_right() # plt.ylim(maglo, maghi) # plt.xlim(0, 100) # plt.xlabel('Coverage fraction') # plt.ylabel('Existing depth') # plt.suptitle('Tile %i' % tile.tileid) # ps.savefig() #if ps.ploti == 100: # break # print('Tiles that could be retired:') # print('# Tileid 0th-percentile-extcorr-depth 1st-pctile 2nd-pctile ...') # for tileid, depths in retirable: # print(tileid, ' '.join(['%.3f' % d for d in depths])) R = fits_table() R.tileid = np.array ([t for t,d in alldepths]) R.depths = np.vstack([d for t,d in alldepths]) R.writeto(depthsfn) if len(retirable): R = fits_table() R.tileid = np.array([t for t,d in retirable]) R.depths = np.vstack([d for t,d in retirable]) R.writeto(retirablefn) else: print('No tiles in pass', passnum, 'are retirable')
def stage0(**kwargs): ps = PlotSequence('cfht') decals = CfhtDecals() B = decals.get_bricks() print('Bricks:') B.about() ra, dec = 190.0, 11.0 #bands = 'ugri' bands = 'gri' B.cut(np.argsort(degrees_between(ra, dec, B.ra, B.dec))) print('Nearest bricks:', B.ra[:5], B.dec[:5], B.brickid[:5]) brick = B[0] pixscale = 0.186 #W,H = 1024,1024 #W,H = 2048,2048 #W,H = 3600,3600 W, H = 4800, 4800 targetwcs = wcs_for_brick(brick, pixscale=pixscale, W=W, H=H) ccdfn = 'cfht-ccds.fits' if os.path.exists(ccdfn): T = fits_table(ccdfn) else: T = get_ccd_list() T.writeto(ccdfn) print(len(T), 'CCDs') T.cut(ccds_touching_wcs(targetwcs, T)) print(len(T), 'CCDs touching brick') T.cut(np.array([b in bands for b in T.filter])) print(len(T), 'in bands', bands) ims = [] for t in T: im = CfhtImage(t) # magzp = hdr['PHOT_C'] + 2.5 * np.log10(hdr['EXPTIME']) # fwhm = t.seeing / (pixscale * 3600) # print '-> FWHM', fwhm, 'pix' im.seeing = t.seeing im.pixscale = t.pixscale print('seeing', t.seeing) print('pixscale', im.pixscale * 3600, 'arcsec/pix') im.run_calibs(t.ra, t.dec, im.pixscale, W=t.width, H=t.height) ims.append(im) # Read images, clip to ROI targetrd = np.array([ targetwcs.pixelxy2radec(x, y) for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)] ]) keepims = [] tims = [] for im in ims: print() print('Reading expnum', im.expnum, 'name', im.extname, 'band', im.band, 'exptime', im.exptime) band = im.band wcs = im.read_wcs() imh, imw = wcs.imageh, wcs.imagew imgpoly = [(1, 1), (1, imh), (imw, imh), (imw, 1)] ok, tx, ty = wcs.radec2pixelxy(targetrd[:-1, 0], targetrd[:-1, 1]) tpoly = zip(tx, ty) clip = clip_polygon(imgpoly, tpoly) clip = np.array(clip) #print 'Clip', clip if len(clip) == 0: continue x0, y0 = np.floor(clip.min(axis=0)).astype(int) x1, y1 = np.ceil(clip.max(axis=0)).astype(int) slc = slice(y0, y1 + 1), slice(x0, x1 + 1) ## FIXME -- it seems I got lucky and the cross product is ## negative == clockwise, as required by clip_polygon. One ## could check this and reverse the polygon vertex order. # dx0,dy0 = tx[1]-tx[0], ty[1]-ty[0] # dx1,dy1 = tx[2]-tx[1], ty[2]-ty[1] # cross = dx0*dy1 - dx1*dy0 # print 'Cross:', cross print('Image slice: x [%i,%i], y [%i,%i]' % (x0, x1, y0, y1)) print('Reading image from', im.imgfn, 'HDU', im.hdu) img, imghdr = im.read_image(header=True, slice=slc) goodpix = (img != 0) print('Number of pixels == 0:', np.sum(img == 0)) print('Number of pixels != 0:', np.sum(goodpix)) if np.sum(goodpix) == 0: continue # print 'Image shape', img.shape print('Image range', img.min(), img.max()) print('Goodpix image range:', (img[goodpix]).min(), (img[goodpix]).max()) if img[goodpix].min() == img[goodpix].max(): print('No dynamic range in image') continue # print 'Reading invvar from', im.wtfn, 'HDU', im.hdu # invvar = im.read_invvar(slice=slc) # # print 'Invvar shape', invvar.shape # # print 'Invvar range:', invvar.min(), invvar.max() # invvar[goodpix == 0] = 0. # if np.all(invvar == 0.): # print 'Skipping zero-invvar image' # continue # assert(np.all(np.isfinite(img))) # assert(np.all(np.isfinite(invvar))) # assert(not(np.all(invvar == 0.))) # # Estimate per-pixel noise via Blanton's 5-pixel MAD # slice1 = (slice(0,-5,10),slice(0,-5,10)) # slice2 = (slice(5,None,10),slice(5,None,10)) # # print 'sliced shapes:', img[slice1].shape, img[slice2].shape # # print 'good shape:', (goodpix[slice1] * goodpix[slice2]).shape # # print 'good values:', np.unique(goodpix[slice1] * goodpix[slice2]) # # print 'sliced[good] shapes:', (img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].shape # mad = np.median(np.abs(img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].ravel()) # sig1 = 1.4826 * mad / np.sqrt(2.) # print 'MAD sig1:', sig1 # # invvar was 1 or 0 # invvar *= (1./(sig1**2)) # medsky = np.median(img[goodpix]) # Read full image for sig1 and sky estimate fullimg = im.read_image() fullgood = (fullimg != 0) # Estimate per-pixel noise via Blanton's 5-pixel MAD slice1 = (slice(0, -5, 10), slice(0, -5, 10)) slice2 = (slice(5, None, 10), slice(5, None, 10)) mad = np.median( np.abs(fullimg[slice1] - fullimg[slice2])[fullgood[slice1] * fullgood[slice2]].ravel()) sig1 = 1.4826 * mad / np.sqrt(2.) print('MAD sig1:', sig1) medsky = np.median(fullimg[fullgood]) invvar = np.zeros_like(img) invvar[goodpix] = 1. / sig1**2 # Median-smooth sky subtraction plt.clf() dimshow(np.round((img - medsky) / sig1), vmin=-3, vmax=5) plt.title('Scalar median: %s' % im.name) ps.savefig() # medsky = np.zeros_like(img) # # astrometry.util.util # median_smooth(img, np.logical_not(goodpix), 256, medsky) fullmed = np.zeros_like(fullimg) median_smooth(fullimg - medsky, np.logical_not(fullgood), 256, fullmed) fullmed += medsky medimg = fullmed[slc] plt.clf() dimshow(np.round((img - medimg) / sig1), vmin=-3, vmax=5) plt.title('Median filtered: %s' % im.name) ps.savefig() #print 'Subtracting median:', medsky #img -= medsky img -= medimg primhdr = im.read_image_primary_header() magzp = decals.get_zeropoint_for(im) print('magzp', magzp) zpscale = NanoMaggies.zeropointToScale(magzp) print('zpscale', zpscale) # Scale images to Nanomaggies img /= zpscale sig1 /= zpscale invvar *= zpscale**2 orig_zpscale = zpscale zpscale = 1. assert (np.sum(invvar > 0) > 0) print('After scaling:') print('sig1', sig1) print('invvar range', invvar.min(), invvar.max()) print('image range', img.min(), img.max()) assert (np.all(np.isfinite(img))) assert (np.all(np.isfinite(invvar))) assert (np.isfinite(sig1)) plt.clf() lo, hi = -5 * sig1, 10 * sig1 n, b, p = plt.hist(img[goodpix].ravel(), 100, range=(lo, hi), histtype='step', color='k') xx = np.linspace(lo, hi, 200) plt.plot(xx, max(n) * np.exp(-xx**2 / (2. * sig1**2)), 'r-') plt.xlim(lo, hi) plt.title('Pixel histogram: %s' % im.name) ps.savefig() twcs = ConstantFitsWcs(wcs) if x0 or y0: twcs.setX0Y0(x0, y0) info = im.get_image_info() fullh, fullw = info['dims'] # read fit PsfEx model psfex = PsfEx.fromFits(im.psffitfn) print('Read', psfex) # HACK -- highly approximate PSF here! #psf_fwhm = imghdr['FWHM'] #psf_fwhm = im.seeing psf_fwhm = im.seeing / (im.pixscale * 3600) print('PSF FWHM', psf_fwhm, 'pixels') psf_sigma = psf_fwhm / 2.35 psf = NCircularGaussianPSF([psf_sigma], [1.]) print('img type', img.dtype) tim = Image(img, invvar=invvar, wcs=twcs, psf=psf, photocal=LinearPhotoCal(zpscale, band=band), sky=ConstantSky(0.), name=im.name + ' ' + band) tim.zr = [-3. * sig1, 10. * sig1] tim.sig1 = sig1 tim.band = band tim.psf_fwhm = psf_fwhm tim.psf_sigma = psf_sigma tim.sip_wcs = wcs tim.x0, tim.y0 = int(x0), int(y0) tim.psfex = psfex tim.imobj = im mn, mx = tim.zr tim.ima = dict(interpolation='nearest', origin='lower', cmap='gray', vmin=mn, vmax=mx) tims.append(tim) keepims.append(im) ims = keepims print('Computing resampling...') # save resampling params for tim in tims: wcs = tim.sip_wcs subh, subw = tim.shape subwcs = wcs.get_subimage(tim.x0, tim.y0, subw, subh) tim.subwcs = subwcs try: Yo, Xo, Yi, Xi, rims = resample_with_wcs(targetwcs, subwcs, [], 2) except OverlapError: print('No overlap') continue if len(Yo) == 0: continue tim.resamp = (Yo, Xo, Yi, Xi) print('Creating coadds...') # Produce per-band coadds, for plots coimgs = [] cons = [] for ib, band in enumerate(bands): coimg = np.zeros((H, W), np.float32) con = np.zeros((H, W), np.uint8) for tim in tims: if tim.band != band: continue (Yo, Xo, Yi, Xi) = tim.resamp if len(Yo) == 0: continue nn = (tim.getInvvar()[Yi, Xi] > 0) coimg[Yo, Xo] += tim.getImage()[Yi, Xi] * nn con[Yo, Xo] += nn # print # print 'tim', tim.name # print 'number of resampled pix:', len(Yo) # reim = np.zeros_like(coimg) # ren = np.zeros_like(coimg) # reim[Yo,Xo] = tim.getImage()[Yi,Xi] * nn # ren[Yo,Xo] = nn # print 'number of resampled pix with positive invvar:', ren.sum() # plt.clf() # plt.subplot(2,2,1) # mn,mx = [np.percentile(reim[ren>0], p) for p in [25,95]] # print 'Percentiles:', mn,mx # dimshow(reim, vmin=mn, vmax=mx) # plt.colorbar() # plt.subplot(2,2,2) # dimshow(con) # plt.colorbar() # plt.subplot(2,2,3) # dimshow(reim, vmin=tim.zr[0], vmax=tim.zr[1]) # plt.colorbar() # plt.subplot(2,2,4) # plt.hist(reim.ravel(), 100, histtype='step', color='b') # plt.hist(tim.getImage().ravel(), 100, histtype='step', color='r') # plt.suptitle('%s: %s' % (band, tim.name)) # ps.savefig() coimg /= np.maximum(con, 1) coimgs.append(coimg) cons.append(con) plt.clf() dimshow(get_rgb(coimgs, bands)) ps.savefig() plt.clf() for i, b in enumerate(bands): plt.subplot(2, 2, i + 1) dimshow(cons[i], ticks=False) plt.title('%s band' % b) plt.colorbar() plt.suptitle('Number of exposures') ps.savefig() print('Grabbing SDSS sources...') bandlist = [b for b in bands] cat, T = get_sdss_sources(bandlist, targetwcs) # record coordinates in target brick image ok, T.tx, T.ty = targetwcs.radec2pixelxy(T.ra, T.dec) T.tx -= 1 T.ty -= 1 T.itx = np.clip(np.round(T.tx).astype(int), 0, W - 1) T.ity = np.clip(np.round(T.ty).astype(int), 0, H - 1) plt.clf() dimshow(get_rgb(coimgs, bands)) ax = plt.axis() plt.plot(T.tx, T.ty, 'o', mec=green, mfc='none', ms=10, mew=1.5) plt.axis(ax) plt.title('SDSS sources') ps.savefig() print('Detmaps...') # Render the detection maps detmaps = dict([(b, np.zeros((H, W), np.float32)) for b in bands]) detivs = dict([(b, np.zeros((H, W), np.float32)) for b in bands]) for tim in tims: iv = tim.getInvvar() psfnorm = 1. / (2. * np.sqrt(np.pi) * tim.psf_sigma) detim = tim.getImage().copy() detim[iv == 0] = 0. detim = gaussian_filter(detim, tim.psf_sigma) / psfnorm**2 detsig1 = tim.sig1 / psfnorm subh, subw = tim.shape detiv = np.zeros((subh, subw), np.float32) + (1. / detsig1**2) detiv[iv == 0] = 0. (Yo, Xo, Yi, Xi) = tim.resamp detmaps[tim.band][Yo, Xo] += detiv[Yi, Xi] * detim[Yi, Xi] detivs[tim.band][Yo, Xo] += detiv[Yi, Xi] rtn = dict() for k in [ 'T', 'coimgs', 'cons', 'detmaps', 'detivs', 'targetrd', 'pixscale', 'targetwcs', 'W', 'H', 'bands', 'tims', 'ps', 'brick', 'cat' ]: rtn[k] = locals()[k] return rtn
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None, use_ceres=True, ceres_block=8, save_fits=False, get_models=False, ps=None, psf_broadening=None, pixelized_psf=False, get_masks=None, move_crpix=False, modelsky_dir=None): ''' Given a list of tractor sources *cat* and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id) runs forced photometry, returning a FITS table the same length as *cat*. *get_masks*: the WCS to resample mask bits into. ''' from tractor import NanoMaggies, PointSource, Tractor, ExpGalaxy, DevGalaxy, FixedCompositeGalaxy if not pixelized_psf and psf_broadening is None: # PSF broadening in post-reactivation data, by band. # Newer version from Aaron's email to decam-chatter, 2018-06-14. broadening = { 1: 1.0405, 2: 1.0346, 3: None, 4: None } psf_broadening = broadening[band] if False: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('wise-forced-w%i' % band) plots = (ps is not None) if plots: import pylab as plt wantims = (plots or save_fits or get_models) wanyband = 'w' if get_models: models = {} wband = 'w%i' % band fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix', 'pronexp'] Nsrcs = len(cat) phot = fits_table() # Filled in based on unique tile overlap phot.wise_coadd_id = np.array([' '] * Nsrcs) phot.set(wband + '_psfdepth', np.zeros(len(phot), np.float32)) ra = np.array([src.getPosition().ra for src in cat]) dec = np.array([src.getPosition().dec for src in cat]) nexp = np.zeros(Nsrcs, np.int16) mjd = np.zeros(Nsrcs, np.float64) central_flux = np.zeros(Nsrcs, np.float32) fitstats = {} tims = [] if get_masks: mh,mw = get_masks.shape maskmap = np.zeros((mh,mw), np.uint32) for tile in tiles: print('Reading WISE tile', tile.coadd_id, 'band', band) tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradecbox) if tim is None: print('Actually, no overlap with tile', tile.coadd_id) continue if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data' % tag) ps.savefig() plt.clf() plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), range=(-5,10), bins=100) plt.xlabel('Per-pixel intensity (Sigma)') plt.title(tag) ps.savefig() if move_crpix and band in [1, 2]: realwcs = tim.wcs.wcs x,y = realwcs.crpix tile_crpix = tile.get('crpix_w%i' % band) dx = tile_crpix[0] - 1024.5 dy = tile_crpix[1] - 1024.5 realwcs.set_crpix(x+dx, y+dy) #print('CRPIX', x,y, 'shift by', dx,dy, 'to', realwcs.crpix) if modelsky_dir and band in [1, 2]: fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band)) if not os.path.exists(fn): raise RuntimeError('WARNING: does not exist:', fn) x0,x1,y0,y1 = tim.roi bg = fitsio.FITS(fn)[2][y0:y1, x0:x1] #print('Read background map:', bg.shape, bg.dtype, 'vs image', tim.shape) if plots: plt.clf() plt.subplot(1,2,1) plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) plt.subplot(1,2,2) plt.imshow(bg, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) tag = '%s W%i' % (tile.coadd_id, band) plt.suptitle(tag) ps.savefig() plt.clf() ha = dict(range=(-5,10), bins=100, histtype='step') plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), color='b', label='Original', **ha) plt.hist(((tim.getImage()-bg) * tim.inverr)[tim.inverr > 0].ravel(), color='g', label='Minus Background', **ha) plt.axvline(0, color='k', alpha=0.5) plt.xlabel('Per-pixel intensity (Sigma)') plt.legend() plt.title(tag + ': background') ps.savefig() # Actually subtract the background! tim.data -= bg # Floor the per-pixel variances if band in [1,2]: # in Vega nanomaggies per pixel floor_sigma = {1: 0.5, 2: 2.0} with np.errstate(divide='ignore'): new_ie = 1. / np.hypot(1./tim.inverr, floor_sigma[band]) new_ie[tim.inverr == 0] = 0. if plots: plt.clf() plt.plot((1. / tim.inverr[tim.inverr>0]).ravel(), (1./new_ie[tim.inverr>0]).ravel(), 'b.') plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band)) plt.xlabel('original') plt.ylabel('floored') ps.savefig() tim.inverr = new_ie # Read mask file? if get_masks: from astrometry.util.resample import resample_with_wcs, OverlapError # unwise_dir can be a colon-separated list of paths tilemask = None for d in tile.unwise_dir.split(':'): fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id, 'unwise-%s-msk.fits.gz' % tile.coadd_id) if os.path.exists(fn): print('Reading unWISE mask file', fn) x0,x1,y0,y1 = tim.roi tilemask = fitsio.FITS(fn)[0][y0:y1,x0:x1] break if tilemask is None: print('unWISE mask file for tile', tile.coadd_id, 'does not exist') else: try: tanwcs = tim.wcs.wcs assert(tanwcs.shape == tilemask.shape) Yo,Xo,Yi,Xi,_ = resample_with_wcs(get_masks, tanwcs, intType=np.int16) # Only deal with mask pixels that are set. I, = np.nonzero(tilemask[Yi,Xi] > 0) # Trim to unique area for this tile rr,dd = get_masks.pixelxy2radec(Yo[I]+1, Xo[I]+1) good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I = I[good] maskmap[Yo[I],Xo[I]] = tilemask[Yi[I], Xi[I]] except OverlapError: # Shouldn't happen by this point print('No overlap between WISE tile', tile.coadd_id, 'and brick') # The tiles have some overlap, so zero out pixels outside the # tile's unique area. th,tw = tim.shape xx,yy = np.meshgrid(np.arange(tw), np.arange(th)) rr,dd = tim.wcs.wcs.pixelxy2radec(xx+1, yy+1) unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) #print(np.sum(unique), 'of', (th*tw), 'pixels in this tile are unique') tim.inverr[unique == False] = 0. del xx,yy,rr,dd,unique if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage() * (tim.inverr > 0), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data (unique)' % tag) ps.savefig() if pixelized_psf: import unwise_psf if (band == 1) or (band == 2): # we only have updated PSFs for W1 and W2 psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id, modelname='neo4_unwisecat') else: psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id) if band == 4: # oversample (the unwise_psf models are at native W4 5.5"/pix, # while the unWISE coadds are made at 2.75"/pix. ph,pw = psfimg.shape subpsf = np.zeros((ph*2-1, pw*2-1), np.float32) from astrometry.util.util import lanczos3_interpolate xx,yy = np.meshgrid(np.arange(0., pw-0.51, 0.5, dtype=np.float32), np.arange(0., ph-0.51, 0.5, dtype=np.float32)) xx = xx.ravel() yy = yy.ravel() ix = xx.astype(np.int32) iy = yy.astype(np.int32) dx = (xx - ix).astype(np.float32) dy = (yy - iy).astype(np.float32) psfimg = psfimg.astype(np.float32) rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg]) if plots: plt.clf() plt.imshow(psfimg, interpolation='nearest', origin='lower') plt.title('Original PSF model') ps.savefig() plt.clf() plt.imshow(subpsf, interpolation='nearest', origin='lower') plt.title('Subsampled PSF model') ps.savefig() psfimg = subpsf del xx, yy, ix, iy, dx, dy from tractor.psf import PixelizedPSF psfimg /= psfimg.sum() fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0} psfimg *= fluxrescales[band] tim.psf = PixelizedPSF(psfimg) if psf_broadening is not None and not pixelized_psf: # psf_broadening is a factor by which the PSF FWHMs # should be scaled; the PSF is a little wider # post-reactivation. psf = tim.getPsf() from tractor import GaussianMixturePSF if isinstance(psf, GaussianMixturePSF): # print('Broadening PSF: from', psf) p0 = psf.getParams() pnames = psf.getParamNames() p1 = [p * psf_broadening**2 if 'var' in name else p for (p, name) in zip(p0, pnames)] psf.setParams(p1) print('Broadened PSF:', psf) else: print('WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf)) wcs = tim.wcs.wcs ok,x,y = wcs.radec2pixelxy(ra, dec) x = np.round(x - 1.).astype(int) y = np.round(y - 1.).astype(int) good = (x >= 0) * (x < tw) * (y >= 0) * (y < th) # Which sources are in this brick's unique area? usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I, = np.nonzero(good * usrc) nexp[I] = tim.nuims[y[I], x[I]] if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): mjd[I] = (tim.mjdmin + tim.mjdmax) / 2. phot.wise_coadd_id[I] = tile.coadd_id central_flux[I] = tim.getImage()[y[I], x[I]] del x,y,good,usrc # PSF norm for depth psf = tim.getPsf() h,w = tim.shape patch = psf.getPointSourcePatch(h//2, w//2).patch psfnorm = np.sqrt(np.sum(patch**2)) # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags. psfdepth = 1. / (tim.sig1 / psfnorm)**2 phot.get(wband + '_psfdepth')[I] = psfdepth tim.tile = tile tims.append(tim) if plots: plt.clf() mn,mx = 0.1, 20000 plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100, range=(np.log10(mn), np.log10(mx))) logt = np.arange(0, 5) plt.xticks(logt, ['%i' % i for i in 10.**logt]) plt.title('Central fluxes (W%i)' % band) plt.axvline(np.log10(20000), color='k') plt.axvline(np.log10(1000), color='k') ps.savefig() # Eddie's non-secret recipe: #- central pixel <= 1000: 19x19 pix box size #- central pixel in 1000 - 20000: 59x59 box size #- central pixel > 20000 or saturated: 149x149 box size #- object near "bright star": 299x299 box size nbig = nmedium = nsmall = 0 for src,cflux in zip(cat, central_flux): if cflux > 20000: R = 100 nbig += 1 elif cflux > 1000: R = 30 nmedium += 1 else: R = 15 nsmall += 1 if isinstance(src, PointSource): src.fixedRadius = R else: ### FIXME -- sizes for galaxies..... can we set PSF size separately? galrad = 0 # RexGalaxy is a subclass of ExpGalaxy if isinstance(src, (ExpGalaxy, DevGalaxy)): galrad = src.shape.re elif isinstance(src, FixedCompositeGalaxy): galrad = max(src.shapeExp.re, src.shapeDev.re) pixscale = 2.75 src.halfsize = int(np.hypot(R, galrad * 5 / pixscale)) #print('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small') minsb = 0. fitsky = False tractor = Tractor(tims, cat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(wanyband) kwa = dict(fitstat_extras=[('pronexp', [tim.nims for tim in tims])]) t0 = Time() R = tractor.optimize_forced_photometry( minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True, variance=True, shared_params=False, wantims=wantims, **kwa) print('unWISE forced photometry took', Time() - t0) if use_ceres: term = R.ceres_status['termination'] # Running out of memory can cause failure to converge # and term status = 2. # Fail completely in this case. if term != 0: print('Ceres termination status:', term) raise RuntimeError( 'Ceres terminated with status %i' % term) if wantims: ims1 = R.ims1 flux_invvars = R.IV if R.fitstats is not None: for k in fskeys: x = getattr(R.fitstats, k) fitstats[k] = np.array(x).astype(np.float32) if save_fits: for i,tim in enumerate(tims): tile = tim.tile (dat, mod, ie, chi, roi) = ims1[i] wcshdr = fitsio.FITSHDR() tim.wcs.wcs.add_to_header(wcshdr) tag = 'fit-%s-w%i' % (tile.coadd_id, band) fitsio.write('%s-data.fits' % tag, dat, clobber=True, header=wcshdr) fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr) fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr) if plots: # Create models for just the brightest sources bright_cat = [src for src in cat if src.getBrightness().getBand(wanyband) > 1000] print('Bright soures:', len(bright_cat)) btr = Tractor(tims, bright_cat) for tim in tims: mod = btr.getModelImage(tim) tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) sig1 = tim.sig1 plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: bright-star models' % tag) ps.savefig() if get_models: for i,tim in enumerate(tims): tile = tim.tile (dat, mod, ie, chi, roi) = ims1[i] models[(tile.coadd_id, band)] = (mod, dat, ie, tim.roi, tim.wcs.wcs) if plots: for i,tim in enumerate(tims): tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) (dat, mod, ie, chi, roi) = ims1[i] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() nm = np.array([src.getBrightness().getBand(wanyband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their default # (1-sigma or whatever) initial fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. phot.set(wband + '_nanomaggies', nm.astype(np.float32)) phot.set(wband + '_nanomaggies_ivar', nm_ivar.astype(np.float32)) dnm = np.zeros(len(nm_ivar), np.float32) okiv = (nm_ivar > 0) dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32) okflux = (nm > 0) mag = np.zeros(len(nm), np.float32) mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux]) ).astype(np.float32) dmag = np.zeros(len(nm), np.float32) ok = (okiv * okflux) dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok]) ).astype(np.float32) mag[np.logical_not(okflux)] = np.nan dmag[np.logical_not(ok)] = np.nan phot.set(wband + '_mag', mag) phot.set(wband + '_mag_err', dmag) for k in fskeys: phot.set(wband + '_' + k, fitstats[k]) phot.set(wband + '_nexp', nexp) if not np.all(mjd == 0): phot.set(wband + '_mjd', mjd) rtn = wphotduck() rtn.phot = phot rtn.models = None rtn.maskmap = None if get_models: rtn.models = models if get_masks: rtn.maskmap = maskmap return rtn
def main(): ps = PlotSequence('cov') survey = LegacySurveyData() ra, dec = 242.0, 10.2 fn = 'coverage-ccds.fits' if not os.path.exists(fn): ccds = survey.get_ccds() ccds.cut(ccds.filter == 'r') ccds.cut(ccds.propid == '2014B-0404') ccds.cut(np.hypot(ccds.ra_bore - ra, ccds.dec_bore - dec) < 2.5) print(np.unique(ccds.expnum), 'unique exposures') print('propids', np.unique(ccds.propid)) ccds.writeto(fn) else: ccds = fits_table(fn) plt.clf() for e in np.unique(ccds.expnum): I = np.flatnonzero(ccds.expnum == e) plt.plot(ccds.ra[I], ccds.dec[I], '.') ps.savefig() degw = 3.0 pixscale = 10. W = degw * 3600 / 10. H = W hi = 6 cmap = cmap_discretize('jet', hi + 1) wcs = Tan(ra, dec, W / 2. + 0.5, H / 2. + 0.5, -pixscale / 3600., 0., 0., pixscale / 3600., float(W), float(H)) r0, d0 = wcs.pixelxy2radec(1, 1) r1, d1 = wcs.pixelxy2radec(W, H) extent = [min(r0, r1), max(r0, r1), min(d0, d1), max(d0, d1)] for expnums in [ [348666], [348666, 348710, 348686], [348659, 348667, 348658, 348666, 348665, 348669, 348668], None, [ 348683, 348687, 347333, 348686, 348685, 348692, 348694, 348659, 348667, 348658, 348666, 348665, 348669, 348668, 348707, 348709, 348708, 348710, 348711, 348716, 348717 ], ]: nexp = np.zeros((H, W), np.uint8) for ccd in ccds: if expnums is not None and not ccd.expnum in expnums: continue ccdwcs = survey.get_approx_wcs(ccd) r, d = ccdwcs.pixelxy2radec(1, 1) ok, x0, y0 = wcs.radec2pixelxy(r, d) r, d = ccdwcs.pixelxy2radec(ccd.width, ccd.height) ok, x1, y1 = wcs.radec2pixelxy(r, d) xlo = np.clip(int(np.round(min(x0, x1))) - 1, 0, W - 1) xhi = np.clip(int(np.round(max(x0, x1))) - 1, 0, W - 1) ylo = np.clip(int(np.round(min(y0, y1))) - 1, 0, H - 1) yhi = np.clip(int(np.round(max(y0, y1))) - 1, 0, H - 1) nexp[ylo:yhi + 1, xlo:xhi + 1] += 1 plt.clf() plt.imshow(nexp, interpolation='nearest', origin='lower', vmin=-0.5, vmax=hi + 0.5, cmap=cmap, extent=extent) plt.colorbar(ticks=np.arange(hi + 1)) ps.savefig() O = fits_table('obstatus/decam-tiles_obstatus.fits') O.cut(np.hypot(O.ra - ra, O.dec - dec) < 2.5) for p in [1, 2, 3]: print('Pass', p, 'exposures:', O.r_expnum[O.get('pass') == p]) O.cut(O.get('pass') == 2) print(len(O), 'pass 2 nearby') d = np.hypot(O.ra - ra, O.dec - dec) print('Dists:', d) I = np.flatnonzero(d < 0.5) assert (len(I) == 1) ocenter = O[I[0]] print('Center expnum', ocenter.r_expnum) I = np.flatnonzero(d >= 0.5) O.cut(I) #center = ccds[ccds.expnum == ocenter.r_expnum] #p2 = ccds[ccds. ok, xc, yc = wcs.radec2pixelxy(ocenter.ra, ocenter.dec) xx, yy = np.meshgrid(np.arange(W) + 1, np.arange(H) + 1) c_d2 = (xc - xx)**2 + (yc - yy)**2 best = np.ones((H, W), bool) for o in O: ok, x, y = wcs.radec2pixelxy(o.ra, o.dec) d2 = (x - xx)**2 + (y - yy)**2 best[d2 < c_d2] = False del d2 del c_d2, xx, yy # plt.clf() # plt.imshow(best, interpolation='nearest', origin='lower', cmap='gray', # vmin=0, vmax=1) # ps.savefig() plt.clf() plt.imshow(nexp * best, interpolation='nearest', origin='lower', vmin=-0.5, vmax=hi + 0.5, cmap=cmap, extent=extent) plt.colorbar(ticks=np.arange(hi + 1)) ps.savefig() plt.clf() n, b, p = plt.hist(np.clip(nexp[best], 0, hi), range=(-0.5, hi + 0.5), bins=hi + 1) plt.xlim(-0.5, hi + 0.5) ps.savefig() print('b', b) print('n', n) print('fracs', np.array(n) / np.sum(n)) print('pcts', ', '.join(['%.1f' % f for f in 100. * np.array(n) / np.sum(n)]))
def main(): global survey init() ps = PlotSequence('sky') # export LEGACY_SURVEY_DIR=/scratch1/scratchdirs/desiproc/DRs/dr4-bootes/legacypipe-dir/ #survey = LegacySurveyData() survey = get_survey('dr4v2') ccds = survey.get_ccds_readonly() print(len(ccds), 'CCDs') ccds = ccds[ccds.camera == 'mosaic'] print(len(ccds), 'Mosaic CCDs') # plt.clf() # plt.hist(ccds.mjd_obs % 1.0, bins=50) # plt.xlabel('MJD mod 1') # ps.savefig() ccds.imjd = np.floor(ccds.mjd_obs).astype(int) mjds = np.unique(ccds.imjd) print(len(mjds), 'unique MJDs') mp = multiproc(nthreads=8, init=init) allvals = [] medmjds = [] args = [] for kk, mjd in enumerate(mjds): I = np.flatnonzero(ccds.imjd == mjd) print('MJD', mjd, ' (%i of %i):' % (kk + 1, len(mjds)), len(I), 'CCDs') if len(I) == 0: continue # pick one near the middle #i = I[len(I)/2] for i in [I[len(I) / 4], I[len(I) / 2], I[3 * len(I) / 4]]: ccd = ccds[i] key = dict(expnum=ccd.expnum, ccdname=ccd.ccdname) oldvals = key vals = cache.find(key) print('Got', vals.count(), 'cache hits for', key) gotone = False for val in vals: #if 'median_adu' in val: if 'mode_adu' in val: print('cache hit:', val) allvals.append(val) medmjds.append(mjd) gotone = True break else: print('partial cache hit:', val) oldvals = val ###! cache.delete_one(val) if gotone: continue print('args: key', key, 'oldvals', oldvals) args.append((mjd, key, oldvals, ccd)) # plt.clf() # plt.hist(tim.getImage().ravel(), range=(-0.1, 0.1), bins=50, # histtype='step', color='b') # plt.axvline(med, color='b', alpha=0.3, lw=2) # plt.axvline(0., color='k', alpha=0.3, lw=2) # plt.xlabel('Sky-subtracted pixel values') # plt.title('Date ' + ccd.date_obs + ': ' + str(im)) # ps.savefig() if len(args): meds = mp.map(read_sky_val, args) print('Medians:', meds) for (mjd, key, oldvals, ccd), val in zip(args, meds): if val is None: continue allvals.append(val) medmjds.append(mjd) medians = [] median_adus = [] mode_adus = [] skyadus = [] keepmjds = [] for mjd, val in zip(medmjds, allvals): madu = val['median_adu'] if madu is None: continue if 'median' in val: medians.append(val['median']) else: from tractor.brightness import NanoMaggies zpscale = NanoMaggies.zeropointToScale(val['ccdzpt']) med = (val['median_adu'] - val['skyadu']) / zpscale print('Computed median diff:', med, 'nmgy') medians.append(med) keepmjds.append(mjd) median_adus.append(val['median_adu']) mode_adus.append(val['mode_adu']) skyadus.append(val['skyadu']) medmjds = keepmjds median_adus = np.array(median_adus) mode_adus = np.array(mode_adus) skyadus = np.array(skyadus) medmjds = np.array(medmjds) medians = np.array(medians) plt.clf() plt.plot(medmjds, median_adus - skyadus, 'b.') plt.xlabel('MJD') #plt.ylabel('Image median after sky subtraction (nmgy)') plt.ylabel('Image median - SKYADU (ADU)') #plt.ylim(-0.03, 0.03) #plt.ylim(-10, 10) plt.ylim(-1, 1) plt.axhline(0, color='k', lw=2, alpha=0.5) ps.savefig() plt.clf() plt.plot(medmjds, mode_adus - skyadus, 'b.') plt.xlabel('MJD') plt.ylabel('Image mode - SKYADU (ADU)') plt.ylim(-1, 1) plt.axhline(0, color='k', lw=2, alpha=0.5) ps.savefig() print('Median pcts:', np.percentile(medians, [0, 2, 50, 98, 100])) mlo, mhi = np.percentile(medians, [2, 98]) I = np.flatnonzero((medians > mlo) * (medians < mhi)) d0 = np.mean(medmjds) dmjd = medmjds[I] - d0 A = np.zeros((len(dmjd), 2)) A[:, 0] = 1. A[:, 1] = dmjd b = np.linalg.lstsq(A, medians[I])[0] print('Lstsq:', b) offset = b[0] slope = b[1] xx = np.array([dmjd.min(), dmjd.max()]) print('Offset', offset) print('Slope', slope) plt.clf() plt.plot(medmjds, medians, 'b.') ax = plt.axis() plt.plot(xx + d0, offset + slope * xx, 'b-') plt.axis(ax) plt.xlabel('MJD') plt.ylabel('Image median after sky subtraction (nmgy)') plt.ylim(-0.03, 0.03) plt.axhline(0, color='k', lw=2, alpha=0.5) ps.savefig() plt.clf() plt.plot(median_adus, skyadus, 'b.') ax = plt.axis() lo = min(ax[0], ax[2]) hi = max(ax[1], ax[3]) plt.plot([lo, hi], [lo, hi], 'k-', alpha=0.5) plt.xlabel('Median image ADU') plt.ylabel('SKYADU') plt.axis(ax) ps.savefig() plt.clf() plt.plot(medmjds, skyadus / median_adus, 'b.') plt.xlabel('MJD') plt.ylim(0.98, 1.02) plt.axhline(1.0, color='k', alpha=0.5) plt.ylabel('SKYADU / median image ADU') ps.savefig()
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('--name1', help='Name for first data set') parser.add_argument('--name2', help='Name for second data set') parser.add_argument('--plot-prefix', default='compare', help='Prefix for plot filenames; default "%default"') parser.add_argument('--match', default=1.0, help='Astrometric cross-match distance in arcsec') parser.add_argument('dir1', help='First directory to compare') parser.add_argument('dir2', help='Second directory to compare') opt = parser.parse_args() ps = PlotSequence(opt.plot_prefix) name1 = opt.name1 if name1 is None: name1 = os.path.basename(opt.dir1) if not len(name1): name1 = os.path.basename(os.path.dirname(opt.dir1)) name2 = opt.name2 if name2 is None: name2 = os.path.basename(opt.dir2) if not len(name2): name2 = os.path.basename(os.path.dirname(opt.dir2)) tt = 'Comparing %s to %s' % (name1, name2) # regex for tractor-*.fits catalog filename catre = re.compile('tractor-.*.fits') cat1,cat2 = [],[] for basedir,cat in [(opt.dir1, cat1), (opt.dir2, cat2)]: for dirpath,dirnames,filenames in os.walk(basedir, followlinks=True): for fn in filenames: if not catre.match(fn): print('Skipping', fn, 'due to filename') continue fn = os.path.join(dirpath, fn) t = fits_table(fn) print(len(t), 'from', fn) cat.append(t) cat1 = merge_tables(cat1, columns='fillzero') cat2 = merge_tables(cat2, columns='fillzero') print('Total of', len(cat1), 'from', name1) print('Total of', len(cat2), 'from', name2) cat1.cut(cat1.brick_primary) cat2.cut(cat2.brick_primary) print('Total of', len(cat1), 'BRICK_PRIMARY from', name1) print('Total of', len(cat2), 'BRICK_PRIMARY from', name2) cat1.cut((cat1.decam_anymask[:,1] == 0) * (cat1.decam_anymask[:,2] == 0) * (cat1.decam_anymask[:,4] == 0)) cat2.cut((cat2.decam_anymask[:,1] == 0) * (cat2.decam_anymask[:,2] == 0) * (cat2.decam_anymask[:,4] == 0)) print('Total of', len(cat1), 'unmasked from', name1) print('Total of', len(cat2), 'unmasked from', name2) I,J,d = match_radec(cat1.ra, cat1.dec, cat2.ra, cat2.dec, opt.match/3600., nearest=True) print(len(I), 'matched') plt.clf() plt.hist(d * 3600., 100) plt.xlabel('Match distance (arcsec)') plt.title(tt) ps.savefig() matched1 = cat1[I] matched2 = cat2[J] for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]: K = np.flatnonzero((matched1.decam_flux_ivar[:,iband] > 0) * (matched2.decam_flux_ivar[:,iband] > 0)) print('Median mw_trans', band, 'is', np.median(matched1.decam_mw_transmission[:,iband])) plt.clf() plt.errorbar(matched1.decam_flux[K,iband], matched2.decam_flux[K,iband], fmt='.', color=cc, xerr=1./np.sqrt(matched1.decam_flux_ivar[K,iband]), yerr=1./np.sqrt(matched2.decam_flux_ivar[K,iband]), alpha=0.1, ) plt.xlabel('%s flux: %s' % (name1, band)) plt.ylabel('%s flux: %s' % (name2, band)) plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.) plt.axis([-100, 1000, -100, 1000]) plt.title(tt) ps.savefig() for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]: good = ((matched1.decam_flux_ivar[:,iband] > 0) * (matched2.decam_flux_ivar[:,iband] > 0)) K = np.flatnonzero(good) psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') P = np.flatnonzero(good * psf1 * psf2) mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband]) iv1 = matched1.decam_flux_ivar[:, iband] iv2 = matched2.decam_flux_ivar[:, iband] std = np.sqrt(1./iv1 + 1./iv2) plt.clf() plt.plot(mag1[K], (matched2.decam_flux[K,iband] - matched1.decam_flux[K,iband]) / std[K], '.', alpha=0.1, color=cc) plt.plot(mag1[P], (matched2.decam_flux[P,iband] - matched1.decam_flux[P,iband]) / std[P], '.', alpha=0.1, color='k') plt.ylabel('(%s - %s) flux / flux errors (sigma): %s' % (name2, name1, band)) plt.xlabel('%s mag: %s' % (name1, band)) plt.axhline(0, color='k', alpha=0.5) plt.axis([24, 16, -10, 10]) plt.title(tt) ps.savefig() plt.clf() lp,lt = [],[] for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]: good = ((matched1.decam_flux_ivar[:,iband] > 0) * (matched2.decam_flux_ivar[:,iband] > 0)) #good = True psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband]) iv1 = matched1.decam_flux_ivar[:, iband] iv2 = matched2.decam_flux_ivar[:, iband] std = np.sqrt(1./iv1 + 1./iv2) #std = np.hypot(std, 0.01) G = np.flatnonzero(good * psf1 * psf2 * np.isfinite(mag1) * (mag1 >= 20) * (mag1 < dict(g=24, r=23.5, z=22.5)[band])) n,b,p = plt.hist((matched2.decam_flux[G,iband] - matched1.decam_flux[G,iband]) / std[G], range=(-4, 4), bins=50, histtype='step', color=cc, normed=True) sig = (matched2.decam_flux[G,iband] - matched1.decam_flux[G,iband]) / std[G] print('Raw mean and std of points:', np.mean(sig), np.std(sig)) med = np.median(sig) rsigma = (np.percentile(sig, 84) - np.percentile(sig, 16)) / 2. print('Median and percentile-based sigma:', med, rsigma) lp.append(p[0]) lt.append('%s: %.2f +- %.2f' % (band, med, rsigma)) bins = [] gaussint = [] for blo,bhi in zip(b, b[1:]): c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo) c /= (bhi - blo) #bins.extend([blo,bhi]) #gaussint.extend([c,c]) bins.append((blo+bhi)/2.) gaussint.append(c) plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5) plt.title(tt) plt.xlabel('Flux difference / error (sigma)') plt.axvline(0, color='k', alpha=0.1) plt.ylim(0, 0.45) plt.legend(lp, lt, loc='upper right') ps.savefig() for iband,band,cc in [(1,'g','g'),(2,'r','r'),(4,'z','m')]: plt.clf() mag1, magerr1 = NanoMaggies.fluxErrorsToMagErrors( matched1.decam_flux[:,iband], matched1.decam_flux_ivar[:,iband]) mag2, magerr2 = NanoMaggies.fluxErrorsToMagErrors( matched2.decam_flux[:,iband], matched2.decam_flux_ivar[:,iband]) meanmag = NanoMaggies.nanomaggiesToMag(( matched1.decam_flux[:,iband] + matched2.decam_flux[:,iband]) / 2.) psf1 = (matched1.type == 'PSF ') psf2 = (matched2.type == 'PSF ') good = ((matched1.decam_flux_ivar[:,iband] > 0) * (matched2.decam_flux_ivar[:,iband] > 0) * np.isfinite(mag1) * np.isfinite(mag2)) K = np.flatnonzero(good) P = np.flatnonzero(good * psf1 * psf2) plt.errorbar(mag1[K], mag2[K], fmt='.', color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1) plt.plot(mag1[P], mag2[P], 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('%s %s (mag)' % (name2, band)) plt.plot([-1e6, 1e6], [-1e6,1e6], 'k-', alpha=1.) plt.axis([24, 16, 24, 16]) plt.title(tt) ps.savefig() plt.clf() plt.errorbar(mag1[K], mag2[K] - mag1[K], fmt='.', color=cc, xerr=magerr1[K], yerr=magerr2[K], alpha=0.1) plt.plot(mag1[P], mag2[P] - mag1[P], 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('%s %s - %s %s (mag)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axis([24, 16, -1, 1]) plt.title(tt) ps.savefig() magbins = np.arange(16, 24.001, 0.5) plt.clf() plt.plot(mag1[K], (mag2[K]-mag1[K]) / np.hypot(magerr1[K], magerr2[K]), '.', color=cc, alpha=0.1) plt.plot(mag1[P], (mag2[P]-mag1[P]) / np.hypot(magerr1[P], magerr2[P]), 'k.', alpha=0.5) plt.xlabel('%s %s (mag)' % (name1, band)) plt.ylabel('(%s %s - %s %s) / errors (sigma)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axis([24, 16, -10, 10]) plt.title(tt) ps.savefig() y = (mag2 - mag1) / np.hypot(magerr1, magerr2) plt.clf() plt.plot(meanmag[P], y[P], 'k.', alpha=0.1) midmag = [] vals = np.zeros((len(magbins)-1, 5)) median_err1 = [] iqd_gauss = scipy.stats.norm.ppf(0.75) - scipy.stats.norm.ppf(0.25) # FIXME -- should we do some stats after taking off the mean difference? for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])): I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)] midmag.append((mlo+mhi)/2.) median_err1.append(np.median(magerr1[I])) if len(I) == 0: continue # median and +- 1 sigma quantiles ybin = y[I] vals[bini,0] = np.percentile(ybin, 16) vals[bini,1] = np.median(ybin) vals[bini,2] = np.percentile(ybin, 84) # +- 2 sigma quantiles vals[bini,3] = np.percentile(ybin, 2.3) vals[bini,4] = np.percentile(ybin, 97.7) iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25) print('Mag bin', midmag[-1], ': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points') # if iqd > iqd_gauss: # # What error adding in quadrature would you need to make the IQD match? # err = median_err1[-1] # target_err = err * (iqd / iqd_gauss) # sys_err = np.sqrt(target_err**2 - err**2) # print('--> add systematic error', sys_err) # ~ Johan's cuts mlo = 21. mhi = dict(g=24., r=23.5, z=22.5)[band] I = P[(meanmag[P] >= mlo) * (meanmag[P] < mhi)] ybin = y[I] iqd = np.percentile(ybin, 75) - np.percentile(ybin, 25) print('Mag bin', mlo, mhi, 'band', band, ': IQD is factor', iqd / iqd_gauss, 'vs expected for Gaussian;', len(ybin), 'points') if iqd > iqd_gauss: # What error adding in quadrature would you need to make # the IQD match? err = np.median(np.hypot(magerr1[I], magerr2[I])) print('Median error (hypot):', err) target_err = err * (iqd / iqd_gauss) print('Target:', target_err) sys_err = np.sqrt((target_err**2 - err**2) / 2.) print('--> add systematic error', sys_err) # check... err_sys = np.hypot(np.hypot(magerr1, sys_err), np.hypot(magerr2, sys_err)) ysys = (mag2 - mag1) / err_sys ysys = ysys[I] print('Resulting median error:', np.median(err_sys[I])) iqd_sys = np.percentile(ysys, 75) - np.percentile(ysys, 25) print('--> IQD', iqd_sys / iqd_gauss, 'vs Gaussian') # Hmmm, this doesn't work... totally overshoots. plt.errorbar(midmag, vals[:,1], fmt='o', color='b', yerr=(vals[:,1]-vals[:,0], vals[:,2]-vals[:,1]), capthick=3, zorder=20) plt.errorbar(midmag, vals[:,1], fmt='o', color='b', yerr=(vals[:,1]-vals[:,3], vals[:,4]-vals[:,1]), capthick=2, zorder=20) plt.axhline( 1., color='b', alpha=0.2) plt.axhline(-1., color='b', alpha=0.2) plt.axhline( 2., color='b', alpha=0.2) plt.axhline(-2., color='b', alpha=0.2) for mag,err,y in zip(midmag, median_err1, vals[:,3]): if not np.isfinite(err): continue if y < -6: continue plt.text(mag, y-0.1, '%.3f' % err, va='top', ha='center', color='k', fontsize=10) plt.xlabel('(%s + %s)/2 %s (mag), PSFs' % (name1, name2, band)) plt.ylabel('(%s %s - %s %s) / errors (sigma)' % (name2, band, name1, band)) plt.axhline(0., color='k', alpha=1.) plt.axvline(21, color='k', alpha=0.3) plt.axvline(dict(g=24, r=23.5, z=22.5)[band], color='k', alpha=0.3) plt.axis([24.1, 16, -6, 6]) plt.title(tt) ps.savefig() #magbins = np.append([16, 18], np.arange(20, 24.001, 0.5)) if band == 'g': magbins = [20, 24] elif band == 'r': magbins = [20, 23.5] elif band == 'z': magbins = [20, 22.5] slo,shi = -5,5 plt.clf() ha = dict(bins=25, range=(slo,shi), histtype='step', normed=True) y = (mag2 - mag1) / np.hypot(magerr1, magerr2) midmag = [] nn = [] rgbs = [] lt,lp = [],[] for bini,(mlo,mhi) in enumerate(zip(magbins, magbins[1:])): I = P[(mag1[P] >= mlo) * (mag1[P] < mhi)] if len(I) == 0: continue ybin = y[I] rgb = [0.,0.,0.] rgb[0] = float(bini) / (len(magbins)-1) rgb[2] = 1. - rgb[0] n,b,p = plt.hist(ybin, color=rgb, **ha) lt.append('mag %g to %g' % (mlo,mhi)) lp.append(p[0]) midmag.append((mlo+mhi)/2.) nn.append(n) rgbs.append(rgb) bins = [] gaussint = [] for blo,bhi in zip(b, b[1:]): #midbin.append((blo+bhi)/2.) #gaussint.append(scipy.stats.norm.cdf(bhi) - # scipy.stats.norm.cdf(blo)) c = scipy.stats.norm.cdf(bhi) - scipy.stats.norm.cdf(blo) c /= (bhi - blo) bins.extend([blo,bhi]) gaussint.extend([c,c]) plt.plot(bins, gaussint, 'k-', lw=2, alpha=0.5) plt.legend(lp, lt) plt.title(tt) plt.xlim(slo,shi) ps.savefig() bincenters = b[:-1] + (b[1]-b[0])/2. plt.clf() lp = [] for n,rgb,mlo,mhi in zip(nn, rgbs, magbins, magbins[1:]): p = plt.plot(bincenters, n, '-', color=rgb) lp.append(p[0]) plt.plot(bincenters, gaussint[::2], 'k-', alpha=0.5, lw=2) plt.legend(lp, lt) plt.title(tt) plt.xlim(slo,shi) ps.savefig()
def stage0(**kwargs): ps = PlotSequence('cfht') decals = CfhtDecals() B = decals.get_bricks() print 'Bricks:' B.about() ra,dec = 190.0, 11.0 #bands = 'ugri' bands = 'gri' B.cut(np.argsort(degrees_between(ra, dec, B.ra, B.dec))) print 'Nearest bricks:', B.ra[:5], B.dec[:5], B.brickid[:5] brick = B[0] pixscale = 0.186 #W,H = 1024,1024 #W,H = 2048,2048 #W,H = 3600,3600 W,H = 4800,4800 targetwcs = wcs_for_brick(brick, pixscale=pixscale, W=W, H=H) ccdfn = 'cfht-ccds.fits' if os.path.exists(ccdfn): T = fits_table(ccdfn) else: T = get_ccd_list() T.writeto(ccdfn) print len(T), 'CCDs' T.cut(ccds_touching_wcs(targetwcs, T)) print len(T), 'CCDs touching brick' T.cut(np.array([b in bands for b in T.filter])) print len(T), 'in bands', bands ims = [] for t in T: im = CfhtImage(t) # magzp = hdr['PHOT_C'] + 2.5 * np.log10(hdr['EXPTIME']) # fwhm = t.seeing / (pixscale * 3600) # print '-> FWHM', fwhm, 'pix' im.seeing = t.seeing im.pixscale = t.pixscale print 'seeing', t.seeing print 'pixscale', im.pixscale*3600, 'arcsec/pix' im.run_calibs(t.ra, t.dec, im.pixscale, W=t.width, H=t.height) ims.append(im) # Read images, clip to ROI targetrd = np.array([targetwcs.pixelxy2radec(x,y) for x,y in [(1,1),(W,1),(W,H),(1,H),(1,1)]]) keepims = [] tims = [] for im in ims: print print 'Reading expnum', im.expnum, 'name', im.extname, 'band', im.band, 'exptime', im.exptime band = im.band wcs = im.read_wcs() imh,imw = wcs.imageh,wcs.imagew imgpoly = [(1,1),(1,imh),(imw,imh),(imw,1)] ok,tx,ty = wcs.radec2pixelxy(targetrd[:-1,0], targetrd[:-1,1]) tpoly = zip(tx,ty) clip = clip_polygon(imgpoly, tpoly) clip = np.array(clip) #print 'Clip', clip if len(clip) == 0: continue x0,y0 = np.floor(clip.min(axis=0)).astype(int) x1,y1 = np.ceil (clip.max(axis=0)).astype(int) slc = slice(y0,y1+1), slice(x0,x1+1) ## FIXME -- it seems I got lucky and the cross product is ## negative == clockwise, as required by clip_polygon. One ## could check this and reverse the polygon vertex order. # dx0,dy0 = tx[1]-tx[0], ty[1]-ty[0] # dx1,dy1 = tx[2]-tx[1], ty[2]-ty[1] # cross = dx0*dy1 - dx1*dy0 # print 'Cross:', cross print 'Image slice: x [%i,%i], y [%i,%i]' % (x0,x1, y0,y1) print 'Reading image from', im.imgfn, 'HDU', im.hdu img,imghdr = im.read_image(header=True, slice=slc) goodpix = (img != 0) print 'Number of pixels == 0:', np.sum(img == 0) print 'Number of pixels != 0:', np.sum(goodpix) if np.sum(goodpix) == 0: continue # print 'Image shape', img.shape print 'Image range', img.min(), img.max() print 'Goodpix image range:', (img[goodpix]).min(), (img[goodpix]).max() if img[goodpix].min() == img[goodpix].max(): print 'No dynamic range in image' continue # print 'Reading invvar from', im.wtfn, 'HDU', im.hdu # invvar = im.read_invvar(slice=slc) # # print 'Invvar shape', invvar.shape # # print 'Invvar range:', invvar.min(), invvar.max() # invvar[goodpix == 0] = 0. # if np.all(invvar == 0.): # print 'Skipping zero-invvar image' # continue # assert(np.all(np.isfinite(img))) # assert(np.all(np.isfinite(invvar))) # assert(not(np.all(invvar == 0.))) # # Estimate per-pixel noise via Blanton's 5-pixel MAD # slice1 = (slice(0,-5,10),slice(0,-5,10)) # slice2 = (slice(5,None,10),slice(5,None,10)) # # print 'sliced shapes:', img[slice1].shape, img[slice2].shape # # print 'good shape:', (goodpix[slice1] * goodpix[slice2]).shape # # print 'good values:', np.unique(goodpix[slice1] * goodpix[slice2]) # # print 'sliced[good] shapes:', (img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].shape # mad = np.median(np.abs(img[slice1] - img[slice2])[goodpix[slice1] * goodpix[slice2]].ravel()) # sig1 = 1.4826 * mad / np.sqrt(2.) # print 'MAD sig1:', sig1 # # invvar was 1 or 0 # invvar *= (1./(sig1**2)) # medsky = np.median(img[goodpix]) # Read full image for sig1 and sky estimate fullimg = im.read_image() fullgood = (fullimg != 0) # Estimate per-pixel noise via Blanton's 5-pixel MAD slice1 = (slice(0,-5,10),slice(0,-5,10)) slice2 = (slice(5,None,10),slice(5,None,10)) mad = np.median(np.abs(fullimg[slice1] - fullimg[slice2])[fullgood[slice1] * fullgood[slice2]].ravel()) sig1 = 1.4826 * mad / np.sqrt(2.) print 'MAD sig1:', sig1 medsky = np.median(fullimg[fullgood]) invvar = np.zeros_like(img) invvar[goodpix] = 1./sig1**2 # Median-smooth sky subtraction plt.clf() dimshow(np.round((img-medsky) / sig1), vmin=-3, vmax=5) plt.title('Scalar median: %s' % im.name) ps.savefig() # medsky = np.zeros_like(img) # # astrometry.util.util # median_smooth(img, np.logical_not(goodpix), 256, medsky) fullmed = np.zeros_like(fullimg) median_smooth(fullimg - medsky, np.logical_not(fullgood), 256, fullmed) fullmed += medsky medimg = fullmed[slc] plt.clf() dimshow(np.round((img - medimg) / sig1), vmin=-3, vmax=5) plt.title('Median filtered: %s' % im.name) ps.savefig() #print 'Subtracting median:', medsky #img -= medsky img -= medimg primhdr = im.read_image_primary_header() magzp = decals.get_zeropoint_for(im) print 'magzp', magzp zpscale = NanoMaggies.zeropointToScale(magzp) print 'zpscale', zpscale # Scale images to Nanomaggies img /= zpscale sig1 /= zpscale invvar *= zpscale**2 orig_zpscale = zpscale zpscale = 1. assert(np.sum(invvar > 0) > 0) print 'After scaling:' print 'sig1', sig1 print 'invvar range', invvar.min(), invvar.max() print 'image range', img.min(), img.max() assert(np.all(np.isfinite(img))) assert(np.all(np.isfinite(invvar))) assert(np.isfinite(sig1)) plt.clf() lo,hi = -5*sig1, 10*sig1 n,b,p = plt.hist(img[goodpix].ravel(), 100, range=(lo,hi), histtype='step', color='k') xx = np.linspace(lo, hi, 200) plt.plot(xx, max(n)*np.exp(-xx**2 / (2.*sig1**2)), 'r-') plt.xlim(lo,hi) plt.title('Pixel histogram: %s' % im.name) ps.savefig() twcs = ConstantFitsWcs(wcs) if x0 or y0: twcs.setX0Y0(x0,y0) info = im.get_image_info() fullh,fullw = info['dims'] # read fit PsfEx model psfex = PsfEx.fromFits(im.psffitfn) print 'Read', psfex # HACK -- highly approximate PSF here! #psf_fwhm = imghdr['FWHM'] #psf_fwhm = im.seeing psf_fwhm = im.seeing / (im.pixscale * 3600) print 'PSF FWHM', psf_fwhm, 'pixels' psf_sigma = psf_fwhm / 2.35 psf = NCircularGaussianPSF([psf_sigma],[1.]) print 'img type', img.dtype tim = Image(img, invvar=invvar, wcs=twcs, psf=psf, photocal=LinearPhotoCal(zpscale, band=band), sky=ConstantSky(0.), name=im.name + ' ' + band) tim.zr = [-3. * sig1, 10. * sig1] tim.sig1 = sig1 tim.band = band tim.psf_fwhm = psf_fwhm tim.psf_sigma = psf_sigma tim.sip_wcs = wcs tim.x0,tim.y0 = int(x0),int(y0) tim.psfex = psfex tim.imobj = im mn,mx = tim.zr tim.ima = dict(interpolation='nearest', origin='lower', cmap='gray', vmin=mn, vmax=mx) tims.append(tim) keepims.append(im) ims = keepims print 'Computing resampling...' # save resampling params for tim in tims: wcs = tim.sip_wcs subh,subw = tim.shape subwcs = wcs.get_subimage(tim.x0, tim.y0, subw, subh) tim.subwcs = subwcs try: Yo,Xo,Yi,Xi,rims = resample_with_wcs(targetwcs, subwcs, [], 2) except OverlapError: print 'No overlap' continue if len(Yo) == 0: continue tim.resamp = (Yo,Xo,Yi,Xi) print 'Creating coadds...' # Produce per-band coadds, for plots coimgs = [] cons = [] for ib,band in enumerate(bands): coimg = np.zeros((H,W), np.float32) con = np.zeros((H,W), np.uint8) for tim in tims: if tim.band != band: continue (Yo,Xo,Yi,Xi) = tim.resamp if len(Yo) == 0: continue nn = (tim.getInvvar()[Yi,Xi] > 0) coimg[Yo,Xo] += tim.getImage ()[Yi,Xi] * nn con [Yo,Xo] += nn # print # print 'tim', tim.name # print 'number of resampled pix:', len(Yo) # reim = np.zeros_like(coimg) # ren = np.zeros_like(coimg) # reim[Yo,Xo] = tim.getImage()[Yi,Xi] * nn # ren[Yo,Xo] = nn # print 'number of resampled pix with positive invvar:', ren.sum() # plt.clf() # plt.subplot(2,2,1) # mn,mx = [np.percentile(reim[ren>0], p) for p in [25,95]] # print 'Percentiles:', mn,mx # dimshow(reim, vmin=mn, vmax=mx) # plt.colorbar() # plt.subplot(2,2,2) # dimshow(con) # plt.colorbar() # plt.subplot(2,2,3) # dimshow(reim, vmin=tim.zr[0], vmax=tim.zr[1]) # plt.colorbar() # plt.subplot(2,2,4) # plt.hist(reim.ravel(), 100, histtype='step', color='b') # plt.hist(tim.getImage().ravel(), 100, histtype='step', color='r') # plt.suptitle('%s: %s' % (band, tim.name)) # ps.savefig() coimg /= np.maximum(con,1) coimgs.append(coimg) cons .append(con) plt.clf() dimshow(get_rgb(coimgs, bands)) ps.savefig() plt.clf() for i,b in enumerate(bands): plt.subplot(2,2,i+1) dimshow(cons[i], ticks=False) plt.title('%s band' % b) plt.colorbar() plt.suptitle('Number of exposures') ps.savefig() print 'Grabbing SDSS sources...' bandlist = [b for b in bands] cat,T = get_sdss_sources(bandlist, targetwcs) # record coordinates in target brick image ok,T.tx,T.ty = targetwcs.radec2pixelxy(T.ra, T.dec) T.tx -= 1 T.ty -= 1 T.itx = np.clip(np.round(T.tx).astype(int), 0, W-1) T.ity = np.clip(np.round(T.ty).astype(int), 0, H-1) plt.clf() dimshow(get_rgb(coimgs, bands)) ax = plt.axis() plt.plot(T.tx, T.ty, 'o', mec=green, mfc='none', ms=10, mew=1.5) plt.axis(ax) plt.title('SDSS sources') ps.savefig() print 'Detmaps...' # Render the detection maps detmaps = dict([(b, np.zeros((H,W), np.float32)) for b in bands]) detivs = dict([(b, np.zeros((H,W), np.float32)) for b in bands]) for tim in tims: iv = tim.getInvvar() psfnorm = 1./(2. * np.sqrt(np.pi) * tim.psf_sigma) detim = tim.getImage().copy() detim[iv == 0] = 0. detim = gaussian_filter(detim, tim.psf_sigma) / psfnorm**2 detsig1 = tim.sig1 / psfnorm subh,subw = tim.shape detiv = np.zeros((subh,subw), np.float32) + (1. / detsig1**2) detiv[iv == 0] = 0. (Yo,Xo,Yi,Xi) = tim.resamp detmaps[tim.band][Yo,Xo] += detiv[Yi,Xi] * detim[Yi,Xi] detivs [tim.band][Yo,Xo] += detiv[Yi,Xi] rtn = dict() for k in ['T', 'coimgs', 'cons', 'detmaps', 'detivs', 'targetrd', 'pixscale', 'targetwcs', 'W','H', 'bands', 'tims', 'ps', 'brick', 'cat']: rtn[k] = locals()[k] return rtn