def apply_picker(picker): # create copy of hand that we'll remove cards from as cards are selected for playing pick_hand = list(hand) picked_cards = [] first_data = [ TractorMetadata(tractor.rank, tractor.length) for tractor in first_tractors ] while first_data: # compute the minimum rank/length tractor we must play suit_tractors = state.get_suit_tractors_from_cards( pick_hand, trick_suit) hand_data = [ TractorMetadata(tractor.rank, tractor.length) for tractor in suit_tractors ] if len(hand_data) == 0: break hand_idx = find_matching_data_index(hand_data, first_data[0]) min_data = get_min_data(first_data[0], hand_data[hand_idx]) # get tractors that exactly match the minimum, and tractors that are larger in rank/length exact = [] larger = [] for tractor in suit_tractors: if tractor.rank == min_data.rank and tractor.length == min_data.length: exact.append(tractor) elif tractor.rank >= min_data.rank and tractor.length >= min_data.length: larger.append(tractor) # run the picker cur_picked = picker(min_data, exact, larger) for card in cur_picked: picked_cards.append(card) pick_hand.remove(card) first_data = update_data_array(first_data, min_data) # at this point there should be no cards in suit matching first_play # so we provide single cards to the picker while len(picked_cards) < len(first_play): exact = [] for card in pick_hand: suit_type = card_to_suit_type(card, trick_card.suit, state.trump_card) tractor = Tractor(1, 1, card.suit_power(state.trump_card), suit_type) tractor.orig_cards = [[card]] exact.append(tractor) cur_picked = picker(TractorMetadata(1, 1), exact, []) for card in cur_picked: picked_cards.append(card) pick_hand.remove(card) return picked_cards
def get_splits(large, small): # first split the length, then for each split, cut the rank splits = [] for i in range(large.length - small.length + 1): tractor = Tractor(large.rank, small.length, large.power + i, large.suit_type) tractor.orig_cards = large.orig_cards[i:i + small.length] splits.append(tractor) for split in splits: split.rank = small.rank split.orig_cards = [t[:small.rank] for t in split.orig_cards] return splits
def _unwise_mod(tim, use_cat, use_srcs, margin=10): # Select sources in play. wisewcs = tim.wcs.wcs timH, timW = tim.shape ok, x, y = wisewcs.radec2pixelxy(use_cat.ra, use_cat.dec) x = (x - 1.).astype(np.float32) y = (y - 1.).astype(np.float32) I = np.flatnonzero((x >= -margin) * (x < timW + margin) * (y >= -margin) * (y < timH + margin)) #print('Found {} sources within the image + margin = {} pixels'.format(len(I), margin)) subcat = [use_srcs[i] for i in I] tractor = Tractor([tim], subcat) mod = tractor.getModelImage(0) return mod
def init(cont): if cont.owner["is3P"]: if cont.owner["isUser"]: import tractor.settings.estractor as settings else: import tractor.settings.estractor_sentient as settings else: import tractor.settings.estractorfp as settings cont.activate("set_hud") trac = Tractor(cont.owner) trac.setup(settings) cont.owner.controller = cont cont.owner["initialized"] = True
def run_forced_phot(cat, tim, ceres=True, derivs=False, agn=False, do_forced=True, do_apphot=True, get_model=False, ps=None, timing=False, fixed_also=False, ceres_threads=1): ''' fixed_also: if derivs=True, also run without derivatives and report that flux too? ''' if timing: tlast = Time() if ps is not None: import pylab as plt opti = None forced_kwargs = {} if ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 try: opti = CeresOptimizer(BW=B, BH=B, threads=ceres_threads) except: if ceres_threads > 1: raise RuntimeError( 'ceres_threads requested but not supported by tractor.ceres version' ) opti = CeresOptimizer(BW=B, BH=B) #forced_kwargs.update(verbose=True) # nsize = 0 for src in cat: # Limit sizes of huge models # from tractor.galaxy import ProfileGalaxy # if isinstance(src, ProfileGalaxy): # px,py = tim.wcs.positionToPixel(src.getPosition()) # h = src._getUnitFluxPatchSize(tim, px, py, tim.modelMinval) # MAXHALF = 128 # if h > MAXHALF: # #print('halfsize', h,'for',src,'-> setting to',MAXHALF) # nsize += 1 # src.halfsize = MAXHALF src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) #print('Limited the size of', nsize, 'large galaxy models') if derivs: realsrcs = [] derivsrcs = [] Iderivs = [] for i, src in enumerate(cat): from tractor import PointSource realsrcs.append(src) if not isinstance(src, PointSource): continue Iderivs.append(i) brightness_dra = src.getBrightness().copy() brightness_ddec = src.getBrightness().copy() brightness_dra.setParams(np.zeros(brightness_dra.numberOfParams())) brightness_ddec.setParams( np.zeros(brightness_ddec.numberOfParams())) brightness_dra.freezeAllBut(tim.band) brightness_ddec.freezeAllBut(tim.band) dsrc = SourceDerivatives(src, [brightness_dra, brightness_ddec], tim, ps) derivsrcs.append(dsrc) Iderivs = np.array(Iderivs) if fixed_also: pass else: # For convenience, put all the real sources at the front of # the list, so we can pull the IVs off the front of the list. cat = realsrcs + derivsrcs if agn: from tractor.galaxy import ExpGalaxy, DevGalaxy, FixedCompositeGalaxy from tractor import PointSource from legacypipe.survey import SimpleGalaxy, RexGalaxy realsrcs = [] agnsrcs = [] iagn = [] for i, src in enumerate(cat): realsrcs.append(src) ## ?? if isinstance(src, (SimpleGalaxy, RexGalaxy)): #print('Skipping SIMP or REX:', src) continue if isinstance(src, (ExpGalaxy, DevGalaxy, FixedCompositeGalaxy)): iagn.append(i) bright = src.getBrightness().copy() bright.setParams(np.zeros(bright.numberOfParams())) bright.freezeAllBut(tim.band) agn = PointSource(src.pos, bright) agn.freezeAllBut('brightness') #print('Adding "agn"', agn, 'to', src) #print('agn params:', agn.getParamNames()) agnsrcs.append(src) iagn = np.array(iagn) cat = realsrcs + agnsrcs print('Added AGN to', len(iagn), 'galaxies') tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') disable_galaxy_cache() F = fits_table() if do_forced: if timing and (derivs or agn): t = Time() print('Setting up:', t - tlast) tlast = t if derivs: if fixed_also: print('Forced photom with fixed positions:') R = tr.optimize_forced_photometry(variance=True, fitstats=False, shared_params=False, priors=False, **forced_kwargs) F.flux_fixed = np.array([ src.getBrightness().getFlux(tim.band) for src in cat ]).astype(np.float32) N = len(cat) F.flux_fixed_ivar = R.IV[:N].astype(np.float32) if timing: t = Time() print('Forced photom with fixed positions finished:', t - tlast) tlast = t cat = realsrcs + derivsrcs tr.setCatalog(Catalog(*cat)) print('Forced photom with position derivatives:') if ps is None and not get_model: forced_kwargs.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, priors=False, **forced_kwargs) if ps is not None or get_model: (data, mod, ie, chi, roi) = R.ims1[0] if ps is not None: ima = dict(vmin=-2. * tim.sig1, vmax=5. * tim.sig1, interpolation='nearest', origin='lower', cmap='gray') imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5, cmap='RdBu') plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() if derivs: trx = Tractor([tim], realsrcs) trx.freezeParam('images') modx = trx.getModelImage(0) chix = (data - modx) * tim.getInvError() plt.clf() plt.imshow(modx, **ima) plt.title('Model without derivatives: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chix, **imchi) plt.title('Chi without derivatives: %s' % tim.name) ps.savefig() if derivs or agn: cat = realsrcs N = len(cat) F.flux = np.array([ src.getBrightness().getFlux(tim.band) for src in cat ]).astype(np.float32) F.flux_ivar = R.IV[:N].astype(np.float32) F.fracflux = R.fitstats.profracflux[:N].astype(np.float32) F.rchisq = R.fitstats.prochi2[:N].astype(np.float32) try: F.fracmasked = R.fitstats.promasked[:N].astype(np.float32) except: print( 'No "fracmasked" available (only in recent Tractor versions)') if derivs: F.flux_dra = np.zeros(len(F), np.float32) F.flux_ddec = np.zeros(len(F), np.float32) F.flux_dra[Iderivs] = np.array( [src.getParams()[0] for src in derivsrcs]).astype(np.float32) F.flux_ddec[Iderivs] = np.array( [src.getParams()[1] for src in derivsrcs]).astype(np.float32) F.flux_dra_ivar = np.zeros(len(F), np.float32) F.flux_ddec_ivar = np.zeros(len(F), np.float32) F.flux_dra_ivar[Iderivs] = R.IV[N::2].astype(np.float32) F.flux_ddec_ivar[Iderivs] = R.IV[N + 1::2].astype(np.float32) if agn: F.flux_agn = np.zeros(len(F), np.float32) F.flux_agn_ivar = np.zeros(len(F), np.float32) F.flux_agn[iagn] = np.array( [src.getParams()[0] for src in agnsrcs]) F.flux_agn_ivar[iagn] = R.IV[N:].astype(np.float32) if timing: t = Time() print('Forced photom:', t - tlast) tlast = t if do_apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack( [tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() #print('Apertures:', apertures, 'pixels') #print('apxy shape', apxy.shape) # --> (2,N) # The aperture photometry routine doesn't like pixel positions outside the image H, W = img.shape Iap = np.flatnonzero((apxy[0, :] >= 0) * (apxy[1, :] >= 0) * (apxy[0, :] <= W - 1) * (apxy[1, :] <= H - 1)) print('Aperture photometry for', len(Iap), 'of', len(apxy[0, :]), 'sources within image bounds') for rad in apertures: aper = photutils.CircularAperture(apxy[:, Iap], rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = np.zeros((len(F), len(apertures)), np.float32) F.apflux[Iap, :] = ap.astype(np.float32) apimgerr = np.vstack(apimgerr).T apiv = np.zeros(apimgerr.shape, np.float32) apiv[apimgerr != 0] = 1. / apimgerr[apimgerr != 0]**2 F.apflux_ivar = np.zeros((len(F), len(apertures)), np.float32) F.apflux_ivar[Iap, :] = apiv if timing: print('Aperture photom:', Time() - tlast) if get_model: return F, mod return F
def main(survey=None, opt=None): '''Driver function for forced photometry of individual Legacy Survey images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if opt.derivs and opt.agn: print('Sorry, can\'t do --derivs AND --agn') sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0, x1, y0, y1) = opt.zoom zoomslice = (slice(y0, y1), slice(x0, x1)) ps = None if opt.plots is not None: import pylab as plt from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.expnum) filename = None except: # make this 'None' for survey.find_ccds() expnum = None filename = opt.expnum # Try parsing HDU number try: hdu = int(opt.ccdname) ccdname = None except: hdu = -1 ccdname = opt.ccdname if survey is None: survey = LegacySurveyData() catsurvey = survey if opt.catalog_dir is not None: catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir) if filename is not None and hdu >= 0: # FIXME -- try looking up in CCDs file? # Read metadata from file print('Warning: faking metadata from file contents') T = exposure_metadata([filename], hdus=[hdu]) print('Metadata:') T.about() if not 'ccdzpt' in T.columns(): phdr = fitsio.read_header(filename) T.ccdzpt = np.array([phdr['MAGZERO']]) print('WARNING: using header MAGZERO') T.ccdraoff = np.array([0.]) T.ccddecoff = np.array([0.]) print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.') else: # Read metadata from survey-ccds.fits table T = survey.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if hdu >= 0: T.cut(T.image_hdu == hdu) print(len(T), 'with HDU', hdu) if filename is not None: T.cut(np.array([f.strip() == filename for f in T.image_filename])) print(len(T), 'with filename', filename) if opt.camera is not None: T.cut(T.camera == opt.camera) print(len(T), 'with camera', opt.camera) assert (len(T) == 1) ccd = T[0] im = survey.get_image_object(ccd) if opt.do_calib: #from legacypipe.survey import run_calibs #kwa = dict(splinesky=True) #run_calibs((im, kwa)) im.run_calibs(splinesky=True) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True, constant_invvar=opt.constant_invvar, hybridPsf=opt.hybrid_psf, normalizePsf=opt.normalize_psf) print('Got tim:', tim) print('Read image:', Time() - t0) if opt.catfn in ['DR1', 'DR2', 'DR3', 'DR5', 'DR']: margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, survey=catsurvey) for b in bricks: # there is some overlap with this brick... read the catalog. fn = catsurvey.find_file('tractor', brick=b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec) W, H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) * (yy >= -margin) * (yy <= (H + margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') for col in ['out_of_bounds', 'left_blob']: if col in T.get_columns(): T.cut(T.get(col) == False) print('Cut to', len(T), 'on', col) if len(T): TT.append(T) if len(TT) == 0: print('No sources to photometer.') return 0 T = merge_tables(TT, columns='fillzero') T._header = TT[0]._header del TT print('Total of', len(T), 'catalog sources') # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) surveydir = survey.get_survey_dir() del survey kwargs = {} cols = T.get_columns() if 'flux_r' in cols and not 'decam_flux_r' in cols: kwargs.update(fluxPrefix='') cat = read_fits_catalog(T, **kwargs) # Replace the brightness (which will be a NanoMaggies with g,r,z) # with a NanoMaggies with this image's band only. for src in cat: src.brightness = NanoMaggies(**{tim.band: 1.}) print('Read catalog:', Time() - t0) print('Forced photom...') F = run_forced_phot(cat, tim, ceres=opt.ceres, derivs=opt.derivs, fixed_also=True, agn=opt.agn, do_forced=opt.forced, do_apphot=opt.apphot, ps=ps) t0 = Time() F.release = T.release F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.camera = np.array([ccd.camera] * len(F)) F.expnum = np.array([im.expnum] * len(F)).astype(np.int32) F.ccdname = np.array([im.ccdname] * len(F)) # "Denormalizing" F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32) F.ra = T.ra F.dec = T.dec ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x - 1).astype(np.float32) F.y = (y - 1).astype(np.float32) h, w = tim.shape F.mask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h - 1), np.clip(np.round(F.x).astype(int), 0, w - 1)] program_name = sys.argv[0] version_hdr = get_version_header(program_name, surveydir) filename = getattr(ccd, 'image_filename') if filename is None: # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) filename = os.path.join(d2, d1, fname) print('Trimmed filename to', filename) version_hdr.add_record( dict(name='CPFILE', value=filename, comment='CP file')) version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext')) version_hdr.add_record( dict(name='CAMERA', value=ccd.camera, comment='Camera')) version_hdr.add_record( dict(name='EXPNUM', value=im.expnum, comment='Exposure num')) version_hdr.add_record( dict(name='CCDNAME', value=im.ccdname, comment='CCD name')) version_hdr.add_record( dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record( dict(name='EXPOSURE', value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname), comment='Name of this image')) keys = [ 'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME' ] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = { 'exptime': 'sec', 'flux': 'nanomaggy', 'flux_ivar': '1/nanomaggy^2' } columns = F.get_columns() for i, col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) if opt.save_model or opt.save_data: hdr = fitsio.FITSHDR() tim.getWcs().wcs.add_to_header(hdr) if opt.save_model: print('Getting model image...') tr = Tractor([tim], cat) mod = tr.getModelImage(tim) fitsio.write(opt.save_model, mod, header=hdr, clobber=True) print('Wrote', opt.save_model) if opt.save_data: fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True) print('Wrote', opt.save_data) print('Finished forced phot:', Time() - t0) return 0
def wise_cutouts(ra, dec, radius, ps, pixscale=2.75, tractor_base=".", unwise_dir="unwise-coadds"): """ radius in arcsec. pixscale: WISE pixel scale in arcsec/pixel; make this smaller than 2.75 to oversample. """ npix = int(np.ceil(radius / pixscale)) print("Image size:", npix) W = H = npix pix = pixscale / 3600.0 wcs = Tan(ra, dec, (W + 1) / 2.0, (H + 1) / 2.0, -pix, 0.0, 0.0, pix, float(W), float(H)) # Find DECaLS bricks overlapping decals = Decals() B = bricks_touching_wcs(wcs, decals=decals) print("Found", len(B), "bricks overlapping") TT = [] for b in B.brickname: fn = os.path.join(tractor_base, "tractor", b[:3], "tractor-%s.fits" % b) T = fits_table(fn) print("Read", len(T), "from", b) primhdr = fitsio.read_header(fn) TT.append(T) T = merge_tables(TT) print("Total of", len(T), "sources") T.cut(T.brick_primary) print(len(T), "primary") margin = 20 ok, xx, yy = wcs.radec2pixelxy(T.ra, T.dec) I = np.flatnonzero((xx > -margin) * (yy > -margin) * (xx < W + margin) * (yy < H + margin)) T.cut(I) print(len(T), "within ROI") # Pull out DECaLS coadds (image, model, resid). dwcs = wcs.scale(2.0 * pixscale / 0.262) dh, dw = dwcs.shape print("DECaLS resampled shape:", dh, dw) tags = ["image", "model", "resid"] coimgs = [np.zeros((dh, dw, 3), np.uint8) for t in tags] for b in B.brickname: fn = os.path.join(tractor_base, "coadd", b[:3], b, "decals-%s-image-r.fits" % b) bwcs = Tan(fn) try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(dwcs, bwcs) except ResampleError: continue if len(Yo) == 0: continue print("Resampling", len(Yo), "pixels from", b) xl, xh, yl, yh = Xi.min(), Xi.max(), Yi.min(), Yi.max() print( "python legacypipe/runbrick.py -b %s --zoom %i %i %i %i --outdir cluster --pixpsf --splinesky --pipe --no-early-coadds" % (b, xl - 5, xh + 5, yl - 5, yh + 5) + " -P 'pickles/cluster-%(brick)s-%%(stage)s.pickle'" ) for i, tag in enumerate(tags): fn = os.path.join(tractor_base, "coadd", b[:3], b, "decals-%s-%s.jpg" % (b, tag)) img = plt.imread(fn) img = np.flipud(img) coimgs[i][Yo, Xo, :] = img[Yi, Xi, :] tt = dict(image="Image", model="Model", resid="Resid") for img, tag in zip(coimgs, tags): plt.clf() dimshow(img, ticks=False) plt.title("DECaLS grz %s" % tt[tag]) ps.savefig() # Find unWISE tiles overlapping tiles = unwise_tiles_touching_wcs(wcs) print("Cut to", len(tiles), "unWISE tiles") # Here we assume the targetwcs is axis-aligned and that the # edge midpoints yield the RA,Dec limits (true for TAN). r, d = wcs.pixelxy2radec(np.array([1, W, W / 2, W / 2]), np.array([H / 2, H / 2, 1, H])) # the way the roiradec box is used, the min/max order doesn't matter roiradec = [r[0], r[1], d[2], d[3]] ra, dec = T.ra, T.dec T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T srcs = read_fits_catalog(T, ellipseClass=EllipseE) wbands = [1, 2] wanyband = "w" for band in wbands: T.wise_flux[:, band - 1] *= 10.0 ** (primhdr["WISEAB%i" % band] / 2.5) coimgs = [np.zeros((H, W), np.float32) for b in wbands] comods = [np.zeros((H, W), np.float32) for b in wbands] con = [np.zeros((H, W), np.uint8) for b in wbands] for iband, band in enumerate(wbands): print("Photometering WISE band", band) wband = "w%i" % band for i, src in enumerate(srcs): # print('Source', src, 'brightness', src.getBrightness(), 'params', src.getBrightness().getParams()) # src.getBrightness().setParams([T.wise_flux[i, band-1]]) src.setBrightness(NanoMaggies(**{wanyband: T.wise_flux[i, band - 1]})) # print('Set source brightness:', src.getBrightness()) # The tiles have some overlap, so for each source, keep the # fit in the tile whose center is closest to the source. for tile in tiles: print("Reading tile", tile.coadd_id) tim = get_unwise_tractor_image(unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradec) if tim is None: print("Actually, no overlap with tile", tile.coadd_id) continue print("Read image with shape", tim.shape) # Select sources in play. wisewcs = tim.wcs.wcs H, W = tim.shape ok, x, y = wisewcs.radec2pixelxy(ra, dec) x = (x - 1.0).astype(np.float32) y = (y - 1.0).astype(np.float32) margin = 10.0 I = np.flatnonzero((x >= -margin) * (x < W + margin) * (y >= -margin) * (y < H + margin)) print(len(I), "within the image + margin") subcat = [srcs[i] for i in I] tractor = Tractor([tim], subcat) mod = tractor.getModelImage(0) # plt.clf() # dimshow(tim.getImage(), ticks=False) # plt.title('WISE %s %s' % (tile.coadd_id, wband)) # ps.savefig() # plt.clf() # dimshow(mod, ticks=False) # plt.title('WISE %s %s' % (tile.coadd_id, wband)) # ps.savefig() try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(wcs, tim.wcs.wcs) except ResampleError: continue if len(Yo) == 0: continue print("Resampling", len(Yo), "pixels from WISE", tile.coadd_id, band) coimgs[iband][Yo, Xo] += tim.getImage()[Yi, Xi] comods[iband][Yo, Xo] += mod[Yi, Xi] con[iband][Yo, Xo] += 1 for img, mod, n in zip(coimgs, comods, con): img /= np.maximum(n, 1) mod /= np.maximum(n, 1) for band, img, mod in zip(wbands, coimgs, comods): lo, hi = np.percentile(img, [25, 99]) plt.clf() dimshow(img, vmin=lo, vmax=hi, ticks=False) plt.title("WISE W%i Data" % band) ps.savefig() plt.clf() dimshow(mod, vmin=lo, vmax=hi, ticks=False) plt.title("WISE W%i Model" % band) ps.savefig() resid = img - mod mx = np.abs(resid).max() plt.clf() dimshow(resid, vmin=-mx, vmax=mx, ticks=False) plt.title("WISE W%i Resid" % band) ps.savefig() # kwa = dict(mn=-0.1, mx=2., arcsinh = 1.) kwa = dict(mn=-0.1, mx=2.0, arcsinh=None) rgb = _unwise_to_rgb(coimgs, **kwa) plt.clf() dimshow(rgb, ticks=False) plt.title("WISE W1/W2 Data") ps.savefig() rgb = _unwise_to_rgb(comods, **kwa) plt.clf() dimshow(rgb, ticks=False) plt.title("WISE W1/W2 Model") ps.savefig() kwa = dict(mn=-1, mx=1, arcsinh=None) rgb = _unwise_to_rgb([img - mod for img, mod in zip(coimgs, comods)], **kwa) plt.clf() dimshow(rgb, ticks=False) plt.title("WISE W1/W2 Resid") ps.savefig()
def psf_residuals(expnum, ccdname, stampsize=35, nstar=30, magrange=(13, 17), verbose=0, splinesky=False): # Set the debugging level. if verbose == 0: lvl = logging.INFO else: lvl = logging.DEBUG logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) pngprefix = 'qapsf-{}-{}'.format(expnum, ccdname) # Gather all the info we need about this CCD. decals = Decals() ccd = decals.find_ccds(expnum=expnum, ccdname=ccdname)[0] band = ccd.filter ps1band = dict(g=0, r=1, i=2, z=3, Y=4) print('Band {}'.format(band)) #scales = dict(g=0.0066, r=0.01, z=0.025) #vmin, vmax = np.arcsinh(-1), np.arcsinh(100) #print(scales[band]) im = decals.get_image_object(ccd) iminfo = im.get_image_info() H, W = iminfo['dims'] wcs = im.get_wcs() # Choose a uniformly selected subset of PS1 stars on this CCD. ps1 = ps1cat(ccdwcs=wcs) cat = ps1.get_stars(band=band, magrange=magrange) rand = np.random.RandomState(seed=expnum * ccd.ccdnum) these = rand.choice(len(cat) - 1, nstar, replace=False) #these = rand.random_integers(0,len(cat)-1,nstar) cat = cat[these] cat = cat[np.argsort(cat.median[:, ps1band[band]])] # sort by magnitude #print(cat.nmag_ok) get_tim_kwargs = dict(const2psf=True, splinesky=splinesky) # Make a QAplot of the positions of all the stars. tim = im.get_tractor_image(**get_tim_kwargs) img = tim.getImage() #img = tim.getImage()/scales[band] fig = plt.figure(figsize=(5, 10)) ax = fig.gca() ax.get_xaxis().get_major_formatter().set_useOffset(False) #ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest', # origin='lower',vmin=vmax,vmax=vmax) ax.imshow(img, **tim.ima) ax.axis('off') ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.format( band, expnum, ccdname, ccd.airmass, ccd.seeing)) for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ax.text(xpos, ypos, '{:2d}'.format(istar + 1), color='red', horizontalalignment='left') circ = plt.Circle((xpos, ypos), radius=30, color='g', fill=False, lw=1) ax.add_patch(circ) #radec = wcs.radec_bounds() #ax.scatter(cat.ra,cat.dec) #ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998]) #ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015]) #ax.set_xlabel('$RA\ (deg)$',fontsize=18) #ax.set_ylabel('$Dec\ (deg)$',fontsize=18) fig.savefig(pngprefix + '-ccd.png', bbox_inches='tight') # Initialize the many-stamp QAplot ncols = 3 nrows = np.ceil(nstar / ncols).astype('int') inchperstamp = 2.0 fig = plt.figure(figsize=(inchperstamp * 3 * ncols, inchperstamp * nrows)) irow = 0 icol = 0 for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) mag = ps1star.median[ps1band[band]] # r-band ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ix, iy = int(xpos), int(ypos) # create a little tractor Image object around the star slc = (slice(max(iy - stampsize, 0), min(iy + stampsize + 1, H)), slice(max(ix - stampsize, 0), min(ix + stampsize + 1, W))) # The PSF model 'const2Psf' is the one used in DR1: a 2-component # Gaussian fit to PsfEx instantiated in the image center. tim = im.get_tractor_image(slc=slc, **get_tim_kwargs) stamp = tim.getImage() ivarstamp = tim.getInvvar() # Initialize a tractor PointSource from PS1 measurements flux = NanoMaggies.magToNanomaggies(mag) star = PointSource(RaDecPos(ra, dec), NanoMaggies(**{band: flux})) # Fit just the source RA,Dec,flux. tractor = Tractor([tim], [star]) tractor.freezeParam('images') print('2-component MOG:', tim.psf) tractor.printThawedParams() for step in range(50): dlnp, X, alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_mog = tractor.getModelImage(0) chi2_mog = -2.0 * tractor.getLogLikelihood() mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0] # Now change the PSF model to a pixelized PSF model from PsfEx instantiated # at this place in the image. psf = PixelizedPsfEx(im.psffn) tim.psf = psf.constantPsfAt(xpos, ypos) #print('PSF model:', tim.psf) #tractor.printThawedParams() for step in range(50): dlnp, X, alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_psfex = tractor.getModelImage(0) chi2_psfex = -2.0 * tractor.getLogLikelihood() mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0] #mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95]) sig = np.std((stamp - model_psfex)[ivarstamp > 0]) mn, mx = [-2.0 * sig, 5 * sig] # Generate a QAplot. if (istar > 0) and (istar % (ncols) == 0): irow = irow + 1 icol = 3 * istar - 3 * ncols * irow #print(istar, irow, icol, icol+1, icol+2) ax1 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol), aspect='equal') ax1.axis('off') #ax1.imshow(stamp, **tim.ima) ax1.imshow(stamp, cmap='gray', interpolation='nearest', origin='lower', vmin=mn, vmax=mx) ax1.text(0.1, 0.9, '{:2d}'.format(istar + 1), color='white', horizontalalignment='left', verticalalignment='top', transform=ax1.transAxes) ax2 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 1), aspect='equal') ax2.axis('off') #ax2.imshow(stamp-model_mog, **tim.ima) ax2.imshow(stamp - model_mog, cmap='gray', interpolation='nearest', origin='lower', vmin=mn, vmax=mx) ax2.text(0.1, 0.9, 'MoG', color='white', horizontalalignment='left', verticalalignment='top', transform=ax2.transAxes) ax2.text(0.08, 0.08, '{:.3f}'.format(mag_mog), color='white', horizontalalignment='left', verticalalignment='bottom', transform=ax2.transAxes) #ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14) #ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex)) ax3 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 2), aspect='equal') ax3.axis('off') #ax3.imshow(stamp-model_psfex, **tim.ima) ax3.imshow(stamp - model_psfex, cmap='gray', interpolation='nearest', origin='lower', vmin=mn, vmax=mx) ax3.text(0.1, 0.9, 'PSFEx', color='white', horizontalalignment='left', verticalalignment='top', transform=ax3.transAxes) ax3.text(0.08, 0.08, '{:.3f}'.format(mag_psfex), color='white', horizontalalignment='left', verticalalignment='bottom', transform=ax3.transAxes) if istar == (nstar - 1): break fig.savefig(pngprefix + '-stargrid.png', bbox_inches='tight')
def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('--catalog', help='Catalog to render') parser.add_argument('--brick-coadd', help='Produce a coadd of the images overlapping the given brickname.') parser.add_argument('--ccds', help='Use this table of CCDs') parser.add_argument('--brick-wcs', help='File containing a WCS header describing the coadd WCS to render.') parser.add_argument('--brick-wcs-ext', type=int, help='FITS file extension containing a WCS header describing the coadd WCS to render.') parser.add_argument('--outlier-mask-brick', help='Comma-separated list of bricknames from which outlier masks should be read.') parser.add_argument('--out', help='Filename pattern ("BAND" will be replaced by band name) of output images.') parser.add_argument('--resid', help='Filename pattern ("BAND" will be replaced by band name) of residual images.') parser.add_argument('--jpeg', help='Write RGB image to this filename') parser.add_argument('--resid-jpeg', help='Write RGB residual image to this filename') opt = parser.parse_args() if opt.catalog is None: print('Need catalog!') return -1 cat = fits_table(opt.catalog) if opt.ccds is None: if opt.brick_coadd is None: print('Need brick catalog!') return -1 brickname = opt.brick_coadd survey = LegacySurveyData() if opt.brick_wcs is None: print('FIXME') return -1 else: wcs = Tan(opt.brick_wcs, opt.brick_wcs_ext) tcat = read_fits_catalog(cat) if opt.ccds: ccdfn = opt.ccds else: ccdfn = survey.find_file('ccds-table', brick=brickname) print('Reading', ccdfn) ccds = fits_table(ccdfn) H,W = wcs.shape targetrd = np.array([wcs.pixelxy2radec(x,y) for x,y in [(1,1),(W,1),(W,H),(1,H),(1,1)]]) tims = [] for ccd in ccds: im = survey.get_image_object(ccd) #slc = slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1) #tim = im.get_tractor_image(slc=slc) tim = im.get_tractor_image(radecpoly=targetrd) print('Read', tim) tims.append(tim) if opt.outlier_mask_brick is not None: bricks = opt.outlier_mask_brick.split(',') for b in bricks: print('Reading outlier mask for brick', b, ':', survey.find_file('outliers_mask', brick=b, output=False)) ok = read_outlier_mask_file(survey, tims, b, subimage=True, output=False) tr = Tractor(tims, tcat) mods = list(tr.getModelImages()) bands = 'grz' def write_model(band, cowimg=None, cowmod=None, **kwargs): if cowmod is None: print('No model for', band) return outfn = opt.out.replace('BAND', band) fitsio.write(outfn, cowmod, clobber=True) print('Wrote model for', band, 'to', outfn) if opt.resid: outfn = opt.resid.replace('BAND', band) fitsio.write(outfn, cowimg - cowmod, clobber=True) print('Wrote resid for', band, 'to', outfn) C = make_coadds(tims, bands, wcs, mods=mods, callback=write_model) if opt.jpeg: from legacypipe.survey import get_rgb import pylab as plt plt.imsave(opt.jpeg, get_rgb(C.comods, bands), origin='lower') if opt.resid_jpeg: from legacypipe.survey import get_rgb import pylab as plt plt.imsave(opt.resid_jpeg, get_rgb([im-mod for im,mod in zip(C.coimgs, C.comods)], bands), origin='lower')
def psf_residuals(expnum,ccdname,stampsize=35,nstar=30, magrange=(13,17),verbose=0): # Set the debugging level. if verbose==0: lvl = logging.INFO else: lvl = logging.DEBUG logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout) pngprefix = 'qapsf-{}-{}'.format(expnum,ccdname) # Gather all the info we need about this CCD. decals = Decals() ccd = decals.find_ccds(expnum=expnum,ccdname=ccdname)[0] band = ccd.filter ps1band = dict(g=0,r=1,i=2,z=3,Y=4) print('Band {}'.format(band)) #scales = dict(g=0.0066, r=0.01, z=0.025) #vmin, vmax = np.arcsinh(-1), np.arcsinh(100) #print(scales[band]) im = DecamImage(decals,ccd) iminfo = im.get_image_info() H,W = iminfo['dims'] wcs = im.get_wcs() # Choose a uniformly selected subset of PS1 stars on this CCD. ps1 = ps1cat(ccdwcs=wcs) cat = ps1.get_stars(band=band,magrange=magrange) rand = np.random.RandomState(seed=expnum*ccd.ccdnum) these = rand.choice(len(cat)-1,nstar,replace=False) #these = rand.random_integers(0,len(cat)-1,nstar) cat = cat[these] cat = cat[np.argsort(cat.median[:,ps1band[band]])] # sort by magnitude #print(cat.nmag_ok) # Make a QAplot of the positions of all the stars. tim = im.get_tractor_image(const2psf=True) img = tim.getImage() #img = tim.getImage()/scales[band] fig = plt.figure(figsize=(5,10)) ax = fig.gca() ax.get_xaxis().get_major_formatter().set_useOffset(False) #ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest', # origin='lower',vmin=vmax,vmax=vmax) ax.imshow(img, **tim.ima) ax.axis('off') ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'. format(band,expnum,ccdname,ccd.airmass,ccd.seeing)) for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ax.text(xpos,ypos,'{:2d}'.format(istar+1),color='red', horizontalalignment='left') circ = plt.Circle((xpos,ypos),radius=30,color='g',fill=False,lw=1) ax.add_patch(circ) #radec = wcs.radec_bounds() #ax.scatter(cat.ra,cat.dec) #ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998]) #ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015]) #ax.set_xlabel('$RA\ (deg)$',fontsize=18) #ax.set_ylabel('$Dec\ (deg)$',fontsize=18) fig.savefig(pngprefix+'-ccd.png',bbox_inches='tight') # Initialize the many-stamp QAplot ncols = 3 nrows = np.ceil(nstar/ncols).astype('int') inchperstamp = 2.0 fig = plt.figure(figsize=(inchperstamp*3*ncols,inchperstamp*nrows)) irow = 0 icol = 0 for istar, ps1star in enumerate(cat): ra, dec = (ps1star.ra, ps1star.dec) mag = ps1star.median[ps1band[band]] # r-band ok, xpos, ypos = wcs.radec2pixelxy(ra, dec) ix,iy = int(xpos), int(ypos) # create a little tractor Image object around the star slc = (slice(max(iy-stampsize, 0), min(iy+stampsize+1, H)), slice(max(ix-stampsize, 0), min(ix+stampsize+1, W))) # The PSF model 'const2Psf' is the one used in DR1: a 2-component # Gaussian fit to PsfEx instantiated in the image center. tim = im.get_tractor_image(slc=slc, const2psf=True) stamp = tim.getImage() ivarstamp = tim.getInvvar() # Initialize a tractor PointSource from PS1 measurements flux = NanoMaggies.magToNanomaggies(mag) star = PointSource(RaDecPos(ra,dec), NanoMaggies(**{band: flux})) # Fit just the source RA,Dec,flux. tractor = Tractor([tim], [star]) tractor.freezeParam('images') print('2-component MOG:', tim.psf) tractor.printThawedParams() for step in range(50): dlnp,X,alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_mog = tractor.getModelImage(0) chi2_mog = -2.0*tractor.getLogLikelihood() mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0] # Now change the PSF model to a pixelized PSF model from PsfEx instantiated # at this place in the image. psf = PixelizedPsfEx(im.psffn) tim.psf = psf.constantPsfAt(xpos, ypos) #print('PSF model:', tim.psf) #tractor.printThawedParams() for step in range(50): dlnp,X,alpha = tractor.optimize() if dlnp < 0.1: break print('Fit:', star) model_psfex = tractor.getModelImage(0) chi2_psfex = -2.0*tractor.getLogLikelihood() mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0] #mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95]) sig = np.std((stamp-model_psfex)[ivarstamp>0]) mn, mx = [-2.0*sig,5*sig] # Generate a QAplot. if (istar>0) and (istar%(ncols)==0): irow = irow+1 icol = 3*istar - 3*ncols*irow #print(istar, irow, icol, icol+1, icol+2) ax1 = plt.subplot2grid((nrows,3*ncols), (irow,icol), aspect='equal') ax1.axis('off') #ax1.imshow(stamp, **tim.ima) ax1.imshow(stamp,cmap='gray',interpolation='nearest', origin='lower',vmin=mn,vmax=mx) ax1.text(0.1,0.9,'{:2d}'.format(istar+1),color='white', horizontalalignment='left',verticalalignment='top', transform=ax1.transAxes) ax2 = plt.subplot2grid((nrows,3*ncols), (irow,icol+1), aspect='equal') ax2.axis('off') #ax2.imshow(stamp-model_mog, **tim.ima) ax2.imshow(stamp-model_mog,cmap='gray',interpolation='nearest', origin='lower',vmin=mn,vmax=mx) ax2.text(0.1,0.9,'MoG',color='white', horizontalalignment='left',verticalalignment='top', transform=ax2.transAxes) ax2.text(0.08,0.08,'{:.3f}'.format(mag_mog),color='white', horizontalalignment='left',verticalalignment='bottom', transform=ax2.transAxes) #ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14) #ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex)) ax3 = plt.subplot2grid((nrows,3*ncols), (irow,icol+2), aspect='equal') ax3.axis('off') #ax3.imshow(stamp-model_psfex, **tim.ima) ax3.imshow(stamp-model_psfex,cmap='gray',interpolation='nearest', origin='lower',vmin=mn,vmax=mx) ax3.text(0.1,0.9,'PSFEx',color='white', horizontalalignment='left',verticalalignment='top', transform=ax3.transAxes) ax3.text(0.08,0.08,'{:.3f}'.format(mag_psfex),color='white', horizontalalignment='left',verticalalignment='bottom', transform=ax3.transAxes) if istar==(nstar-1): break fig.savefig(pngprefix+'-stargrid.png',bbox_inches='tight')
def main(decals=None, opt=None): '''Driver function for forced photometry of individual DECam images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0,x1,y0,y1) = opt.zoom zoomslice = (slice(y0,y1), slice(x0,x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.filename) opt.filename = None except: # make this 'None' for decals.find_ccds() expnum = None # Try parsing HDU number try: opt.hdu = int(opt.hdu) ccdname = None except: ccdname = opt.hdu opt.hdu = -1 if decals is None: decals = Decals() if opt.filename is not None and opt.hdu >= 0: # Read metadata from file T = exposure_metadata([opt.filename], hdus=[opt.hdu]) print('Metadata:') T.about() else: # Read metadata from decals-ccds.fits table T = decals.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if opt.hdu >= 0: T.cut(T.image_hdu == opt.hdu) print(len(T), 'with HDU', opt.hdu) if opt.filename is not None: T.cut(np.array([f.strip() == opt.filename for f in T.image_filename])) print(len(T), 'with filename', opt.filename) assert(len(T) == 1) im = decals.get_image_object(T[0]) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True) print('Got tim:', tim) if opt.catfn in ['DR1', 'DR2']: if opt.catalog_path is None: opt.catalog_path = opt.catfn.lower() margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, decals=decals) for b in bricks: # there is some overlap with this brick... read the catalog. fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3], 'tractor-%s.fits' % b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec) W,H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) * (yy >= -margin) * (yy <= (H+margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') T.cut((T.out_of_bounds == False) * (T.left_blob == False)) print('Cut to', len(T), 'on out_of_bounds and left_blob') TT.append(T) T = merge_tables(TT) T._header = TT[0]._header del TT # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE) # print('Got cat:', cat) print('Forced photom...') opti = None if opt.ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 opti = CeresOptimizer(BW=B, BH=B) tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') for src in cat: src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) F = fits_table() F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)) ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x-1).astype(np.float32) F.y = (y-1).astype(np.float32) if opt.apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() print('Apertures:', apertures, 'pixels') for rad in apertures: aper = photutils.CircularAperture(apxy, rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = ap ap = 1./(np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux_ivar = ap if opt.forced: kwa = {} if opt.plots is None: kwa.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, **kwa) if opt.plots: (data,mod,ie,chi,roi) = R.ims1[0] ima = tim.ima imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5) plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() F.flux = np.array([src.getBrightness().getFlux(tim.band) for src in cat]).astype(np.float32) F.flux_ivar = R.IV.astype(np.float32) F.fracflux = R.fitstats.profracflux.astype(np.float32) F.rchi2 = R.fitstats.prochi2 .astype(np.float32) program_name = sys.argv[0] version_hdr = get_version_header(program_name, decals.decals_dir) # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) fname = os.path.join(d2, d1, fname) print('Trimmed filename to', fname) #version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file')) version_hdr.add_record(dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file')) version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext')) version_hdr.add_record(dict(name='CAMERA', value='DECam', comment='Dark Energy Camera')) version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num')) version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name')) version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record(dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image')) keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV', 'INSTRUME'] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = {'mjd':'sec', 'exptime':'sec', 'flux':'nanomaggy', 'flux_ivar':'1/nanomaggy^2'} columns = F.get_columns() for i,col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) print('Finished forced phot:', Time()-t0) return 0
def main(): decals = Decals() catpattern = 'pipebrick-cats/tractor-phot-b%06i.fits' ra, dec = 242, 7 # Region-of-interest, in pixels: x0, x1, y0, y1 #roi = None roi = [500, 1000, 500, 1000] if roi is not None: x0, x1, y0, y1 = roi #expnum = 346623 #ccdname = 'N12' #chips = decals.find_ccds(expnum=expnum, extname=ccdname) #print 'Found', len(chips), 'chips for expnum', expnum, 'extname', ccdname #if len(chips) != 1: #return False chips = decals.get_ccds() D = np.argsort(np.hypot(chips.ra - ra, chips.dec - dec)) print('Closest chip:', chips[D[0]]) chips = [chips[D[0]]] im = DecamImage(decals, chips[0]) print('Image:', im) targetwcs = Sip(im.wcsfn) if roi is not None: targetwcs = targetwcs.get_subimage(x0, y0, x1 - x0, y1 - y0) r0, r1, d0, d1 = targetwcs.radec_bounds() # ~ 30-pixel margin margin = 2e-3 if r0 > r1: # RA wrap-around TT = [ brick_catalog_for_radec_box(ra, rb, d0 - margin, d1 + margin, decals, catpattern) for (ra, rb) in [(0, r1 + margin), (r0 - margin, 360.)] ] T = merge_tables(TT) T._header = TT[0]._header else: T = brick_catalog_for_radec_box(r0 - margin, r1 + margin, d0 - margin, d1 + margin, decals, catpattern) print('Got', len(T), 'catalog entries within range') cat = read_fits_catalog(T, T._header) print('Got', len(cat), 'catalog objects') print('Switching ellipse parameterizations') switch_to_soft_ellipses(cat) keepcat = [] for src in cat: if not np.all(np.isfinite(src.getParams())): print('Src has infinite params:', src) continue if isinstance(src, FixedCompositeGalaxy): f = src.fracDev.getClippedValue() if f == 0.: src = ExpGalaxy(src.pos, src.brightness, src.shapeExp) elif f == 1.: src = DevGalaxy(src.pos, src.brightness, src.shapeDev) keepcat.append(src) cat = keepcat slc = None if roi is not None: slc = slice(y0, y1), slice(x0, x1) tim = im.get_tractor_image(slc=slc) print('Got', tim) tim.psfex.fitSavedData(*tim.psfex.splinedata) tim.psfex.radius = 20 tim.psf = CachingPsfEx.fromPsfEx(tim.psfex) tractor = Tractor([tim], cat) print('Created', tractor) mod = tractor.getModelImage(0) plt.clf() dimshow(tim.getImage(), **tim.ima) plt.title('Image') plt.savefig('1.png') plt.clf() dimshow(mod, **tim.ima) plt.title('Model') plt.savefig('2.png') ok, x, y = targetwcs.radec2pixelxy([src.getPosition().ra for src in cat], [src.getPosition().dec for src in cat]) ax = plt.axis() plt.plot(x, y, 'rx') #plt.savefig('3.png') plt.axis(ax) plt.title('Sources') plt.savefig('3.png') bands = [im.band] import runbrick runbrick.photoobjdir = '.' scat, T = get_sdss_sources(bands, targetwcs, local=False) print('Got', len(scat), 'SDSS sources in bounds') stractor = Tractor([tim], scat) print('Created', stractor) smod = stractor.getModelImage(0) plt.clf() dimshow(smod, **tim.ima) plt.title('SDSS model') plt.savefig('4.png')
def galex_forcedphot(galex_dir, cat, tiles, band, roiradecbox, pixelized_psf=False, ps=None): ''' Given a list of tractor sources *cat* and a list of GALEX tiles *tiles* (a fits_table with RA,Dec,tilename) runs forced photometry, returning a FITS table the same length as *cat*. ''' from tractor import Tractor from astrometry.util.ttime import Time if False: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('wise-forced-w%i' % band) plots = (ps is not None) if plots: import pylab as plt use_ceres = True wantims = True get_models = True gband = 'galex' phot = fits_table() tims = [] for tile in tiles: info('Reading GALEX tile', tile.visitname.strip(), 'band', band) tim = galex_tractor_image(tile, band, galex_dir, roiradecbox, gband) if tim is None: debug('Actually, no overlap with tile', tile.tilename) continue # if plots: # sig1 = tim.sig1 # plt.clf() # plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', # cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) # plt.colorbar() # tag = '%s W%i' % (tile.tilename, band) # plt.title('%s: tim data' % tag) # ps.savefig() if pixelized_psf: psfimg = galex_psf(band, galex_dir) tim.psf = PixelizedPSF(psfimg) # if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): # mjd[I] = (tim.mjdmin + tim.mjdmax) / 2. # # PSF norm for depth # psf = tim.getPsf() # h,w = tim.shape # patch = psf.getPointSourcePatch(h//2, w//2).patch # psfnorm = np.sqrt(np.sum(patch**2)) # # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags. # psfdepth = 1. / (tim.sig1 / psfnorm)**2 # phot.get(wband + '_psfdepth')[I] = psfdepth tim.tile = tile tims.append(tim) tractor = Tractor(tims, cat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer ceres_block = 8 tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(gband) t0 = Time() R = tractor.optimize_forced_photometry( fitstats=True, variance=True, shared_params=False, wantims=wantims) info('GALEX forced photometry took', Time() - t0) #info('Result:', R) if use_ceres: term = R.ceres_status['termination'] # Running out of memory can cause failure to converge and term # status = 2. Fail completely in this case. if term != 0: info('Ceres termination status:', term) raise RuntimeError('Ceres terminated with status %i' % term) if wantims: ims1 = R.ims1 flux_invvars = R.IV # if plots: # # Create models for just the brightest sources # bright_cat = [src for src in cat # if src.getBrightness().getBand(wanyband) > 1000] # debug('Bright soures:', len(bright_cat)) # btr = Tractor(tims, bright_cat) # for tim in tims: # mod = btr.getModelImage(tim) # tile = tim.tile # tag = '%s W%i' % (tile.tilename, band) # sig1 = tim.sig1 # plt.clf() # plt.imshow(mod, interpolation='nearest', origin='lower', # cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) # plt.colorbar() # plt.title('%s: bright-star models' % tag) # ps.savefig() if get_models: models = [] for i,tim in enumerate(tims): tile = tim.tile (dat, mod, ie, _, _) = ims1[i] models.append((tile.visitname, band, tim.wcs.wcs, dat, mod, ie)) if plots: for i,tim in enumerate(tims): tile = tim.tile tag = '%s %s' % (tile.tilename, band) (dat, mod, _, chi, _) = ims1[i] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() nm = np.array([src.getBrightness().getBand(gband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their default # (1-sigma or whatever) initial fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. niceband = band + 'uv' phot.set('flux_' + niceband, nm.astype(np.float32)) phot.set('flux_ivar_' + niceband, nm_ivar.astype(np.float32)) #phot.set(band + '_mjd', mjd) rtn = gphotduck() rtn.phot = phot rtn.models = None if get_models: rtn.models = models return rtn
def scan_dchisq(seeing, target_dchisq, ps, e1=0.): pixscale = 0.262 psfsigma = seeing / pixscale / 2.35 print('PSF sigma:', psfsigma, 'pixels') psf = GaussianMixturePSF(1., 0., 0., psfsigma**2, psfsigma**2, 0.) sig1 = 0.01 psfnorm = 1./(2. * np.sqrt(np.pi) * psfsigma) detsig1 = sig1 / psfnorm sz = 50 cd = pixscale / 3600. wcs = Tan(0., 0., float(sz/2), float(sz/2), -cd, 0., 0., cd, float(sz), float(sz)) band = 'r' tim = Image(data=np.zeros((sz,sz)), inverr=np.ones((sz,sz)) / sig1, psf=psf, wcs = ConstantFitsWcs(wcs), photocal = LinearPhotoCal(1., band=band)) re_vals = np.logspace(-1., 0., 50) all_runs = [] mods = [] for i,re in enumerate(re_vals): true_src = ExpGalaxy(RaDecPos(0., 0.), NanoMaggies(**{band: 1.}), EllipseE(re, e1, 0.)) print('True source:', true_src) tr = Tractor([tim], [true_src]) tr.freezeParams('images') true_mod = tr.getModelImage(0) dchisq_none = np.sum((true_mod * tim.inverr)**2) scale = np.sqrt(target_dchisq / dchisq_none) true_src.brightness.setParams([scale]) true_mod = tr.getModelImage(0) dchisq_none = np.sum((true_mod * tim.inverr)**2) mods.append(true_mod) tim.data = true_mod exp_src = true_src.copy() psf_src = PointSource(true_src.pos.copy(), true_src.brightness.copy()) simp_src = SimpleGalaxy(true_src.pos.copy(), true_src.brightness.copy()) dchisqs = [] #for src in [psf_src, simp_src, exp_src]: for src in [psf_src, simp_src]: src.freezeParam('pos') #print('Fitting source:', src) #src.printThawedParams() tr.catalog[0] = src tr.optimize_loop() #print('Fitted:', src) mod = tr.getModelImage(0) dchisqs.append(dchisq_none - np.sum(((true_mod - mod) * tim.inverr)**2)) #print('dchisq:', dchisqs[-1]) dchisqs.append(dchisq_none) all_runs.append([re,] + dchisqs) all_runs = np.array(all_runs) re = all_runs[:,0] dchi_psf = all_runs[:,1] dchi_simp = all_runs[:,2] dchi_exp = all_runs[:,3] dchi_ps = np.maximum(dchi_psf, dchi_simp) dchi_cut1 = dchi_ps + 3+9 dchi_cut2 = dchi_ps + dchi_psf * 0.02 dchi_cut3 = dchi_ps + dchi_psf * 0.008 plt.clf() plt.plot(re, dchi_psf, 'k-', label='PSF') plt.plot(re, dchi_simp, 'b-', label='SIMP') plt.plot(re, dchi_exp, 'r-', label='EXP') plt.plot(re, dchi_cut2, 'm--', alpha=0.5, lw=2, label='Cut: 2%') plt.plot(re, dchi_cut3, 'm:', alpha=0.5, lw=2, label='Cut: 0.08%') plt.plot(re, dchi_cut1, 'm-', alpha=0.5, lw=2, label='Cut: 12') plt.xlabel('True r_e (arcsec)') plt.ylabel('dchisq') #plt.legend(loc='lower left') plt.legend(loc='upper right') tt = 'Seeing = %g arcsec, S/N ~ %i' % (seeing, int(np.round(np.sqrt(target_dchisq)))) if e1 != 0.: tt += ', Ellipticity %g' % e1 plt.title(tt) plt.ylim(0.90 * target_dchisq, 1.05 * target_dchisq) # aspect = 1.2 # ax = plt.axis() # dre = (ax[1]-ax[0]) / 20 / aspect # dchi = (ax[3]-ax[2]) / 20 # I = np.linspace(0, len(re_vals)-1, 8).astype(int) # for mod,re in [(mods[i], re_vals[i]) for i in I]: # print('extent:', [re-dre, re+dre, ax[2], ax[2]+dchi]) # plt.imshow(mod, interpolation='nearest', origin='lower', aspect='auto', # extent=[re-dre, re+dre, ax[2], ax[2]+dchi], cmap='gray') # plt.axis(ax) ps.savefig()
psf=psf, wcs = ConstantFitsWcs(wcs), photocal = LinearPhotoCal(1., band=band)) for i,sn in enumerate(sn_vals): for j,re in enumerate(re_vals): ## HACK -- this is the flux required for a PSF to be ## detected at target S/N... adjust for galaxy? flux = sn * detsig1 # Create round EXP galaxy #PixPos(sz/2, sz/2), true_src = ExpGalaxy(RaDecPos(0., 0.), NanoMaggies(**{band: flux}), EllipseE(re, 0., 0.)) tr = Tractor([tim], [true_src]) tr.freezeParams('images') true_mod = tr.getModelImage(0) ima = dict(interpolation='nearest', origin='lower', vmin=-2.*sig1, vmax=5.*sig1, cmap='hot') this_dchisqs = [] flux_sns = [] for k in range(Nper): noise = np.random.normal(scale=sig1, size=true_mod.shape) tim.data = true_mod + noise if k == 0 and False:
from tractor import Image, GaussianMixturePSF, LinearPhotoCal from tractor import PixPos, Flux, EllipseE, Tractor, ModelMask import pylab as plt # Create a Tractor Image that works in pixel space (WCS not specified). tim = Image(data=np.zeros((h,w)), inverr=np.ones((h,w)), psf=GaussianMixturePSF(1., 0., 0., 3., 3., 0.), photocal=LinearPhotoCal(1.)) # Create a plain Exp galaxy to generate a postage stamp that we'll try to fit with # the MogGalaxy model. gal = ExpGalaxy(PixPos(w//2, h//2), Flux(1000.), EllipseE(10., 0.5, 0.3)) # Get the model tractor = Tractor([tim], [gal]) mod = tractor.getModelImage(0) #mog = gal._getAffineProfile(tim, w//2, h//2) #print('Exp galaxy profile:', str(mog)) # Plot the model plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower') plt.savefig('mod.png') # Set the tractor Image to the Exp model postage stamp -- this is what we'll try to fit. tim.data = mod # Initialize the MoG components amp = np.array([0.4, 0.3, 0.3])
def main(decals=None, opt=None): '''Driver function for forced photometry of individual DECam images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0, x1, y0, y1) = opt.zoom zoomslice = (slice(y0, y1), slice(x0, x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.filename) opt.filename = None except: # make this 'None' for decals.find_ccds() expnum = None # Try parsing HDU number try: opt.hdu = int(opt.hdu) ccdname = None except: ccdname = opt.hdu opt.hdu = -1 if decals is None: decals = Decals() if opt.filename is not None and opt.hdu >= 0: # Read metadata from file T = exposure_metadata([opt.filename], hdus=[opt.hdu]) print('Metadata:') T.about() else: # Read metadata from decals-ccds.fits table T = decals.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if opt.hdu >= 0: T.cut(T.image_hdu == opt.hdu) print(len(T), 'with HDU', opt.hdu) if opt.filename is not None: T.cut( np.array([f.strip() == opt.filename for f in T.image_filename])) print(len(T), 'with filename', opt.filename) assert (len(T) == 1) im = decals.get_image_object(T[0]) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True) print('Got tim:', tim) if opt.catfn in ['DR1', 'DR2']: if opt.catalog_path is None: opt.catalog_path = opt.catfn.lower() margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, decals=decals) for b in bricks: # there is some overlap with this brick... read the catalog. fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3], 'tractor-%s.fits' % b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec) W, H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) * (yy >= -margin) * (yy <= (H + margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') T.cut((T.out_of_bounds == False) * (T.left_blob == False)) print('Cut to', len(T), 'on out_of_bounds and left_blob') TT.append(T) T = merge_tables(TT) T._header = TT[0]._header del TT # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE) # print('Got cat:', cat) print('Forced photom...') opti = None if opt.ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 opti = CeresOptimizer(BW=B, BH=B) tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') for src in cat: src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) F = fits_table() F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)) ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x - 1).astype(np.float32) F.y = (y - 1).astype(np.float32) if opt.apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack( [tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() print('Apertures:', apertures, 'pixels') for rad in apertures: aper = photutils.CircularAperture(apxy, rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = ap ap = 1. / (np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux_ivar = ap if opt.forced: kwa = {} if opt.plots is None: kwa.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, **kwa) if opt.plots: (data, mod, ie, chi, roi) = R.ims1[0] ima = tim.ima imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5) plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() F.flux = np.array([ src.getBrightness().getFlux(tim.band) for src in cat ]).astype(np.float32) F.flux_ivar = R.IV.astype(np.float32) F.fracflux = R.fitstats.profracflux.astype(np.float32) F.rchi2 = R.fitstats.prochi2.astype(np.float32) program_name = sys.argv[0] version_hdr = get_version_header(program_name, decals.decals_dir) # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) fname = os.path.join(d2, d1, fname) print('Trimmed filename to', fname) #version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file')) version_hdr.add_record( dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file')) version_hdr.add_record( dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext')) version_hdr.add_record( dict(name='CAMERA', value='DECam', comment='Dark Energy Camera')) version_hdr.add_record( dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num')) version_hdr.add_record( dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name')) version_hdr.add_record( dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record( dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image')) keys = [ 'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME' ] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = { 'mjd': 'sec', 'exptime': 'sec', 'flux': 'nanomaggy', 'flux_ivar': '1/nanomaggy^2' } columns = F.get_columns() for i, col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) print('Finished forced phot:', Time() - t0) return 0
photocal=LinearPhotoCal(1., 'w1')) tim2 = Image(data=w2, inverr=ie2, psf=NCircularGaussianPSF([1.],[1.]), photocal=LinearPhotoCal(1., 'w2')) # ExpGalaxy at pixel (423.64, 444.09) with Fluxes: w1=9.06254e+08, w2=1.02335e+09 and Galaxy Shape: re=400.31, ab=0.38, phi=89.8 cx,cy = 423.64, 444.09 flux1 = 9.06e8 flux2 = 1.02e9 #shape = GalaxyShape(400.31, 0.38, 89.8) shape = EllipseESoft.fromRAbPhi(400.31, 0.38, 89.8) gal = ExpGalaxy(PixPos(cx, cy), Fluxes(w1=flux1, w2=flux2), shape) tractor = Tractor([tim1, tim2],[gal]) tractor.freezeParam('images') # Create emcee sampler nw = 30 ndim = len(tractor.getParams()) print('N dim:', ndim) sampler = emcee.EnsembleSampler(nw, ndim, tractor) p0 = tractor.getParams() #std = tractor.getStepSizes() std = np.array([1.0, 1.0, 1e7, 1e7, 0.01, 0.01, 0.01]) pp = emcee.utils.sample_ball(p0, std, size=nw) print('Fitting params: (%i):' % len(p0))
GalaxyShape(5., 0.5, 45.)) gal2 = DevGalaxy(RaDecPos(ra + 10 * pscale, dec), brightness(50.), GalaxyShape(5., 0.25, 135.)) gal3 = FixedCompositeGalaxy( RaDecPos(ra, dec + 10 * pscale), brightness(50.), 0.6, # fraction of light in deV component GalaxyShape(5., 0.6, 30.), GalaxyShape(5., 0.3, 45.)) cat = Catalog(ptsrc, gal1, gal2, gal3) # Create Tractor object tr = Tractor([tim], cat) # Evaluate likelihood lnp = tr.getLogProb() print('Logprob:', lnp) for nm, val in zip(tr.getParamNames(), tr.getParams()): print(' ', nm, val) # Or, without creating a Tractor object: model = np.zeros_like(data) sky.addTo(model) for src in cat: patch = src.getModelPatch(tim) if patch is None:
ps.savefig() subtim = tim.subimage(x0, x1, y0, y1) #mn,mx = np.percentile(subtim.getImage().ravel(), [25, 98]) # plt.clf() # plt.imshow(subtim.getImage(), interpolation='nearest', origin='lower', # cmap='gray', vmin=mn, vmax=mx) # ps.savefig() cat.cut((cat.xx >= x0) * (cat.xx <= x1) * (cat.yy >= y0) * (cat.yy <= y1)) print('Cut to', len(cat), 'sources in subimage') srcs = gim2d_catalog(cat, band) tractor = Tractor([subtim], srcs) if False: plt.clf() plt.imshow(img, **ima) ax = plt.axis() I = np.flatnonzero(cat.type == 1) plt.plot(cat.xx[I], cat.yy[I], 'r.') I = np.flatnonzero(cat.type == 2) plt.plot(cat.xx[I], cat.yy[I], 'b.') I = np.flatnonzero(cat.type == 3) plt.plot(cat.xx[I], cat.yy[I], 'm.') plt.axis(ax) ps.savefig() plt.clf()
def main(): ''' This function generates the plots in the paper. Some files and directories are assumed to exist in the current directory: * WISE atlas tiles, from http://unwise.me/data/allsky-atlas.fits * unwise-neo1-coadds, from http://unwise.me/data/neo1/ * unwise-neo1-coadds-half, unwise-neo1-coadds-quarter: directories ''' # First, create the WCS into which we want to render # degrees width to render in galactic coords # |l| < 60 # |b| < 30 width = 120 # ~2 arcmin per pixel W = int(width * 60.) / 2 H = W/2 zoom = 360. / width wcs = anwcs_create_hammer_aitoff(0., 0., zoom, W, H, 0) # Select WISE tiles that overlap. This atlas table is available # from http://unwise.me/data/allsky-atlas.fits # Select WISE tiles that overlap. T = fits_table('allsky-atlas.fits') print(len(T), 'tiles total') T.ll,T.bb = radectolb(T.ra, T.dec) I = np.flatnonzero(np.logical_or(T.ll < width+1, T.ll > (360-width-1)) * (T.bb > -width/2-1) * (T.bb < width/2+1)) T.cut(I) print(len(I), 'tiles in L,B range') # Create a coadd for each WISE band lbpat = 'unwise-neo1-w%i-lb.fits' imgs = [] for band in [1,2]: outfn = lbpat % (band) if os.path.exists(outfn): print('Exists:', outfn) img = fitsio.read(outfn) imgs.append(img) continue coimg = np.zeros((H,W), np.float32) conimg = np.zeros((H,W), np.float32) for i,brick in enumerate(T.coadd_id): # We downsample by 2, twice, just to make repeat runs a # little faster. # unWISE fn = os.path.join('unwise-neo1-coadds', brick[:3], brick, 'unwise-%s-w%i-img-u.fits' % (brick, band)) qfn = os.path.join('unwise-neo1-coadds-quarter', 'unwise-%s-w%i.fits' % (brick, band)) hfn = os.path.join('unwise-neo1-coadds-half', 'unwise-%s-w%i.fits' % (brick, band)) if not os.path.exists(qfn): if not os.path.exists(hfn): print('Reading', fn) halfsize(fn, hfn) halfsize(hfn, qfn) fn = qfn print('Reading', fn) img = fitsio.read(fn) bwcs = Tan(fn, 0) bh,bw = img.shape # Coadd each unWISE pixel into the nearest target pixel. xx,yy = np.meshgrid(np.arange(bw), np.arange(bh)) rr,dd = bwcs.pixelxy2radec(xx, yy) ll,bb = radectolb(rr.ravel(), dd.ravel()) ll = ll.reshape(rr.shape) bb = bb.reshape(rr.shape) ok,ox,oy = wcs.radec2pixelxy(ll, bb) ox = np.round(ox - 1).astype(int) oy = np.round(oy - 1).astype(int) K = (ox >= 0) * (ox < W) * (oy >= 0) * (oy < H) * ok #print('ok:', np.unique(ok), 'x', ox.min(), ox.max(), 'y', oy.min(), oy.max()) assert(np.all(np.isfinite(img))) if np.sum(K) == 0: # no overlap print('No overlap') continue np.add.at( coimg, (oy[K], ox[K]), img[K]) np.add.at(conimg, (oy[K], ox[K]), 1) img = coimg / np.maximum(conimg, 1) # Hack -- write and then read FITS WCS header. fn = 'wiselb.wcs' wcs.writeto(fn) hdr = fitsio.read_header(fn) hdr['CTYPE1'] = 'GLON-AIT' hdr['CTYPE2'] = 'GLAT-AIT' fitsio.write(outfn, img, header=hdr, clobber=True) fitsio.write(outfn.replace('.fits', '-n.fits'), conimg, header=hdr, clobber=True) imgs.append(img) w1,w2 = imgs # Get/confirm L,B bounds... H,W = w1.shape print('Image size', W, 'x', H) ok,l1,b1 = wcs.pixelxy2radec(1, (H+1)/2.) ok,l2,b2 = wcs.pixelxy2radec(W, (H+1)/2.) ok,l3,b3 = wcs.pixelxy2radec((W+1)/2., 1) ok,l4,b4 = wcs.pixelxy2radec((W+1)/2., H) print('L,B', (l1,b1), (l2,b2), (l3,b3), (l4,b4)) llo,lhi = l2,l1+360 blo,bhi = b3,b4 # Set plot sizes plt.figure(1, figsize=(10,5)) plt.subplots_adjust(left=0.1, right=0.95, bottom=0.1, top=0.95) plt.figure(2, figsize=(5,5)) plt.subplots_adjust(left=0.11, right=0.96, bottom=0.1, top=0.95) suffix = '.pdf' rgb = wise_rgb(w1, w2) xlo,ylo = 0,0 plt.figure(1) plt.clf() plt.imshow(rgb, origin='lower', interpolation='nearest') lbticks(wcs, xlo, ylo, lticks=[60,30,0,330,300], bticks=[-30,-15,0,15,30]) plt.savefig('xbulge-00' + suffix) # Compute the median of each row as a crude way of suppressing the # Galactic plane medy1 = np.median(w1, axis=1) medy2 = np.median(w2, axis=1) rgb = wise_rgb(w1 - medy1[:,np.newaxis], w2 - medy2[:,np.newaxis]) # Zoom in a bit for Galactic plane subtracted version lhi,llo,blo,bhi = 40, 320, -20, 20 okxy = np.array([wcs.radec2pixelxy(l,b) for l,b in [ (llo, blo), (llo, bhi), (lhi, blo), (lhi, bhi)]]) xlo = int(np.floor(min(okxy[:,-2]))) xhi = int(np.ceil (max(okxy[:,-2]))) ylo = int(np.floor(min(okxy[:,-1]))) yhi = int(np.ceil (max(okxy[:,-1]))) plt.clf() plt.imshow(rgb[ylo:yhi, xlo:xhi, :],origin='lower', interpolation='nearest') #lbticks(wcs, xlo, ylo, lticks=[40,20,0,340,320], bticks=[-20,-10,0,10,20]) lbticks(wcs, xlo, ylo, lticks=[30,15,0,345,330], bticks=[-20,-10,0,10,20]) plt.savefig('xbulge-01' + suffix) # Zoom in on the core lhi,llo,blo,bhi = 15, 345, -15, 15 ok,x1,y1 = wcs.radec2pixelxy(llo, blo) ok,x2,y2 = wcs.radec2pixelxy(llo, bhi) ok,x3,y3 = wcs.radec2pixelxy(lhi, blo) ok,x4,y4 = wcs.radec2pixelxy(lhi, bhi) xlo = int(np.floor(min(x1,x2,x3,x4))) xhi = int(np.ceil (max(x1,x2,x3,x4))) ylo = int(np.floor(min(y1,y2,y3,y4))) yhi = int(np.ceil (max(y1,y2,y3,y4))) print('xlo,ylo', xlo, ylo) w1 = w1[ylo:yhi, xlo:xhi] w2 = w2[ylo:yhi, xlo:xhi] plt.figure(2) # Apply color cut w1mag = -2.5*(np.log10(w1) - 9.) w2mag = -2.5*(np.log10(w2) - 9.) cc = w1mag - w2mag goodcolor = np.isfinite(cc) mlo,mhi = np.percentile(cc[goodcolor], [5,95]) print('W1 - W2 color masks:', mlo,mhi) mask = goodcolor * (cc > mlo) * (cc < mhi) plt.clf() rgb = wise_rgb(w1, w2) plt.imshow(rgb, origin='lower', interpolation='nearest') lbticks(wcs, xlo,ylo) plt.title('Data') plt.savefig('xbulge-fit-data' + suffix) plt.clf() rgb = wise_rgb(w1 * mask, w2 * mask) plt.imshow(rgb, origin='lower', interpolation='nearest') lbticks(wcs, xlo,ylo) plt.title('Data (masked)') plt.savefig('xbulge-fit-masked' + suffix) ie = mask.astype(np.float32) from tractor import (Image, NCircularGaussianPSF, LinearPhotoCal, Tractor, PixPos, Fluxes) from tractor.galaxy import ExpGalaxy, GalaxyShape # Create Tractor images tim1 = Image(data=w1 * mask, inverr=ie, psf=NCircularGaussianPSF([1.],[1.]), photocal=LinearPhotoCal(1., 'w1')) tim2 = Image(data=w2 * mask, inverr=ie, psf=NCircularGaussianPSF([1.],[1.]), photocal=LinearPhotoCal(1., 'w2')) H,W = w1.shape gal = ExpGalaxy(PixPos(W/2, H/2), Fluxes(w1=w1.sum(), w2=w2.sum()), GalaxyShape(200, 0.4, 90.)) tractor = Tractor([tim1, tim2],[gal]) # fitsio.write('data-w1.fits', w1 * mask, clobber=True) # fitsio.write('data-w2.fits', w2 * mask, clobber=True) # fitsio.write('mask.fits', mask.astype(np.uint8), clobber=True) # Optimize galaxy model tractor.freezeParam('images') for step in range(50): dlnp,x,alpha = tractor.optimize() print('dlnp', dlnp) print('x', x) print('alpha', alpha) print('Galaxy', gal) if dlnp == 0: break # Get galaxy model images, compute residuals mod1 = tractor.getModelImage(0) resid1 = w1 - mod1 mod2 = tractor.getModelImage(1) resid2 = w2 - mod2 rgb = wise_rgb(mod1, mod2) plt.clf() plt.imshow(rgb, origin='lower', interpolation='nearest') lbticks(wcs, xlo,ylo) plt.title('Model') plt.savefig('xbulge-fit-model' + suffix) rgb = resid_rgb(resid1, resid2) plt.clf() plt.imshow(rgb, origin='lower', interpolation='nearest') lbticks(wcs, xlo,ylo) plt.title('Residuals') plt.savefig('xbulge-fit-resid' + suffix) rgb = resid_rgb(resid1*mask, resid2*mask) plt.clf() plt.imshow(rgb, origin='lower', interpolation='nearest') lbticks(wcs, xlo,ylo) plt.title('Residuals (masked)') plt.savefig('xbulge-fit-residmasked' + suffix) # fitsio.write('resid1.fits', resid1, clobber=True) # fitsio.write('resid2.fits', resid2, clobber=True) # Compute median-smoothed residuals fr1 = np.zeros_like(resid1) fr2 = np.zeros_like(resid2) median_smooth(resid1, np.logical_not(mask), 25, fr1) median_smooth(resid2, np.logical_not(mask), 25, fr2) rgb = resid_rgb(fr1, fr2) plt.clf() plt.imshow(rgb, origin='lower', interpolation='nearest') lbticks(wcs, xlo,ylo) plt.title('Residuals (smoothed)') plt.savefig('xbulge-fit-smooth2' + suffix)
def main(): fn = '/global/cscratch1/sd/dstn/c4d_190730_024955_ori/c4d_190730_024955_ori.52.fits' survey_dir = '/global/cscratch1/sd/dstn/subtractor-survey-dir' imagedir = os.path.join(survey_dir, 'images') trymakedirs(imagedir) calibdir = os.path.join(survey_dir, 'calib') psfexdir = os.path.join(calibdir, 'decam', 'psfex-merged') trymakedirs(psfexdir) skydir = os.path.join(calibdir, 'decam', 'splinesky-merged') trymakedirs(skydir) basename = os.path.basename(fn) basename = basename.replace('.fits', '') # Output filenames for legacyzpts calibration/zeropoint files f, photfn = tempfile.mkstemp() os.close(f) surveyfn = os.path.join(survey_dir, 'survey-ccds-%s.fits.gz' % basename) annfn = os.path.join(survey_dir, 'annotated-%s.fits' % basename) mp = multiproc() survey = LegacySurveyData(survey_dir) # Use the subclass above to handle DECam images! survey.image_typemap.update(decam=GoldsteinDecamImage) # Grab the exposure number and CCD name hdr = fitsio.read_header(fn) expnum = hdr['EXPNUM'] ccdname = hdr['EXTNAME'].strip() print('Exposure', expnum, 'CCD', ccdname) import logging lvl = logging.INFO logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) # tractor logging is *soooo* chatty logging.getLogger('tractor.engine').setLevel(lvl + 10) if not os.path.exists(surveyfn): # Run calibrations and zeropoints runit(fn, photfn, surveyfn, annfn, mp, survey=survey, camera='decam', debug=False, choose_ccd=ccdname, splinesky=True, calibdir=calibdir, measureclass=GoldsteinDecamMeasurer) # Find catalog sources touching this CCD ccds = survey.find_ccds(expnum=expnum, ccdname=ccdname) assert (len(ccds) == 1) ccd = ccds[0] print('Got CCD', ccd) # Create Tractor image im = survey.get_image_object(ccd) print('Got image:', im) # Look at this sub-image, or the whole chip? #zoomslice=None zoomslice = (slice(0, 1000), slice(0, 1000)) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True, hybridPsf=True, normalizePsf=True, old_calibs_ok=True) print('Got tim:', tim) # Read catalog files touching this CCD catsurvey = LegacySurveyData( '/global/project/projectdirs/cosmo/work/legacysurvey/dr8/south') T = get_catalog_in_wcs(tim.subwcs, catsurvey) print('Got', len(T), 'DR8 catalog sources within CCD') # Gaia stars: move RA,Dec to the epoch of this image. I = np.flatnonzero(T.ref_epoch > 0) if len(I): from legacypipe.survey import radec_at_mjd print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd()) ra, dec = radec_at_mjd(T.ra[I], T.dec[I], T.ref_epoch[I].astype(float), T.pmra[I], T.pmdec[I], T.parallax[I], tim.time.toMjd()) T.ra[I] = ra T.dec[I] = dec # Create Tractor Source objects from the catalog cat = read_fits_catalog(T, bands=tim.band) print('Created', len(cat), 'source objects') # Render model image! tr = Tractor([tim], cat) mod = tr.getModelImage(0) # plots ima = dict(interpolation='nearest', origin='lower', vmin=-2 * tim.sig1, vmax=10 * tim.sig1, cmap='gray') plt.clf() plt.imshow(tim.getImage(), **ima) plt.title('Image') plt.savefig('img.jpg') plt.clf() plt.imshow(mod, **ima) plt.title('Model') plt.savefig('mod.jpg') plt.clf() plt.imshow(tim.getImage() - mod, **ima) plt.title('Residual') plt.savefig('res.jpg')
def main(): # Where are the data? datadir = os.path.join(os.path.dirname(__file__), 'data-decam') name = 'decam-520206-S16' imagefn = os.path.join(datadir, '%s-image-sub.fits' % name) invvarfn = os.path.join(datadir, '%s-invvar-sub.fits' % name) psfexfn = os.path.join(datadir, '%s-psfex.fits' % name) catfn = os.path.join(datadir, 'tractor-1816p325-sub.fits') # Read the image and inverse-variance maps. image = fitsio.read(imagefn) invvar = fitsio.read(invvarfn) # The DECam inverse-variance maps are unfortunately corrupted # by fpack, causing zeros to become negative. Fix those. invvar[invvar < np.median(invvar) * 0.1] = 0. H, W = image.shape print('Subimage size:', image.shape) # For the PSF model, we need to know what subimage region this is: subimage_offset = (35, 1465) # We also need the calibrated zeropoint. zeropoint = 24.7787 # What filter was this image taken in? (z) prim_header = fitsio.read_header(imagefn) band = prim_header['FILTER'].strip()[0] print('Band:', band) # These DECam images were calibrated so that the zeropoints need # an exposure-time factor, so add that in. exptime = prim_header['EXPTIME'] zeropoint += 2.5 * np.log10(exptime) # Read the PsfEx model file psf = PixelizedPsfEx(psfexfn) # Instantiate a constant pixelized PSF at the image center # (of the subimage) x0, y0 = subimage_offset psf = psf.constantPsfAt(x0 + W / 2., y0 + H / 2.) # Load the WCS model from the header # We convert from the RA---TPV type to RA---SIP header = fitsio.read_header(imagefn, ext=1) wcsobj = wcs_pv2sip_hdr(header, stepsize=10) # We'll just use a rough sky estimate... skyval = np.median(image) # Create the Tractor Image (tim). tim = Image(data=image, invvar=invvar, psf=psf, wcs=ConstantFitsWcs(wcsobj), sky=ConstantSky(skyval), photocal=LinearPhotoCal( NanoMaggies.zeropointToScale(zeropoint), band=band)) # Read the official DECaLS DR3 catalog -- it has only two sources in this subimage. catalog = fits_table(catfn) print('Read', len(catalog), 'sources') print('Source types:', catalog.type) # Create Tractor sources corresponding to these two catalog # entries. # In DECaLS, the "SIMP" type is a round Exponential galaxy with a # fixed 0.45" radius, but we'll treat it as a general Exp galaxy. sources = [] for c in catalog: # Create a "position" object given the catalog RA,Dec position = RaDecPos(c.ra, c.dec) # Create a "brightness" object; in the catalog, the fluxes are # stored in a [ugrizY] array, so pull out the right index band_index = 'ugrizY'.index(band) flux = c.decam_flux[band_index] brightness = NanoMaggies(**{band: flux}) # Depending on the source classification in the catalog, pull # out different fields for the galaxy shape, and for the # galaxy type. The DECaLS catalogs, conveniently, store # galaxy shapes as (radius, e1, e2) ellipses. if c.type.strip() == 'DEV': shape = EllipseE(c.shapedev_r, c.shapedev_e1, c.shapedev_e2) galclass = DevGalaxy elif c.type.strip() == 'SIMP': shape = EllipseE(c.shapeexp_r, c.shapeexp_e1, c.shapeexp_e2) galclass = ExpGalaxy else: assert (False) # Create the tractor galaxy object source = galclass(position, brightness, shape) print('Created', source) sources.append(source) # Create the Tractor object -- a list of tractor Images and a list of tractor sources. tractor = Tractor([tim], sources) # Render the initial model image. print('Getting initial model...') mod = tractor.getModelImage(0) make_plot(tim, mod, 'Initial Scene', 'mod0.png') # Instantiate a new source at the location of the unmodelled peak. print('Adding new source...') # Find the peak very naively... ipeak = np.argmax((image - mod) * tim.inverr) iy, ix = np.unravel_index(ipeak, tim.shape) print('Residual peak at', ix, iy) # Compute the RA,Dec location of the peak... radec = tim.getWcs().pixelToPosition(ix, iy) print('RA,Dec', radec) # Try modelling it as a point source. # We'll initialize the brightness arbitrarily to 1 nanomaggy (= mag 22.5) brightness = NanoMaggies(**{band: 1.}) source = PointSource(radec, brightness) # Add it to the catalog! tractor.catalog.append(source) # Render the new model image with this source added. mod = tractor.getModelImage(0) make_plot(tim, mod, 'New Source (Before Fit)', 'mod1.png') print('Fitting new source...') # Now we're going to fit for the properties of the new source we # added. # We don't want to fit for any of the image calibration properties: tractor.freezeParam('images') # And we don't (yet) want to fit the existing sources. The new # source is index number 2, so freeze everything else in the catalog. tractor.catalog.freezeAllBut(2) print('Fitting parameters:') tractor.printThawedParams() # Do the actual optimization: tractor.optimize_loop() mod = tractor.getModelImage(0) make_plot(tim, mod, 'New Source Fit', 'mod2.png') print('Fitting sources simultaneously...') # Now let's unfreeze all the sources and fit them simultaneously. tractor.catalog.thawAllParams() tractor.printThawedParams() tractor.optimize_loop() mod = tractor.getModelImage(0) make_plot(tim, mod, 'Simultaneous Fit', 'mod3.png')
def main(): # I read a DESI DR8 target catalog, cut to ELGs, then took a narrow # r-mag slice around the peak of that distribution; # desi/target/catalogs/dr8/0.31.1/targets/main/resolve/targets-dr8-hp-1,5,11,50,55,60,83,84,86,89,91,98,119,155,158,174,186,187,190.fits') # Then took the median g and z mags # And opened the coadd invvar mags for a random brick in that footprint # (0701p000) to find the median per-pixel invvars. r = 23.0 g = 23.4 z = 22.2 # Selecting EXPs, the peak of the shapeexp_r was ~ 0.75". r_e = 0.75 # Image properties: giv = 77000. riv = 27000. ziv = 8000. # PSF sizes were roughly equal, 1.16, 1.17, 1.19" # -> sigma = 1.9 DECam pixels psf_sigma = 1.9 H, W = 1000, 1000 seed = 42 np.random.seed(seed) ra, dec = 70., 1. brick = BrickDuck(ra, dec, 'custom-0700p010') wcs = wcs_for_brick(brick, W=W, H=H) bands = 'grz' tims = [] for band, iv in zip(bands, [giv, riv, ziv]): img = np.zeros((H, W), np.float32) tiv = np.zeros_like(img) + iv s = psf_sigma**2 psf = GaussianMixturePSF(1., 0., 0., s, s, 0.) twcs = ConstantFitsWcs(wcs) sky = ConstantSky(0.) photocal = LinearPhotoCal(1., band=band) tai = TAITime(None, mjd=55700.) tim = tractor.Image(data=img, invvar=tiv, psf=psf, wcs=twcs, sky=sky, photocal=photocal, name='fake %s' % band, time=tai) tim.skyver = ('1', '1') tim.psfver = ('1', '1') tim.plver = '1' tim.x0 = tim.y0 = 0 tim.subwcs = wcs tim.psfnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma) tim.galnorm = tim.psfnorm tim.propid = '2020A-000' tim.band = band tim.dq = None tim.sig1 = 1. / np.sqrt(iv) tim.psf_sigma = psf_sigma tim.primhdr = fitsio.FITSHDR() tims.append(tim) # Simulated catalog gflux = NanoMaggies.magToNanomaggies(g) rflux = NanoMaggies.magToNanomaggies(r) zflux = NanoMaggies.magToNanomaggies(z) box = 50 CX, CY = np.meshgrid(np.arange(box // 2, W, box), np.arange(box // 2, H, box)) ny, nx = CX.shape BA, PHI = np.meshgrid(np.linspace(0.1, 1.0, nx), np.linspace(0., 180., ny)) cat = [] for cx, cy, ba, phi in zip(CX.ravel(), CY.ravel(), BA.ravel(), PHI.ravel()): #print('x,y %.1f,%.1f, ba %.2f, phi %.2f' % (cx, cy, ba, phi)) r, d = wcs.pixelxy2radec(cx + 1, cy + 1) src = ExpGalaxy(RaDecPos(r, d), NanoMaggies(order=bands, g=gflux, r=rflux, z=zflux), EllipseE.fromRAbPhi(r_e, ba, phi)) cat.append(src) from legacypipe.catalog import prepare_fits_catalog TT = fits_table() TT.bx = CX.ravel() TT.by = CY.ravel() TT.ba = BA.ravel() TT.phi = PHI.ravel() tcat = tractor.Catalog(*cat) T2 = prepare_fits_catalog(tcat, None, TT, bands, save_invvars=False) T2.writeto('sim-cat.fits') tr = Tractor(tims, cat) for band, tim in zip(bands, tims): mod = tr.getModelImage(tim) mod += np.random.standard_normal( size=tim.shape) * 1. / tim.getInvError() fitsio.write('sim-%s.fits' % band, mod, clobber=True) tim.data = mod ccds = fits_table() ccds.filter = np.array([f for f in bands]) ccds.ccd_cuts = np.zeros(len(ccds), np.int16) ccds.imgfn = np.array([tim.name for tim in tims]) ccds.propid = np.array(['2020A-000'] * len(ccds)) ccds.fwhm = np.zeros(len(ccds), np.float32) + psf_sigma * 2.35 ccds.mjd_obs = np.zeros(len(ccds)) ccds.camera = np.array(['fake'] * len(ccds)) survey = FakeLegacySurvey(ccds, tims) import logging verbose = False if verbose == 0: lvl = logging.INFO else: lvl = logging.DEBUG logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) # tractor logging is *soooo* chatty logging.getLogger('tractor.engine').setLevel(lvl + 10) run_brick(None, survey, radec=(ra, dec), width=W, height=H, do_calibs=False, gaia_stars=False, large_galaxies=False, tycho_stars=False, forceall=True, outliers=False) #, stages=['image_coadds'])
def unwise_forcedphot(cat, tiles, bands=[1, 2, 3, 4], roiradecbox=None, unwise_dir='.', use_ceres=True, ceres_block=8, save_fits=False, ps=None, psf_broadening=None): ''' Given a list of tractor sources *cat* and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id) runs forced photometry, returning a FITS table the same length as *cat*. ''' # Severely limit sizes of models for src in cat: if isinstance(src, PointSource): src.fixedRadius = 10 else: src.halfsize = 10 wantims = ((ps is not None) or save_fits) wanyband = 'w' fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix', 'pronexp'] Nsrcs = len(cat) phot = fits_table() phot.tile = np.array([' '] * Nsrcs) ra = np.array([src.getPosition().ra for src in cat]) dec = np.array([src.getPosition().dec for src in cat]) for band in bands: print('Photometering WISE band', band) wband = 'w%i' % band # The tiles have some overlap, so for each source, keep the # fit in the tile whose center is closest to the source. tiledists = np.empty(Nsrcs) tiledists[:] = 1e100 flux_invvars = np.zeros(Nsrcs, np.float32) fitstats = dict([(k, np.zeros(Nsrcs, np.float32)) for k in fskeys]) nexp = np.zeros(Nsrcs, np.int16) mjd = np.zeros(Nsrcs, np.float64) for tile in tiles: print('Reading tile', tile.coadd_id) tim = get_unwise_tractor_image(unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradecbox) if tim is None: print('Actually, no overlap with tile', tile.coadd_id) continue if psf_broadening is not None: # psf_broadening is a factor by which the PSF FWHMs # should be scaled; the PSF is a little wider # post-reactivation. psf = tim.getPsf() from tractor import GaussianMixturePSF if isinstance(psf, GaussianMixturePSF): # print('Broadening PSF: from', psf) p0 = psf.getParams() #print('Params:', p0) pnames = psf.getParamNames() #print('Param names:', pnames) p1 = [p * psf_broadening**2 if 'var' in name else p for (p, name) in zip(p0, pnames)] #print('Broadened:', p1) psf.setParams(p1) print('Broadened PSF:', psf) else: print( 'WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf)) print('Read image with shape', tim.shape) # Select sources in play. wcs = tim.wcs.wcs H, W = tim.shape ok, x, y = wcs.radec2pixelxy(ra, dec) x = (x - 1.).astype(np.float32) y = (y - 1.).astype(np.float32) margin = 10. I = np.flatnonzero((x >= -margin) * (x < W + margin) * (y >= -margin) * (y < H + margin)) print(len(I), 'within the image + margin') inbox = ((x[I] >= -0.5) * (x[I] < (W - 0.5)) * (y[I] >= -0.5) * (y[I] < (H - 0.5))) print(sum(inbox), 'strictly within the image') # Compute L_inf distance to (full) tile center. tilewcs = unwise_tile_wcs(tile.ra, tile.dec) cx, cy = tilewcs.crpix ok, tx, ty = tilewcs.radec2pixelxy(ra[I], dec[I]) td = np.maximum(np.abs(tx - cx), np.abs(ty - cy)) closest = (td < tiledists[I]) tiledists[I[closest]] = td[closest] keep = inbox * closest # Source indices (in the full "cat") to keep (the fit values for) srci = I[keep] if not len(srci): print('No sources to be kept; skipping.') continue phot.tile[srci] = tile.coadd_id nexp[srci] = tim.nuims[np.clip(np.round(y[srci]).astype(int), 0, H - 1), np.clip(np.round(x[srci]).astype(int), 0, W - 1)] # Source indices in the margins margi = I[np.logical_not(keep)] # sources in the box -- at the start of the subcat list. subcat = [cat[i] for i in srci] # include *copies* of sources in the margins # (that way we automatically don't save the results) subcat.extend([cat[i].copy() for i in margi]) assert(len(subcat) == len(I)) # FIXME -- set source radii, ...? minsb = 0. fitsky = False # Look in image and set radius based on peak height?? tractor = Tractor([tim], subcat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(wanyband) kwa = dict(fitstat_extras=[('pronexp', [tim.nims])]) t0 = Time() R = tractor.optimize_forced_photometry( minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True, variance=True, shared_params=False, wantims=wantims, **kwa) print('unWISE forced photometry took', Time() - t0) if use_ceres: term = R.ceres_status['termination'] print('Ceres termination status:', term) # Running out of memory can cause failure to converge # and term status = 2. # Fail completely in this case. if term != 0: raise RuntimeError( 'Ceres terminated with status %i' % term) if wantims: ims0 = R.ims0 ims1 = R.ims1 IV, fs = R.IV, R.fitstats if save_fits: import fitsio (dat, mod, ie, chi, roi) = ims1[0] wcshdr = fitsio.FITSHDR() tim.wcs.wcs.add_to_header(wcshdr) tag = 'fit-%s-w%i' % (tile.coadd_id, band) fitsio.write('%s-data.fits' % tag, dat, clobber=True, header=wcshdr) fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr) fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr) if ps: tag = '%s W%i' % (tile.coadd_id, band) (dat, mod, ie, chi, roi) = ims1[0] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() # Save results for this tile. # the "keep" sources are at the beginning of the "subcat" list flux_invvars[srci] = IV[:len(srci)].astype(np.float32) if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): mjd[srci] = (tim.mjdmin + tim.mjdmax) / 2. if fs is None: continue for k in fskeys: x = getattr(fs, k) # fitstats are returned only for un-frozen sources fitstats[k][srci] = np.array(x).astype(np.float32)[:len(srci)] # Note, this is *outside* the loop over tiles. # The fluxes are saved in the source objects, and will be set based on # the 'tiledists' logic above. nm = np.array([src.getBrightness().getBand(wanyband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their default # (1-sigma or whatever) initial fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. phot.set(wband + '_nanomaggies', nm.astype(np.float32)) phot.set(wband + '_nanomaggies_ivar', nm_ivar) dnm = np.zeros(len(nm_ivar), np.float32) okiv = (nm_ivar > 0) dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32) okflux = (nm > 0) mag = np.zeros(len(nm), np.float32) mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux]) ).astype(np.float32) dmag = np.zeros(len(nm), np.float32) ok = (okiv * okflux) dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok]) ).astype(np.float32) mag[np.logical_not(okflux)] = np.nan dmag[np.logical_not(ok)] = np.nan phot.set(wband + '_mag', mag) phot.set(wband + '_mag_err', dmag) for k in fskeys: phot.set(wband + '_' + k, fitstats[k]) phot.set(wband + '_nexp', nexp) if not np.all(mjd == 0): phot.set(wband + '_mjd', mjd) return phot
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None, use_ceres=True, ceres_block=8, save_fits=False, get_models=False, ps=None, psf_broadening=None, pixelized_psf=False, get_masks=None, move_crpix=False, modelsky_dir=None, tag=None): ''' Given a list of tractor sources *cat* and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id) runs forced photometry, returning a FITS table the same length as *cat*. *get_masks*: the WCS to resample mask bits into. ''' from tractor import PointSource, Tractor, ExpGalaxy, DevGalaxy from tractor.sersic import SersicGalaxy if tag is None: tag = '' else: tag = tag + ': ' if not pixelized_psf and psf_broadening is None: # PSF broadening in post-reactivation data, by band. # Newer version from Aaron's email to decam-chatter, 2018-06-14. broadening = {1: 1.0405, 2: 1.0346, 3: None, 4: None} psf_broadening = broadening[band] if False: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('wise-forced-w%i' % band) plots = (ps is not None) if plots: import pylab as plt wantims = (plots or save_fits or get_models) wanyband = 'w' if get_models: models = [] wband = 'w%i' % band Nsrcs = len(cat) phot = fits_table() # Filled in based on unique tile overlap phot.wise_coadd_id = np.array([' '] * Nsrcs, dtype='U8') phot.wise_x = np.zeros(Nsrcs, np.float32) phot.wise_y = np.zeros(Nsrcs, np.float32) phot.set('psfdepth_%s' % wband, np.zeros(Nsrcs, np.float32)) nexp = np.zeros(Nsrcs, np.int16) mjd = np.zeros(Nsrcs, np.float64) central_flux = np.zeros(Nsrcs, np.float32) ra = np.array([src.getPosition().ra for src in cat]) dec = np.array([src.getPosition().dec for src in cat]) fskeys = ['prochi2', 'profracflux'] fitstats = {} if get_masks: mh, mw = get_masks.shape maskmap = np.zeros((mh, mw), np.uint32) tims = [] for tile in tiles: info(tag + 'Reading WISE tile', tile.coadd_id, 'band', band) tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradecbox) if tim is None: debug('Actually, no overlap with WISE coadd tile', tile.coadd_id) continue if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data' % tag) ps.savefig() plt.clf() plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), range=(-5, 10), bins=100) plt.xlabel('Per-pixel intensity (Sigma)') plt.title(tag) ps.savefig() if move_crpix and band in [1, 2]: realwcs = tim.wcs.wcs x, y = realwcs.crpix tile_crpix = tile.get('crpix_w%i' % band) dx = tile_crpix[0] - 1024.5 dy = tile_crpix[1] - 1024.5 realwcs.set_crpix(x + dx, y + dy) debug('unWISE', tile.coadd_id, 'band', band, 'CRPIX', x, y, 'shift by', dx, dy, 'to', realwcs.crpix) if modelsky_dir and band in [1, 2]: fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band)) if not os.path.exists(fn): raise RuntimeError('WARNING: does not exist:', fn) x0, x1, y0, y1 = tim.roi bg = fitsio.FITS(fn)[2][y0:y1, x0:x1] assert (bg.shape == tim.shape) if plots: plt.clf() plt.subplot(1, 2, 1) plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) plt.subplot(1, 2, 2) plt.imshow(bg, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) tag = '%s W%i' % (tile.coadd_id, band) plt.suptitle(tag) ps.savefig() plt.clf() ha = dict(range=(-5, 10), bins=100, histtype='step') plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), color='b', label='Original', **ha) plt.hist(((tim.getImage() - bg) * tim.inverr)[tim.inverr > 0].ravel(), color='g', label='Minus Background', **ha) plt.axvline(0, color='k', alpha=0.5) plt.xlabel('Per-pixel intensity (Sigma)') plt.legend() plt.title(tag + ': background') ps.savefig() # Actually subtract the background! tim.data -= bg # Floor the per-pixel variances, # and add Poisson contribution from sources if band in [1, 2]: # in Vega nanomaggies per pixel floor_sigma = {1: 0.5, 2: 2.0} poissons = {1: 0.15, 2: 0.3} with np.errstate(divide='ignore'): new_ie = 1. / np.sqrt( (1. / tim.inverr)**2 + floor_sigma[band] + poissons[band]**2 * np.maximum(0., tim.data)) new_ie[tim.inverr == 0] = 0. if plots: plt.clf() plt.plot((1. / tim.inverr[tim.inverr > 0]).ravel(), (1. / new_ie[tim.inverr > 0]).ravel(), 'b.') plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band)) plt.xlabel('original') plt.ylabel('floored') ps.savefig() assert (np.all(np.isfinite(new_ie))) assert (np.all(new_ie >= 0.)) tim.inverr = new_ie # Expand a 3-pixel radius around weight=0 (saturated) pixels # from Eddie via crowdsource # https://github.com/schlafly/crowdsource/blob/7069da3e7d9d3124be1cbbe1d21ffeb63fc36dcc/python/wise_proc.py#L74 ## FIXME -- W3/W4 ?? satlimit = 85000 msat = ((tim.data > satlimit) | ((tim.nims == 0) & (tim.nuims > 1))) from scipy.ndimage.morphology import binary_dilation xx, yy = np.mgrid[-3:3 + 1, -3:3 + 1] dilate = xx**2 + yy**2 <= 3**2 msat = binary_dilation(msat, dilate) nbefore = np.sum(tim.inverr == 0) tim.inverr[msat] = 0 nafter = np.sum(tim.inverr == 0) debug('Masking an additional', (nafter - nbefore), 'near-saturated pixels in unWISE', tile.coadd_id, 'band', band) # Read mask file? if get_masks: from astrometry.util.resample import resample_with_wcs, OverlapError # unwise_dir can be a colon-separated list of paths tilemask = None for d in tile.unwise_dir.split(':'): fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id, 'unwise-%s-msk.fits.gz' % tile.coadd_id) if os.path.exists(fn): debug('Reading unWISE mask file', fn) x0, x1, y0, y1 = tim.roi tilemask = fitsio.FITS(fn)[0][y0:y1, x0:x1] break if tilemask is None: info('unWISE mask file for tile', tile.coadd_id, 'does not exist') else: try: tanwcs = tim.wcs.wcs assert (tanwcs.shape == tilemask.shape) Yo, Xo, Yi, Xi, _ = resample_with_wcs(get_masks, tanwcs, intType=np.int16) # Only deal with mask pixels that are set. I, = np.nonzero(tilemask[Yi, Xi] > 0) # Trim to unique area for this tile rr, dd = get_masks.pixelxy2radec(Xo[I] + 1, Yo[I] + 1) good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I = I[good] maskmap[Yo[I], Xo[I]] = tilemask[Yi[I], Xi[I]] except OverlapError: # Shouldn't happen by this point print('Warning: no overlap between WISE tile', tile.coadd_id, 'and brick') if plots: plt.clf() plt.imshow(tilemask, interpolation='nearest', origin='lower') plt.title('Tile %s: mask' % tile.coadd_id) ps.savefig() plt.clf() plt.imshow(maskmap, interpolation='nearest', origin='lower') plt.title('Tile %s: accumulated maskmap' % tile.coadd_id) ps.savefig() # The tiles have some overlap, so zero out pixels outside the # tile's unique area. th, tw = tim.shape xx, yy = np.meshgrid(np.arange(tw), np.arange(th)) rr, dd = tim.wcs.wcs.pixelxy2radec(xx + 1, yy + 1) unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) debug('Tile', tile.coadd_id, '- total of', np.sum(unique), 'unique pixels out of', len(unique.flat), 'total pixels') if get_models: # Save the inverr before blanking out non-unique pixels, for making coadds with no gaps! # (actually, slightly more subtly, expand unique area by 1 pixel) from scipy.ndimage.morphology import binary_dilation du = binary_dilation(unique) tim.coadd_inverr = tim.inverr * du tim.inverr[unique == False] = 0. del xx, yy, rr, dd, unique if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage() * (tim.inverr > 0), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data (unique)' % tag) ps.savefig() if pixelized_psf: from unwise_psf import unwise_psf if (band == 1) or (band == 2): # we only have updated PSFs for W1 and W2 psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id, modelname='neo6_unwisecat') else: psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id) if band == 4: # oversample (the unwise_psf models are at native W4 5.5"/pix, # while the unWISE coadds are made at 2.75"/pix. ph, pw = psfimg.shape subpsf = np.zeros((ph * 2 - 1, pw * 2 - 1), np.float32) from astrometry.util.util import lanczos3_interpolate xx, yy = np.meshgrid( np.arange(0., pw - 0.51, 0.5, dtype=np.float32), np.arange(0., ph - 0.51, 0.5, dtype=np.float32)) xx = xx.ravel() yy = yy.ravel() ix = xx.astype(np.int32) iy = yy.astype(np.int32) dx = (xx - ix).astype(np.float32) dy = (yy - iy).astype(np.float32) psfimg = psfimg.astype(np.float32) rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg]) if plots: plt.clf() plt.imshow(psfimg, interpolation='nearest', origin='lower') plt.title('Original PSF model') ps.savefig() plt.clf() plt.imshow(subpsf, interpolation='nearest', origin='lower') plt.title('Subsampled PSF model') ps.savefig() psfimg = subpsf del xx, yy, ix, iy, dx, dy from tractor.psf import PixelizedPSF psfimg /= psfimg.sum() fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0} psfimg *= fluxrescales[band] tim.psf = PixelizedPSF(psfimg) if psf_broadening is not None and not pixelized_psf: # psf_broadening is a factor by which the PSF FWHMs # should be scaled; the PSF is a little wider # post-reactivation. psf = tim.getPsf() from tractor import GaussianMixturePSF if isinstance(psf, GaussianMixturePSF): debug('Broadening PSF: from', psf) p0 = psf.getParams() pnames = psf.getParamNames() p1 = [ p * psf_broadening**2 if 'var' in name else p for (p, name) in zip(p0, pnames) ] psf.setParams(p1) debug('Broadened PSF:', psf) else: print( 'WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf)) wcs = tim.wcs.wcs _, fx, fy = wcs.radec2pixelxy(ra, dec) x = np.round(fx - 1.).astype(int) y = np.round(fy - 1.).astype(int) good = (x >= 0) * (x < tw) * (y >= 0) * (y < th) # Which sources are in this brick's unique area? usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I, = np.nonzero(good * usrc) nexp[I] = tim.nuims[y[I], x[I]] if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): mjd[I] = (tim.mjdmin + tim.mjdmax) / 2. phot.wise_coadd_id[I] = tile.coadd_id phot.wise_x[I] = fx[I] - 1. phot.wise_y[I] = fy[I] - 1. central_flux[I] = tim.getImage()[y[I], x[I]] del x, y, good, usrc # PSF norm for depth psf = tim.getPsf() h, w = tim.shape patch = psf.getPointSourcePatch(h // 2, w // 2).patch psfnorm = np.sqrt(np.sum(patch**2)) # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags. # In the small empty patches of the sky (eg W4 in 0922p702), we get sig1 = NaN if np.isfinite(tim.sig1): phot.get('psfdepth_%s' % wband)[I] = 1. / (tim.sig1 / psfnorm)**2 tim.tile = tile tims.append(tim) if plots: plt.clf() mn, mx = 0.1, 20000 plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100, range=(np.log10(mn), np.log10(mx))) logt = np.arange(0, 5) plt.xticks(logt, ['%i' % i for i in 10.**logt]) plt.title('Central fluxes (W%i)' % band) plt.axvline(np.log10(20000), color='k') plt.axvline(np.log10(1000), color='k') ps.savefig() # Eddie's non-secret recipe: #- central pixel <= 1000: 19x19 pix box size #- central pixel in 1000 - 20000: 59x59 box size #- central pixel > 20000 or saturated: 149x149 box size #- object near "bright star": 299x299 box size nbig = nmedium = nsmall = 0 for src, cflux in zip(cat, central_flux): if cflux > 20000: R = 100 nbig += 1 elif cflux > 1000: R = 30 nmedium += 1 else: R = 15 nsmall += 1 if isinstance(src, PointSource): src.fixedRadius = R else: ### FIXME -- sizes for galaxies..... can we set PSF size separately? galrad = 0 # RexGalaxy is a subclass of ExpGalaxy if isinstance(src, (ExpGalaxy, DevGalaxy, SersicGalaxy)): galrad = src.shape.re pixscale = 2.75 src.halfsize = int(np.hypot(R, galrad * 5 / pixscale)) debug('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small') tractor = Tractor(tims, cat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(wanyband) t0 = Time() R = tractor.optimize_forced_photometry(fitstats=True, variance=True, shared_params=False, wantims=wantims) info(tag + 'unWISE forced photometry took', Time() - t0) if use_ceres: term = R.ceres_status['termination'] # Running out of memory can cause failure to converge and term # status = 2. Fail completely in this case. if term != 0: info(tag + 'Ceres termination status:', term) raise RuntimeError('Ceres terminated with status %i' % term) if wantims: ims1 = R.ims1 # can happen if empty source list (we still want to generate coadds) if ims1 is None: ims1 = R.ims0 flux_invvars = R.IV if R.fitstats is not None: for k in fskeys: x = getattr(R.fitstats, k) fitstats[k] = np.array(x).astype(np.float32) if save_fits: for i, tim in enumerate(tims): tile = tim.tile (dat, mod, _, chi, _) = ims1[i] wcshdr = fitsio.FITSHDR() tim.wcs.wcs.add_to_header(wcshdr) tag = 'fit-%s-w%i' % (tile.coadd_id, band) fitsio.write('%s-data.fits' % tag, dat, clobber=True, header=wcshdr) fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr) fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr) if plots: # Create models for just the brightest sources bright_cat = [ src for src in cat if src.getBrightness().getBand(wanyband) > 1000 ] debug('Bright soures:', len(bright_cat)) btr = Tractor(tims, bright_cat) for tim in tims: mod = btr.getModelImage(tim) tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) sig1 = tim.sig1 plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: bright-star models' % tag) ps.savefig() if get_models: for i, tim in enumerate(tims): tile = tim.tile (dat, mod, _, _, _) = ims1[i] models.append( (tile.coadd_id, band, tim.wcs.wcs, dat, mod, tim.coadd_inverr)) if plots: for i, tim in enumerate(tims): tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) (dat, mod, _, chi, _) = ims1[i] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() nm = np.array([src.getBrightness().getBand(wanyband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their initial # fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. phot.set('flux_%s' % wband, nm.astype(np.float32)) phot.set('flux_ivar_%s' % wband, nm_ivar.astype(np.float32)) for k in fskeys: phot.set(k + '_' + wband, fitstats.get(k, np.zeros(len(phot), np.float32))) phot.set('nobs_%s' % wband, nexp) phot.set('mjd_%s' % wband, mjd) rtn = wphotduck() rtn.phot = phot rtn.models = None rtn.maskmap = None if get_models: rtn.models = models if get_masks: rtn.maskmap = maskmap return rtn
def run_one_ccd(survey, catsurvey_north, catsurvey_south, resolve_dec, ccd, opt, zoomslice, ps): from functools import reduce from legacypipe.bits import DQ_BITS tlast = Time() #print('Opt:', opt) im = survey.get_image_object(ccd) print('Run_one_ccd: checking cache', survey.cache_dir) if survey.cache_dir is not None: im.check_for_cached_files(survey) if opt.do_calib: im.run_calibs(splinesky=True) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, constant_invvar=opt.constant_invvar, hybridPsf=opt.hybrid_psf, normalizePsf=opt.normalize_psf, old_calibs_ok=True, trim_edges=False) print('Got tim:', tim) #, 'x0,y0', tim.x0, tim.y0) chipwcs = tim.subwcs H, W = tim.shape tnow = Time() print('Read image:', tnow - tlast) tlast = tnow if ccd.camera == 'decam': # Halo subtraction from legacypipe.halos import subtract_one from legacypipe.reference import mask_radius_for_mag, read_gaia ref_margin = mask_radius_for_mag(0.) mpix = int(np.ceil(ref_margin * 3600. / chipwcs.pixel_scale())) marginwcs = chipwcs.get_subimage(-mpix, -mpix, W + 2 * mpix, H + 2 * mpix) gaia = read_gaia(marginwcs, None) keeprad = np.ceil(gaia.keep_radius * 3600. / chipwcs.pixel_scale()).astype(int) _, xx, yy = chipwcs.radec2pixelxy(gaia.ra, gaia.dec) # cut to those touching the chip gaia.cut((xx > -keeprad) * (xx < W + keeprad) * (yy > -keeprad) * (yy < H + keeprad)) Igaia, = np.nonzero(gaia.isgaia * gaia.pointsource) halostars = gaia[Igaia] print('Got', len(gaia), 'Gaia stars,', len(halostars), 'for halo subtraction') moffat = True halos = subtract_one((tim, halostars, moffat)) tim.data -= halos # The "north" and "south" directories often don't have # 'survey-bricks" files of their own -- use the 'survey' one # instead. if catsurvey_south is not None: try: catsurvey_south.get_bricks_readonly() except: catsurvey_south.bricks = survey.get_bricks_readonly() if catsurvey_north is not None: try: catsurvey_north.get_bricks_readonly() except: catsurvey_north.bricks = survey.get_bricks_readonly() # Apply outlier masks outlier_header = None outlier_mask = None posneg_mask = None if opt.outlier_mask is not None: posneg_mask = np.zeros(tim.shape, np.uint8) # Outliers masks are computed within a survey (north/south for dr9), and are stored # in a brick-oriented way, in the results directories. north_ccd = (ccd.camera.strip() != 'decam') catsurvey = catsurvey_north if not north_ccd and catsurvey_south is not None: catsurvey = catsurvey_south bricks = bricks_touching_wcs(chipwcs, survey=catsurvey) for b in bricks: print( 'Reading outlier mask for brick', b.brickname, ':', catsurvey.find_file('outliers_mask', brick=b.brickname, output=False)) ok = read_outlier_mask_file(catsurvey, [tim], b.brickname, pos_neg_mask=posneg_mask, subimage=False, output=False, ps=ps) if not ok: print('WARNING: failed to read outliers mask file for brick', b.brickname) if opt.outlier_mask is not None: outlier_mask = np.zeros((ccd.height, ccd.width), np.uint8) outlier_mask[tim.y0:tim.y0 + H, tim.x0:tim.x0 + W] = posneg_mask del posneg_mask # Grab original image headers (including WCS) im = survey.get_image_object(ccd) imhdr = im.read_image_header() imhdr['CAMERA'] = ccd.camera imhdr['EXPNUM'] = ccd.expnum imhdr['CCDNAME'] = ccd.ccdname imhdr['IMGFILE'] = ccd.image_filename.strip() outlier_header = imhdr if opt.catalog: T = fits_table(opt.catalog) else: chipwcs = tim.subwcs T = get_catalog_in_wcs(chipwcs, survey, catsurvey_north, catsurvey_south=catsurvey_south, resolve_dec=resolve_dec) if T is None: print('No sources to photometer.') return None if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) surveydir = survey.get_survey_dir() del survey if opt.move_gaia: # Gaia stars: move RA,Dec to the epoch of this image. I = np.flatnonzero(T.ref_epoch > 0) if len(I): print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd()) ra, dec = radec_at_mjd(T.ra[I], T.dec[I], T.ref_epoch[I].astype(float), T.pmra[I], T.pmdec[I], T.parallax[I], tim.time.toMjd()) T.ra[I] = ra T.dec[I] = dec tnow = Time() print('Read catalog:', tnow - tlast) tlast = tnow # Find SGA galaxies outside this chip and subtract them before we begin. chipwcs = tim.subwcs _, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec) W, H = chipwcs.get_width(), chipwcs.get_height() sga_out = (T.ref_cat == 'L3') * np.logical_not( (xx >= 1) * (xx <= W) * (yy >= 1) * (yy <= H)) I = np.flatnonzero(sga_out) if len(I): print(len(I), 'SGA galaxies are outside the image. Subtracting...') cat = read_fits_catalog(T[I], bands=[tim.band]) tr = Tractor([tim], cat) mod = tr.getModelImage(0) tim.data -= mod I = np.flatnonzero(np.logical_not(sga_out)) T.cut(I) cat = read_fits_catalog(T, bands='r') # Replace the brightness (which will be a NanoMaggies with g,r,z) # with a NanoMaggies with this image's band only. for src in cat: src.brightness = NanoMaggies(**{tim.band: 1.}) tnow = Time() print('Parse catalog:', tnow - tlast) tlast = tnow print('Forced photom...') F = run_forced_phot(cat, tim, ceres=opt.ceres, derivs=opt.derivs, fixed_also=True, agn=opt.agn, do_forced=opt.forced, do_apphot=opt.apphot, get_model=opt.save_model, ps=ps, timing=True, ceres_threads=opt.ceres_threads) if opt.save_model: # unpack results F, model_img = F F.release = T.release F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.camera = np.array([ccd.camera] * len(F)) F.expnum = np.array([im.expnum] * len(F), dtype=np.int64) F.ccdname = np.array([im.ccdname] * len(F)) # "Denormalizing" F.filter = np.array([tim.band] * len(F)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(F)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(F), dtype=np.float32) F.psfsize = np.array([tim.psf_fwhm * tim.imobj.pixscale] * len(F), dtype=np.float32) F.ccd_cuts = np.array([ccd.ccd_cuts] * len(F)) F.airmass = np.array([ccd.airmass] * len(F), dtype=np.float32) ### --> also add units to the dict below so the FITS headers have units F.sky = np.array([tim.midsky / tim.zpscale / tim.imobj.pixscale**2] * len(F), dtype=np.float32) # in the same units as the depth maps -- flux inverse-variance. F.psfdepth = np.array([(1. / (tim.sig1 / tim.psfnorm)**2)] * len(F), dtype=np.float32) F.galdepth = np.array([(1. / (tim.sig1 / tim.galnorm)**2)] * len(F), dtype=np.float32) F.fwhm = np.array([tim.psf_fwhm] * len(F), dtype=np.float32) F.skyrms = np.array([ccd.skyrms] * len(F), dtype=np.float32) F.ccdzpt = np.array([ccd.ccdzpt] * len(F), dtype=np.float32) F.ccdrarms = np.array([ccd.ccdrarms] * len(F), dtype=np.float32) F.ccddecrms = np.array([ccd.ccddecrms] * len(F), dtype=np.float32) F.ccdphrms = np.array([ccd.ccdphrms] * len(F), dtype=np.float32) if opt.derivs: cosdec = np.cos(np.deg2rad(T.dec)) with np.errstate(divide='ignore', invalid='ignore'): F.dra = (F.flux_dra / F.flux) * 3600. / cosdec F.ddec = (F.flux_ddec / F.flux) * 3600. F.dra[F.flux == 0] = 0. F.ddec[F.flux == 0] = 0. F.dra_ivar = F.flux_dra_ivar * (F.flux / 3600. * cosdec)**2 F.ddec_ivar = F.flux_ddec_ivar * (F.flux / 3600.)**2 F.delete_column('flux_dra') F.delete_column('flux_ddec') F.delete_column('flux_dra_ivar') F.delete_column('flux_ddec_ivar') F.flux = F.flux_fixed F.flux_ivar = F.flux_fixed_ivar F.delete_column('flux_fixed') F.delete_column('flux_fixed_ivar') for c in ['dra', 'ddec', 'dra_ivar', 'ddec_ivar', 'flux', 'flux_ivar']: F.set(c, F.get(c).astype(np.float32)) F.ra = T.ra F.dec = T.dec _, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x - 1).astype(np.float32) F.y = (y - 1).astype(np.float32) h, w = tim.shape ix = np.round(F.x).astype(int) iy = np.round(F.y).astype(int) F.dqmask = tim.dq[np.clip(iy, 0, h - 1), np.clip(ix, 0, w - 1)] # Set an OUT-OF-BOUNDS bit. F.dqmask[reduce(np.logical_or, [ix < 0, ix >= w, iy < 0, iy >= h])] |= DQ_BITS['edge2'] program_name = sys.argv[0] ## FIXME -- from catalog? release = 9999 version_hdr = get_version_header(program_name, surveydir, release) filename = getattr(ccd, 'image_filename') if filename is None: # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn.strip()) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) filename = os.path.join(d2, d1, fname) print('Trimmed filename to', filename) version_hdr.add_record( dict(name='CPFILE', value=filename, comment='CP file')) version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext')) version_hdr.add_record( dict(name='CAMERA', value=ccd.camera, comment='Camera')) version_hdr.add_record( dict(name='EXPNUM', value=im.expnum, comment='Exposure num')) version_hdr.add_record( dict(name='CCDNAME', value=im.ccdname, comment='CCD name')) version_hdr.add_record( dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record( dict(name='PLVER', value=ccd.plver, comment='CP pipeline version')) version_hdr.add_record( dict(name='PLPROCID', value=ccd.plprocid, comment='CP pipeline id')) version_hdr.add_record( dict(name='PROCDATE', value=ccd.procdate, comment='CP image DATE')) keys = [ 'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME' ] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) if opt.save_model or opt.save_data: hdr = fitsio.FITSHDR() tim.getWcs().wcs.add_to_header(hdr) if opt.save_model: fitsio.write(opt.save_model, model_img, header=hdr, clobber=True) print('Wrote', opt.save_model) if opt.save_data: fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True) print('Wrote', opt.save_data) tnow = Time() print('Forced phot:', tnow - tlast) return F, version_hdr, outlier_mask, outlier_header
c = False distance = 19 Coordinate.default_order = 'yx' #LatLng scene = canvas(width=1200, height=700, background=color.white) old_bearing = False bearings = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] mesh = [] left = curve(color=color.orange, radius=0.3, retain=500) right = curve(color=color.orange, radius=0.3, retain=500) ab = curve(color=color.black, radius=0.5) limit_line = curve( color=color.white, radius=1, ) rate(5) tractor = Tractor() tractor_3d = box(color=color.red, size=vec(1, 1, 1)) qtemp = box(color=color.green, size=vec(1, 1, 1)) qobj = [] desv = 0 def move(LatLon): global old_pos, init, old_bearing, mesh, bearings, tractor utm_pos = Coordinate(x=LatLon.x, y=LatLon.y) if init is False: tractor.m_p_s = 2 tractor.run() init = True old_pos = utm_pos scene.camera.pos = vec(utm_pos.x, 4, utm_pos.y - 50)
srcs = get_tractor_sources_dr8(4381,2,114,'u', roi=[0.0,1746.0,0.0,712.0]) labels.append((len(mem), 'sources')) mem.append(memuse()) track('sources') #im,inf = get_tractor_image_dr8(3063,4,60,'g', roi=[100,300,100,300]) im,inf = get_tractor_image_dr8(4381,2,114,'u', roi=[0.0,1746.0,0.0,712.0]) labels.append((len(mem), 'image')) mem.append(memuse()) track('image') tractor = Tractor(Images(im), Catalog(*srcs)) #tractor.cache = NullCache() #disable_galaxy_cache() tractor.cache = Cache(100) set_galaxy_cache_size(100) labels.append((len(mem), 'tractor')) mem.append(memuse()) mod = tractor.getModelImage(im) labels.append((len(mem), 'model')) mem.append(memuse())
def galex_coadds(onegal, galaxy=None, radius_mosaic=30, radius_mask=None, pixscale=1.5, ref_pixscale=0.262, output_dir=None, galex_dir=None, log=None, centrals=True, verbose=False): '''Generate custom GALEX cutouts. radius_mosaic and radius_mask in arcsec pixscale: GALEX pixel scale in arcsec/pixel. ''' import fitsio import matplotlib.pyplot as plt from astrometry.libkd.spherematch import match_radec from astrometry.util.resample import resample_with_wcs, OverlapError from tractor import (Tractor, NanoMaggies, Image, LinearPhotoCal, NCircularGaussianPSF, ConstantFitsWcs, ConstantSky) from legacypipe.survey import imsave_jpeg from legacypipe.catalog import read_fits_catalog if galaxy is None: galaxy = 'galaxy' if galex_dir is None: galex_dir = os.environ.get('GALEX_DIR') if output_dir is None: output_dir = '.' if radius_mask is None: radius_mask = radius_mosaic radius_search = 5.0 # [arcsec] else: radius_search = radius_mask W = H = np.ceil(2 * radius_mosaic / pixscale).astype('int') # [pixels] targetwcs = Tan(onegal['RA'], onegal['DEC'], (W + 1) / 2.0, (H + 1) / 2.0, -pixscale / 3600.0, 0.0, 0.0, pixscale / 3600.0, float(W), float(H)) # Read the custom Tractor catalog tractorfile = os.path.join(output_dir, '{}-tractor.fits'.format(galaxy)) if not os.path.isfile(tractorfile): print('Missing Tractor catalog {}'.format(tractorfile)) return 0 cat = fits_table(tractorfile) print('Read {} sources from {}'.format(len(cat), tractorfile), flush=True, file=log) keep = np.ones(len(cat)).astype(bool) if centrals: # Find the large central galaxy and mask out (ignore) all the models # which are within its elliptical mask. # This algorithm will have to change for mosaics not centered on large # galaxies, e.g., in galaxy groups. m1, m2, d12 = match_radec(cat.ra, cat.dec, onegal['RA'], onegal['DEC'], radius_search / 3600.0, nearest=False) if len(m1) == 0: print('No central galaxies found at the central coordinates!', flush=True, file=log) else: pixfactor = ref_pixscale / pixscale # shift the optical Tractor positions for mm in m1: morphtype = cat.type[mm].strip() if morphtype == 'EXP' or morphtype == 'COMP': e1, e2, r50 = cat.shapeexp_e1[mm], cat.shapeexp_e2[ mm], cat.shapeexp_r[mm] # [arcsec] elif morphtype == 'DEV' or morphtype == 'COMP': e1, e2, r50 = cat.shapedev_e1[mm], cat.shapedev_e2[ mm], cat.shapedev_r[mm] # [arcsec] else: r50 = None if r50: majoraxis = r50 * 5 / pixscale # [pixels] ba, phi = SGA.misc.convert_tractor_e1e2(e1, e2) these = SGA.misc.ellipse_mask(W / 2, W / 2, majoraxis, ba * majoraxis, np.radians(phi), cat.bx * pixfactor, cat.by * pixfactor) if np.sum(these) > 0: #keep[these] = False pass print('Hack!') keep[mm] = False #srcs = read_fits_catalog(cat) #_srcs = np.array(srcs)[~keep].tolist() #mod = SGA.misc.srcs2image(_srcs, ConstantFitsWcs(targetwcs), psf_sigma=3.0) #import matplotlib.pyplot as plt ##plt.imshow(mod, origin='lower') ; plt.savefig('junk.png') #plt.imshow(np.log10(mod), origin='lower') ; plt.savefig('junk.png') #pdb.set_trace() srcs = read_fits_catalog(cat) for src in srcs: src.freezeAllBut('brightness') #srcs_nocentral = np.array(srcs)[keep].tolist() # Find all overlapping GALEX tiles and then read the tims. galex_tiles = _read_galex_tiles(targetwcs, galex_dir, log=log, verbose=verbose) gbands = ['n', 'f'] nicegbands = ['NUV', 'FUV'] zps = dict(n=20.08, f=18.82) coimgs, comods, coresids, coimgs_central, comods_nocentral = [], [], [], [], [] for niceband, band in zip(nicegbands, gbands): J = np.flatnonzero(galex_tiles.get('has_' + band)) print(len(J), 'GALEX tiles have coverage in band', band) coimg = np.zeros((H, W), np.float32) comod = np.zeros((H, W), np.float32) cowt = np.zeros((H, W), np.float32) comod_nocentral = np.zeros((H, W), np.float32) for src in srcs: src.setBrightness(NanoMaggies(**{band: 1})) for j in J: brick = galex_tiles[j] fn = os.path.join( galex_dir, brick.tilename.strip(), '%s-%sd-intbgsub.fits.gz' % (brick.brickname, band)) #print(fn) gwcs = Tan(*[ float(f) for f in [ brick.crval1, brick.crval2, brick.crpix1, brick.crpix2, brick.cdelt1, 0., 0., brick.cdelt2, 3840., 3840. ] ]) img = fitsio.read(fn) #print('Read', img.shape) try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(targetwcs, gwcs, [], 3) except OverlapError: continue K = np.flatnonzero(img[Yi, Xi] != 0.) if len(K) == 0: continue Yo, Xo, Yi, Xi = Yo[K], Xo[K], Yi[K], Xi[K] wt = brick.get(band + 'exptime') coimg[Yo, Xo] += wt * img[Yi, Xi] cowt[Yo, Xo] += wt x0, x1, y0, y1 = min(Xi), max(Xi), min(Yi), max(Yi) subwcs = gwcs.get_subimage(x0, y0, x1 - x0 + 1, y1 - y0 + 1) twcs = ConstantFitsWcs(subwcs) timg = img[y0:y1 + 1, x0:x1 + 1] tie = np.ones_like(timg) ## HACK! #hdr = fitsio.read_header(fn) #zp = hdr[''] zp = zps[band] photocal = LinearPhotoCal(NanoMaggies.zeropointToScale(zp), band=band) tsky = ConstantSky(0.0) # HACK -- circular Gaussian PSF of fixed size... # in arcsec #fwhms = dict(NUV=6.0, FUV=6.0) # -> sigma in pixels #sig = fwhms[band] / 2.35 / twcs.pixel_scale() sig = 6.0 / np.sqrt(8 * np.log(2)) / twcs.pixel_scale() tpsf = NCircularGaussianPSF([sig], [1.]) tim = Image(data=timg, inverr=tie, psf=tpsf, wcs=twcs, sky=tsky, photocal=photocal, name='GALEX ' + band + brick.brickname) ## Build the model image with and without the central galaxy model. tractor = Tractor([tim], srcs) mod = tractor.getModelImage(0) tractor.freezeParam('images') tractor.optimize_forced_photometry(priors=False, shared_params=False) mod = tractor.getModelImage(0) srcs_nocentral = np.array(srcs)[keep].tolist() #srcs_nocentral = np.array(srcs)[nocentral].tolist() tractor_nocentral = Tractor([tim], srcs_nocentral) mod_nocentral = tractor_nocentral.getModelImage(0) comod[Yo, Xo] += wt * mod[Yi - y0, Xi - x0] comod_nocentral[Yo, Xo] += wt * mod_nocentral[Yi - y0, Xi - x0] coimg /= np.maximum(cowt, 1e-18) comod /= np.maximum(cowt, 1e-18) comod_nocentral /= np.maximum(cowt, 1e-18) coresid = coimg - comod # Subtract the model image which excludes the central (comod_nocentral) # from the data (coimg) to isolate the light of the central # (coimg_central). coimg_central = coimg - comod_nocentral coimgs.append(coimg) comods.append(comod) coresids.append(coresid) comods_nocentral.append(comod_nocentral) coimgs_central.append(coimg_central) # Write out the final images with and without the central, making sure # to apply the zeropoint to go from counts/s to AB nanomaggies. # https://asd.gsfc.nasa.gov/archive/galex/FAQ/counts_background.html for thisimg, imtype in zip((coimg, comod, comod_nocentral), ('image', 'model', 'model-nocentral')): fitsfile = os.path.join( output_dir, '{}-{}-{}.fits'.format(galaxy, imtype, niceband)) if verbose: print('Writing {}'.format(fitsfile)) fitsio.write(fitsfile, thisimg * 10**(-0.4 * (zp - 22.5)), clobber=True) # Build a color mosaic (but note that the images here are in units of # background-subtracted counts/s). #_galex_rgb = _galex_rgb_moustakas #_galex_rgb = _galex_rgb_dstn _galex_rgb = _galex_rgb_official for imgs, imtype in zip( (coimgs, comods, coresids, comods_nocentral, coimgs_central), ('image', 'model', 'resid', 'model-nocentral', 'image-central')): rgb = _galex_rgb(imgs) jpgfile = os.path.join(output_dir, '{}-{}-FUVNUV.jpg'.format(galaxy, imtype)) if verbose: print('Writing {}'.format(jpgfile)) imsave_jpeg(jpgfile, rgb, origin='lower') return 1
if len(I): print 'Converting', len(I), 'bogus COMP galaxies to DEV' for i in I: T.type[i] = 'DEV' else: T = fits_table(catfn) T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE) #print 'Got cat:', cat print 'Forced photom...' tr = Tractor([tim], cat) tr.freezeParam('images') for src in cat: src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) kwa = {} if opt.ceres: B = 8 kwa.update(use_ceres=True, BW=B, BH=B) if opt.plots is None: kwa.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, **kwa)
# HACK -- circular Gaussian PSF of fixed size... # in arcsec #fwhms = dict(NUV=6.0, FUV=6.0) # -> sigma in pixels #sig = fwhms[band] / 2.35 / twcs.pixel_scale() sig = 6.0 / 2.35 / twcs.pixel_scale() tpsf = NCircularGaussianPSF([sig], [1.]) tim = Image(data=timg, inverr=tie, psf=tpsf, wcs=twcs, sky=tsky, photocal=photocal, name='GALEX ' + band + brick.brickname) tractor = Tractor([tim], srcs) mod = tractor.getModelImage(0) print('Tractor image', tim.name) plt.clf() plt.imshow(timg, interpolation='nearest', origin='lower') ps.savefig() print('Tractor model', tim.name) plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower') ps.savefig() tractor.freezeParam('images') print('Params:')
def wise_cutouts(ra, dec, radius, ps, pixscale=2.75, survey_dir=None, unwise_dir=None): ''' radius in arcsec. pixscale: WISE pixel scale in arcsec/pixel; make this smaller than 2.75 to oversample. ''' if unwise_dir is None: unwise_dir = os.environ.get('UNWISE_COADDS_DIR') npix = int(np.ceil(radius / pixscale)) print('Image size:', npix) W = H = npix pix = pixscale / 3600. wcs = Tan(ra, dec, (W + 1) / 2., (H + 1) / 2., -pix, 0., 0., pix, float(W), float(H)) # Find DECaLS bricks overlapping survey = LegacySurveyData(survey_dir=survey_dir) B = bricks_touching_wcs(wcs, survey=survey) print('Found', len(B), 'bricks overlapping') TT = [] for b in B.brickname: fn = survey.find_file('tractor', brick=b) T = fits_table(fn) print('Read', len(T), 'from', b) primhdr = fitsio.read_header(fn) TT.append(T) T = merge_tables(TT) print('Total of', len(T), 'sources') T.cut(T.brick_primary) print(len(T), 'primary') margin = 20 ok, xx, yy = wcs.radec2pixelxy(T.ra, T.dec) I = np.flatnonzero((xx > -margin) * (yy > -margin) * (xx < W + margin) * (yy < H + margin)) T.cut(I) print(len(T), 'within ROI') #return wcs,T # Pull out DECaLS coadds (image, model, resid). dwcs = wcs.scale(2. * pixscale / 0.262) dh, dw = dwcs.shape print('DECaLS resampled shape:', dh, dw) tags = ['image', 'model', 'resid'] coimgs = [np.zeros((dh, dw, 3), np.uint8) for t in tags] for b in B.brickname: fn = survey.find_file('image', brick=b, band='r') bwcs = Tan(fn, 1) # ext 1: .fz try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(dwcs, bwcs) except ResampleError: continue if len(Yo) == 0: continue print('Resampling', len(Yo), 'pixels from', b) xl, xh, yl, yh = Xi.min(), Xi.max(), Yi.min(), Yi.max() #print('python legacypipe/runbrick.py -b %s --zoom %i %i %i %i --outdir cluster --pixpsf --splinesky --pipe --no-early-coadds' % # (b, xl-5, xh+5, yl-5, yh+5) + ' -P \'pickles/cluster-%(brick)s-%%(stage)s.pickle\'') for i, tag in enumerate(tags): fn = survey.find_file(tag + '-jpeg', brick=b) img = plt.imread(fn) img = np.flipud(img) coimgs[i][Yo, Xo, :] = img[Yi, Xi, :] tt = dict(image='Image', model='Model', resid='Resid') for img, tag in zip(coimgs, tags): plt.clf() dimshow(img, ticks=False) plt.title('DECaLS grz %s' % tt[tag]) ps.savefig() # Find unWISE tiles overlapping tiles = unwise_tiles_touching_wcs(wcs) print('Cut to', len(tiles), 'unWISE tiles') # Here we assume the targetwcs is axis-aligned and that the # edge midpoints yield the RA,Dec limits (true for TAN). r, d = wcs.pixelxy2radec(np.array([1, W, W / 2, W / 2]), np.array([H / 2, H / 2, 1, H])) # the way the roiradec box is used, the min/max order doesn't matter roiradec = [r[0], r[1], d[2], d[3]] ra, dec = T.ra, T.dec srcs = read_fits_catalog(T) wbands = [1, 2, 3, 4] wanyband = 'w' for band in wbands: f = T.get('flux_w%i' % band) f *= 10.**(primhdr['WISEAB%i' % band] / 2.5) coimgs = [np.zeros((H, W), np.float32) for b in wbands] comods = [np.zeros((H, W), np.float32) for b in wbands] con = [np.zeros((H, W), np.uint8) for b in wbands] for iband, band in enumerate(wbands): print('Photometering WISE band', band) wband = 'w%i' % band for i, src in enumerate(srcs): #print('Source', src, 'brightness', src.getBrightness(), 'params', src.getBrightness().getParams()) #src.getBrightness().setParams([T.wise_flux[i, band-1]]) src.setBrightness( NanoMaggies(**{wanyband: T.get('flux_w%i' % band)[i]})) # print('Set source brightness:', src.getBrightness()) # The tiles have some overlap, so for each source, keep the # fit in the tile whose center is closest to the source. for tile in tiles: print('Reading tile', tile.coadd_id) tim = get_unwise_tractor_image(unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradec) if tim is None: print('Actually, no overlap with tile', tile.coadd_id) continue print('Read image with shape', tim.shape) # Select sources in play. wisewcs = tim.wcs.wcs H, W = tim.shape ok, x, y = wisewcs.radec2pixelxy(ra, dec) x = (x - 1.).astype(np.float32) y = (y - 1.).astype(np.float32) margin = 10. I = np.flatnonzero((x >= -margin) * (x < W + margin) * (y >= -margin) * (y < H + margin)) print(len(I), 'within the image + margin') subcat = [srcs[i] for i in I] tractor = Tractor([tim], subcat) mod = tractor.getModelImage(0) # plt.clf() # dimshow(tim.getImage(), ticks=False) # plt.title('WISE %s %s' % (tile.coadd_id, wband)) # ps.savefig() # plt.clf() # dimshow(mod, ticks=False) # plt.title('WISE %s %s' % (tile.coadd_id, wband)) # ps.savefig() try: Yo, Xo, Yi, Xi, nil = resample_with_wcs(wcs, tim.wcs.wcs) except ResampleError: continue if len(Yo) == 0: continue print('Resampling', len(Yo), 'pixels from WISE', tile.coadd_id, band) coimgs[iband][Yo, Xo] += tim.getImage()[Yi, Xi] comods[iband][Yo, Xo] += mod[Yi, Xi] con[iband][Yo, Xo] += 1 for img, mod, n in zip(coimgs, comods, con): img /= np.maximum(n, 1) mod /= np.maximum(n, 1) for band, img, mod in zip(wbands, coimgs, comods): lo, hi = np.percentile(img, [25, 99]) plt.clf() dimshow(img, vmin=lo, vmax=hi, ticks=False) plt.title('WISE W%i Data' % band) ps.savefig() plt.clf() dimshow(mod, vmin=lo, vmax=hi, ticks=False) plt.title('WISE W%i Model' % band) ps.savefig() resid = img - mod mx = np.abs(resid).max() plt.clf() dimshow(resid, vmin=-mx, vmax=mx, ticks=False) plt.title('WISE W%i Resid' % band) ps.savefig() #kwa = dict(mn=-0.1, mx=2., arcsinh = 1.) kwa = dict(mn=-0.1, mx=2., arcsinh=None) rgb = _unwise_to_rgb(coimgs[:2], **kwa) plt.clf() dimshow(rgb, ticks=False) plt.title('WISE W1/W2 Data') ps.savefig() rgb = _unwise_to_rgb(comods[:2], **kwa) plt.clf() dimshow(rgb, ticks=False) plt.title('WISE W1/W2 Model') ps.savefig() kwa = dict(mn=-1, mx=1, arcsinh=None) rgb = _unwise_to_rgb( [img - mod for img, mod in list(zip(coimgs, comods))[:2]], **kwa) plt.clf() dimshow(rgb, ticks=False) plt.title('WISE W1/W2 Resid') ps.savefig() return wcs, T
def unwise_forcedphot(cat, tiles, band=1, roiradecbox=None, use_ceres=True, ceres_block=8, save_fits=False, get_models=False, ps=None, psf_broadening=None, pixelized_psf=False, get_masks=None, move_crpix=False, modelsky_dir=None): ''' Given a list of tractor sources *cat* and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id) runs forced photometry, returning a FITS table the same length as *cat*. *get_masks*: the WCS to resample mask bits into. ''' from tractor import NanoMaggies, PointSource, Tractor, ExpGalaxy, DevGalaxy, FixedCompositeGalaxy if not pixelized_psf and psf_broadening is None: # PSF broadening in post-reactivation data, by band. # Newer version from Aaron's email to decam-chatter, 2018-06-14. broadening = { 1: 1.0405, 2: 1.0346, 3: None, 4: None } psf_broadening = broadening[band] if False: from astrometry.util.plotutils import PlotSequence ps = PlotSequence('wise-forced-w%i' % band) plots = (ps is not None) if plots: import pylab as plt wantims = (plots or save_fits or get_models) wanyband = 'w' if get_models: models = {} wband = 'w%i' % band fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix', 'pronexp'] Nsrcs = len(cat) phot = fits_table() # Filled in based on unique tile overlap phot.wise_coadd_id = np.array([' '] * Nsrcs) phot.set(wband + '_psfdepth', np.zeros(len(phot), np.float32)) ra = np.array([src.getPosition().ra for src in cat]) dec = np.array([src.getPosition().dec for src in cat]) nexp = np.zeros(Nsrcs, np.int16) mjd = np.zeros(Nsrcs, np.float64) central_flux = np.zeros(Nsrcs, np.float32) fitstats = {} tims = [] if get_masks: mh,mw = get_masks.shape maskmap = np.zeros((mh,mw), np.uint32) for tile in tiles: print('Reading WISE tile', tile.coadd_id, 'band', band) tim = get_unwise_tractor_image(tile.unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradecbox) if tim is None: print('Actually, no overlap with tile', tile.coadd_id) continue if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data' % tag) ps.savefig() plt.clf() plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), range=(-5,10), bins=100) plt.xlabel('Per-pixel intensity (Sigma)') plt.title(tag) ps.savefig() if move_crpix and band in [1, 2]: realwcs = tim.wcs.wcs x,y = realwcs.crpix tile_crpix = tile.get('crpix_w%i' % band) dx = tile_crpix[0] - 1024.5 dy = tile_crpix[1] - 1024.5 realwcs.set_crpix(x+dx, y+dy) #print('CRPIX', x,y, 'shift by', dx,dy, 'to', realwcs.crpix) if modelsky_dir and band in [1, 2]: fn = os.path.join(modelsky_dir, '%s.%i.mod.fits' % (tile.coadd_id, band)) if not os.path.exists(fn): raise RuntimeError('WARNING: does not exist:', fn) x0,x1,y0,y1 = tim.roi bg = fitsio.FITS(fn)[2][y0:y1, x0:x1] #print('Read background map:', bg.shape, bg.dtype, 'vs image', tim.shape) if plots: plt.clf() plt.subplot(1,2,1) plt.imshow(tim.getImage(), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) plt.subplot(1,2,2) plt.imshow(bg, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=5 * sig1) tag = '%s W%i' % (tile.coadd_id, band) plt.suptitle(tag) ps.savefig() plt.clf() ha = dict(range=(-5,10), bins=100, histtype='step') plt.hist((tim.getImage() * tim.inverr)[tim.inverr > 0].ravel(), color='b', label='Original', **ha) plt.hist(((tim.getImage()-bg) * tim.inverr)[tim.inverr > 0].ravel(), color='g', label='Minus Background', **ha) plt.axvline(0, color='k', alpha=0.5) plt.xlabel('Per-pixel intensity (Sigma)') plt.legend() plt.title(tag + ': background') ps.savefig() # Actually subtract the background! tim.data -= bg # Floor the per-pixel variances if band in [1,2]: # in Vega nanomaggies per pixel floor_sigma = {1: 0.5, 2: 2.0} with np.errstate(divide='ignore'): new_ie = 1. / np.hypot(1./tim.inverr, floor_sigma[band]) new_ie[tim.inverr == 0] = 0. if plots: plt.clf() plt.plot((1. / tim.inverr[tim.inverr>0]).ravel(), (1./new_ie[tim.inverr>0]).ravel(), 'b.') plt.title('unWISE per-pixel error: %s band %i' % (tile.coadd_id, band)) plt.xlabel('original') plt.ylabel('floored') ps.savefig() tim.inverr = new_ie # Read mask file? if get_masks: from astrometry.util.resample import resample_with_wcs, OverlapError # unwise_dir can be a colon-separated list of paths tilemask = None for d in tile.unwise_dir.split(':'): fn = os.path.join(d, tile.coadd_id[:3], tile.coadd_id, 'unwise-%s-msk.fits.gz' % tile.coadd_id) if os.path.exists(fn): print('Reading unWISE mask file', fn) x0,x1,y0,y1 = tim.roi tilemask = fitsio.FITS(fn)[0][y0:y1,x0:x1] break if tilemask is None: print('unWISE mask file for tile', tile.coadd_id, 'does not exist') else: try: tanwcs = tim.wcs.wcs assert(tanwcs.shape == tilemask.shape) Yo,Xo,Yi,Xi,_ = resample_with_wcs(get_masks, tanwcs, intType=np.int16) # Only deal with mask pixels that are set. I, = np.nonzero(tilemask[Yi,Xi] > 0) # Trim to unique area for this tile rr,dd = get_masks.pixelxy2radec(Yo[I]+1, Xo[I]+1) good = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I = I[good] maskmap[Yo[I],Xo[I]] = tilemask[Yi[I], Xi[I]] except OverlapError: # Shouldn't happen by this point print('No overlap between WISE tile', tile.coadd_id, 'and brick') # The tiles have some overlap, so zero out pixels outside the # tile's unique area. th,tw = tim.shape xx,yy = np.meshgrid(np.arange(tw), np.arange(th)) rr,dd = tim.wcs.wcs.pixelxy2radec(xx+1, yy+1) unique = radec_in_unique_area(rr, dd, tile.ra1, tile.ra2, tile.dec1, tile.dec2) #print(np.sum(unique), 'of', (th*tw), 'pixels in this tile are unique') tim.inverr[unique == False] = 0. del xx,yy,rr,dd,unique if plots: sig1 = tim.sig1 plt.clf() plt.imshow(tim.getImage() * (tim.inverr > 0), interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() tag = '%s W%i' % (tile.coadd_id, band) plt.title('%s: tim data (unique)' % tag) ps.savefig() if pixelized_psf: import unwise_psf if (band == 1) or (band == 2): # we only have updated PSFs for W1 and W2 psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id, modelname='neo4_unwisecat') else: psfimg = unwise_psf.get_unwise_psf(band, tile.coadd_id) if band == 4: # oversample (the unwise_psf models are at native W4 5.5"/pix, # while the unWISE coadds are made at 2.75"/pix. ph,pw = psfimg.shape subpsf = np.zeros((ph*2-1, pw*2-1), np.float32) from astrometry.util.util import lanczos3_interpolate xx,yy = np.meshgrid(np.arange(0., pw-0.51, 0.5, dtype=np.float32), np.arange(0., ph-0.51, 0.5, dtype=np.float32)) xx = xx.ravel() yy = yy.ravel() ix = xx.astype(np.int32) iy = yy.astype(np.int32) dx = (xx - ix).astype(np.float32) dy = (yy - iy).astype(np.float32) psfimg = psfimg.astype(np.float32) rtn = lanczos3_interpolate(ix, iy, dx, dy, [subpsf.flat], [psfimg]) if plots: plt.clf() plt.imshow(psfimg, interpolation='nearest', origin='lower') plt.title('Original PSF model') ps.savefig() plt.clf() plt.imshow(subpsf, interpolation='nearest', origin='lower') plt.title('Subsampled PSF model') ps.savefig() psfimg = subpsf del xx, yy, ix, iy, dx, dy from tractor.psf import PixelizedPSF psfimg /= psfimg.sum() fluxrescales = {1: 1.04, 2: 1.005, 3: 1.0, 4: 1.0} psfimg *= fluxrescales[band] tim.psf = PixelizedPSF(psfimg) if psf_broadening is not None and not pixelized_psf: # psf_broadening is a factor by which the PSF FWHMs # should be scaled; the PSF is a little wider # post-reactivation. psf = tim.getPsf() from tractor import GaussianMixturePSF if isinstance(psf, GaussianMixturePSF): # print('Broadening PSF: from', psf) p0 = psf.getParams() pnames = psf.getParamNames() p1 = [p * psf_broadening**2 if 'var' in name else p for (p, name) in zip(p0, pnames)] psf.setParams(p1) print('Broadened PSF:', psf) else: print('WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf)) wcs = tim.wcs.wcs ok,x,y = wcs.radec2pixelxy(ra, dec) x = np.round(x - 1.).astype(int) y = np.round(y - 1.).astype(int) good = (x >= 0) * (x < tw) * (y >= 0) * (y < th) # Which sources are in this brick's unique area? usrc = radec_in_unique_area(ra, dec, tile.ra1, tile.ra2, tile.dec1, tile.dec2) I, = np.nonzero(good * usrc) nexp[I] = tim.nuims[y[I], x[I]] if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): mjd[I] = (tim.mjdmin + tim.mjdmax) / 2. phot.wise_coadd_id[I] = tile.coadd_id central_flux[I] = tim.getImage()[y[I], x[I]] del x,y,good,usrc # PSF norm for depth psf = tim.getPsf() h,w = tim.shape patch = psf.getPointSourcePatch(h//2, w//2).patch psfnorm = np.sqrt(np.sum(patch**2)) # To handle zero-depth, we return 1/nanomaggies^2 units rather than mags. psfdepth = 1. / (tim.sig1 / psfnorm)**2 phot.get(wband + '_psfdepth')[I] = psfdepth tim.tile = tile tims.append(tim) if plots: plt.clf() mn,mx = 0.1, 20000 plt.hist(np.log10(np.clip(central_flux, mn, mx)), bins=100, range=(np.log10(mn), np.log10(mx))) logt = np.arange(0, 5) plt.xticks(logt, ['%i' % i for i in 10.**logt]) plt.title('Central fluxes (W%i)' % band) plt.axvline(np.log10(20000), color='k') plt.axvline(np.log10(1000), color='k') ps.savefig() # Eddie's non-secret recipe: #- central pixel <= 1000: 19x19 pix box size #- central pixel in 1000 - 20000: 59x59 box size #- central pixel > 20000 or saturated: 149x149 box size #- object near "bright star": 299x299 box size nbig = nmedium = nsmall = 0 for src,cflux in zip(cat, central_flux): if cflux > 20000: R = 100 nbig += 1 elif cflux > 1000: R = 30 nmedium += 1 else: R = 15 nsmall += 1 if isinstance(src, PointSource): src.fixedRadius = R else: ### FIXME -- sizes for galaxies..... can we set PSF size separately? galrad = 0 # RexGalaxy is a subclass of ExpGalaxy if isinstance(src, (ExpGalaxy, DevGalaxy)): galrad = src.shape.re elif isinstance(src, FixedCompositeGalaxy): galrad = max(src.shapeExp.re, src.shapeDev.re) pixscale = 2.75 src.halfsize = int(np.hypot(R, galrad * 5 / pixscale)) #print('Set WISE source sizes:', nbig, 'big', nmedium, 'medium', nsmall, 'small') minsb = 0. fitsky = False tractor = Tractor(tims, cat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(wanyband) kwa = dict(fitstat_extras=[('pronexp', [tim.nims for tim in tims])]) t0 = Time() R = tractor.optimize_forced_photometry( minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True, variance=True, shared_params=False, wantims=wantims, **kwa) print('unWISE forced photometry took', Time() - t0) if use_ceres: term = R.ceres_status['termination'] # Running out of memory can cause failure to converge # and term status = 2. # Fail completely in this case. if term != 0: print('Ceres termination status:', term) raise RuntimeError( 'Ceres terminated with status %i' % term) if wantims: ims1 = R.ims1 flux_invvars = R.IV if R.fitstats is not None: for k in fskeys: x = getattr(R.fitstats, k) fitstats[k] = np.array(x).astype(np.float32) if save_fits: for i,tim in enumerate(tims): tile = tim.tile (dat, mod, ie, chi, roi) = ims1[i] wcshdr = fitsio.FITSHDR() tim.wcs.wcs.add_to_header(wcshdr) tag = 'fit-%s-w%i' % (tile.coadd_id, band) fitsio.write('%s-data.fits' % tag, dat, clobber=True, header=wcshdr) fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr) fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr) if plots: # Create models for just the brightest sources bright_cat = [src for src in cat if src.getBrightness().getBand(wanyband) > 1000] print('Bright soures:', len(bright_cat)) btr = Tractor(tims, bright_cat) for tim in tims: mod = btr.getModelImage(tim) tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) sig1 = tim.sig1 plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: bright-star models' % tag) ps.savefig() if get_models: for i,tim in enumerate(tims): tile = tim.tile (dat, mod, ie, chi, roi) = ims1[i] models[(tile.coadd_id, band)] = (mod, dat, ie, tim.roi, tim.wcs.wcs) if plots: for i,tim in enumerate(tims): tile = tim.tile tag = '%s W%i' % (tile.coadd_id, band) (dat, mod, ie, chi, roi) = ims1[i] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=25 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() nm = np.array([src.getBrightness().getBand(wanyband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their default # (1-sigma or whatever) initial fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. phot.set(wband + '_nanomaggies', nm.astype(np.float32)) phot.set(wband + '_nanomaggies_ivar', nm_ivar.astype(np.float32)) dnm = np.zeros(len(nm_ivar), np.float32) okiv = (nm_ivar > 0) dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32) okflux = (nm > 0) mag = np.zeros(len(nm), np.float32) mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux]) ).astype(np.float32) dmag = np.zeros(len(nm), np.float32) ok = (okiv * okflux) dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok]) ).astype(np.float32) mag[np.logical_not(okflux)] = np.nan dmag[np.logical_not(ok)] = np.nan phot.set(wband + '_mag', mag) phot.set(wband + '_mag_err', dmag) for k in fskeys: phot.set(wband + '_' + k, fitstats[k]) phot.set(wband + '_nexp', nexp) if not np.all(mjd == 0): phot.set(wband + '_mjd', mjd) rtn = wphotduck() rtn.phot = phot rtn.models = None rtn.maskmap = None if get_models: rtn.models = models if get_masks: rtn.maskmap = maskmap return rtn
slc = (slice(y0, s.iy + margin+1), slice(x0, s.ix + margin+1)) subpsf = tim.psf.constantPsfAt(s.ix, s.iy) subtim = Image(data=tim.data[slc], inverr=tim.inverr[slc], psf=subpsf, photocal=tim.photocal, name=tim.name + '/star %i' % i) flux = NanoMaggies.magToNanomaggies(s.mag) print 'Flux:', flux flux = max(flux, 1.) src = PointSource(PixPos(s.xx - x0, s.yy - y0), NanoMaggies(**{tim.band: flux})) tr = Tractor([subtim], [src]) mod = tr.getModelImage(0) #mod = src.getModelImage(tsubtim) tractors.append(tr) resids.append((subtim.data - mod) * (subtim.inverr > 0)) chis.append((subtim.data - mod) * subtim.inverr) plt.subplot(R, C, i+1) dimshow(mod, **ima) plt.suptitle('Models for Pan-STARRS stars: ' + tim.name) ps.savefig() resa = dict(vmin=-5.*tim.sig1, vmax=5.*tim.sig1, ticks=False) chia = dict(vmin=-5., vmax=5., ticks=False)
def main(): decals = Decals() catpattern = 'pipebrick-cats/tractor-phot-b%06i.fits' ra,dec = 242, 7 # Region-of-interest, in pixels: x0, x1, y0, y1 #roi = None roi = [500, 1000, 500, 1000] if roi is not None: x0,x1,y0,y1 = roi #expnum = 346623 #ccdname = 'N12' #chips = decals.find_ccds(expnum=expnum, extname=ccdname) #print 'Found', len(chips), 'chips for expnum', expnum, 'extname', ccdname #if len(chips) != 1: #return False chips = decals.get_ccds() D = np.argsort(np.hypot(chips.ra - ra, chips.dec - dec)) print('Closest chip:', chips[D[0]]) chips = [chips[D[0]]] im = DecamImage(decals, chips[0]) print('Image:', im) targetwcs = Sip(im.wcsfn) if roi is not None: targetwcs = targetwcs.get_subimage(x0, y0, x1-x0, y1-y0) r0,r1,d0,d1 = targetwcs.radec_bounds() # ~ 30-pixel margin margin = 2e-3 if r0 > r1: # RA wrap-around TT = [brick_catalog_for_radec_box(ra,rb, d0-margin,d1+margin, decals, catpattern) for (ra,rb) in [(0, r1+margin), (r0-margin, 360.)]] T = merge_tables(TT) T._header = TT[0]._header else: T = brick_catalog_for_radec_box(r0-margin,r1+margin,d0-margin, d1+margin, decals, catpattern) print('Got', len(T), 'catalog entries within range') cat = read_fits_catalog(T, T._header) print('Got', len(cat), 'catalog objects') print('Switching ellipse parameterizations') switch_to_soft_ellipses(cat) keepcat = [] for src in cat: if not np.all(np.isfinite(src.getParams())): print('Src has infinite params:', src) continue if isinstance(src, FixedCompositeGalaxy): f = src.fracDev.getClippedValue() if f == 0.: src = ExpGalaxy(src.pos, src.brightness, src.shapeExp) elif f == 1.: src = DevGalaxy(src.pos, src.brightness, src.shapeDev) keepcat.append(src) cat = keepcat slc = None if roi is not None: slc = slice(y0,y1), slice(x0,x1) tim = im.get_tractor_image(slc=slc) print('Got', tim) tim.psfex.fitSavedData(*tim.psfex.splinedata) tim.psfex.radius = 20 tim.psf = CachingPsfEx.fromPsfEx(tim.psfex) tractor = Tractor([tim], cat) print('Created', tractor) mod = tractor.getModelImage(0) plt.clf() dimshow(tim.getImage(), **tim.ima) plt.title('Image') plt.savefig('1.png') plt.clf() dimshow(mod, **tim.ima) plt.title('Model') plt.savefig('2.png') ok,x,y = targetwcs.radec2pixelxy([src.getPosition().ra for src in cat], [src.getPosition().dec for src in cat]) ax = plt.axis() plt.plot(x, y, 'rx') #plt.savefig('3.png') plt.axis(ax) plt.title('Sources') plt.savefig('3.png') bands = [im.band] import runbrick runbrick.photoobjdir = '.' scat,T = get_sdss_sources(bands, targetwcs, local=False) print('Got', len(scat), 'SDSS sources in bounds') stractor = Tractor([tim], scat) print('Created', stractor) smod = stractor.getModelImage(0) plt.clf() dimshow(smod, **tim.ima) plt.title('SDSS model') plt.savefig('4.png')