def get_context_data(self, **kwargs): print('get_context_data called') t0 = Time() from viewer import settings print('Using query description:', self.querydesc) context = super(CatalogSearchList, self).get_context_data(**kwargs) print('Got context data', context, 'in', Time() - t0) context.update( root_url=settings.ROOT_URL, search_description=self.querydesc, ) req = self.request args = req.GET.copy() args.pop('page', None) qstring = '?' + '&'.join( ['%s=%s' % (k, v) for k, v in args.items() if len(v)]) context['myurl'] = req.path + qstring context['fitsurl'] = reverse(fits_results) + qstring context['viewurl'] = reverse(viewer_results) + qstring pager = context.get('paginator') context['total_items'] = pager.count print('Done updating context:', Time() - t0) return context
def map(self, *args, **kwargs): tstart = Time() res = super(MyMultiproc, self).map(*args, **kwargs) tend = Time() self.serial.append((self.t0, tstart)) self.parallel.append((tstart, tend)) self.t0 = tend return res
def test_jumbo(): # Test jumbo (> 2 GB) args/results import numpy as np from astrometry.util.ttime import Time with TimingPool(2) as pool: Time.add_measurement(TimingPoolMeas(pool, pickleTraffic=True)) t0 = Time() R = pool.map(test_func_4, [np.ones(int(2.1 * 1024 * 1024 * 1024 / 8))]) print('Jumbo:', np.sum(R)) print(Time() - t0)
def time_run_calibs(*args): from astrometry.util.ttime import Time t0 = Time() rtn = run_calibs(*args) t1 = Time() print('Time run_calibs:', t1 - t0) import sys sys.stdout.flush() sys.stderr.flush() return rtn
def __call__(self, stage, **kwargs): from astrometry.util.ttime import Time from datetime import datetime t0 = Time() print 'Running stage', stage, 'at', datetime.now().isoformat() rtn = super(CallGlobalTime, self).__call__(stage, **kwargs) t1 = Time() print 'Stage', stage, ':', t1 - t0 print 'Stage', stage, 'finished:', datetime.now().isoformat() return rtn
def _optimize_forcedphot_core(self, tractor, result, umodels, imlist, mod0, scales, skyderivs, minFlux, nonneg=None, wantims0=None, wantims1=None, negfluxval=None, rois=None, priors=None, sky=None, justims0=None, subimgs=None, damp=None, alphas=None, Nsky=None, mindlnp=None, shared_params=None): # print(len(umodels), 'umodels, lengths', [len(x) for x in umodels]) if len(umodels) == 0: return Nsourceparams = len(umodels[0]) imgs = tractor.images t0 = Time() derivs = [[] for i in range(Nsourceparams)] for i, (tim, umods, scale) in enumerate(zip(imlist, umodels, scales)): for um, dd in zip(umods, derivs): if um is None: continue dd.append((um * scale, tim)) logverb('forced phot: derivs', Time() - t0) if sky: # Sky derivatives are part of the image derivatives, so go # first in the derivative list. derivs = skyderivs + derivs assert (len(derivs) == tractor.numberOfParams()) self._lsqr_forced_photom(tractor, result, derivs, mod0, imgs, umodels, rois, scales, priors, sky, minFlux, justims0, subimgs, damp, alphas, Nsky, mindlnp, shared_params)
def global_init(loglevel): ''' Global initialization routine called by mpi4py when each worker is started. ''' import socket import os from mpi4py import MPI print('MPI4PY process starting on', socket.gethostname(), 'pid', os.getpid(), 'MPI rank', MPI.COMM_WORLD.Get_rank()) import logging logging.basicConfig(level=loglevel, format='%(message)s', stream=sys.stdout) # tractor logging is *soooo* chatty logging.getLogger('tractor.engine').setLevel(loglevel + 10) from astrometry.util.ttime import Time, MemMeas Time.add_measurement(MemMeas)
def start_subphase(self, name): # push current state to stack tstart = Time() self.serial.append((self.t0, tstart)) self.t0 = tstart self.phases.append((name, self.serial, self.parallel, self.t0)) # print('Starting subphase', name) # print(' pushing serial', self.serial) # print(' pushing parallel', self.parallel) self.serial = [] self.parallel = []
def test(): #import logging #multiprocessing.log_to_stderr().setLevel(logging.DEBUG) # per multiprocess.Pool documentation (as of Python 3.8 at least), can't just # let a Pool fall out of scope; must close or you risk the process hanging. # This definitely happens on Linux! with TimingPool(4) as pool: R = pool.map(test_func, [10, 20, 30]) print('worker cpu:', pool.get_worker_cpu()) print('worker wall:', pool.get_worker_wall()) print('pickles:', pool.get_pickle_traffic_string()) import numpy as np print('Creating second pool') with TimingPool(4) as pool: print('Using second pool') R = pool.map(test_func_2, [np.random.normal(size=1000000) for x in range(5)]) print('Got result from second pool') print('worker cpu:', pool.get_worker_cpu()) print('worker wall:', pool.get_worker_wall()) print('pickles:', pool.get_pickle_traffic_string()) from astrometry.util.ttime import Time with TimingPool(4, track_send_pickles=False, track_recv_pickles=False) as pool: m = TimingPoolMeas(pool, pickleTraffic=False) Time.add_measurement(m) t0 = Time() R = pool.map(test_func, [20, 20, 20]) print(Time() - t0) Time.remove_measurement(m) with TimingPool(4) as pool: Time.add_measurement(TimingPoolMeas(pool, pickleTraffic=True)) t0 = Time() R = pool.map(test_func_3, [np.random.normal(size=1000000) for x in range(5)]) print(Time() - t0)
def report(self, nthreads): # Tally the serial time up to now tend = Time() self.serial.append((self.t0, tend)) self.t0 = tend # Nasty... peek into Time members scpu = 0. swall = 0. print('Serial:') for t0, t1 in self.serial: print(t1 - t0) for m0, m1 in zip(t0.meas, t1.meas): if isinstance(m0, CpuMeas): scpu += m1.cpu_seconds_since(m0) swall += m1.wall_seconds_since(m0) #print ' total cpu', scpu, 'wall', swall pworkercpu = 0. pworkerwall = 0. pwall = 0. pcpu = 0. print('Parallel:') for t0, t1 in self.parallel: print(t1 - t0) for m0, m1 in zip(t0.meas, t1.meas): if isinstance(m0, TimingPoolTimestamp): mt0 = m0.t0 mt1 = m1.t0 pworkercpu += mt1['worker_cpu'] - mt0['worker_cpu'] pworkerwall += mt1['worker_wall'] - mt0['worker_wall'] elif isinstance(m0, CpuMeas): pwall += m1.wall_seconds_since(m0) pcpu += m1.cpu_seconds_since(m0) print() print('Total serial CPU ', scpu) print('Total serial Wall ', swall) print('Total worker CPU ', pworkercpu) print('Total worker Wall ', pworkerwall) print('Total parallel Wall', pwall) print('Total parallel CPU ', pcpu) print() tcpu = scpu + pworkercpu + pcpu twall = swall + pwall if nthreads is None: nthreads = 1 print('Grand total CPU: %.1f sec' % tcpu) print('Grand total Wall: %.1f sec' % twall) print('Grand total CPU utilization: %.2f cores' % (tcpu / twall)) print('Grand total efficiency: %.1f %%' % (100. * tcpu / (twall * nthreads))) print()
def optimize(self, tractor, alphas=None, damp=0, priors=True, scale_columns=True, shared_params=True, variance=False, just_variance=False, **nil): logverb(tractor.getName() + ': Finding derivs...') t0 = Time() allderivs = tractor.getDerivs() tderivs = Time() - t0 #print(Time() - t0) #print('allderivs:', allderivs) # for d in allderivs: # for (p,im) in d: # print('patch mean', np.mean(p.patch)) logverb('Finding optimal update direction...') t0 = Time() X = self.getUpdateDirection(tractor, allderivs, damp=damp, priors=priors, scale_columns=scale_columns, shared_params=shared_params, variance=variance) if X is None: # Failure return (0., None, 0.) if variance: if len(X) == 0: return 0, X, 0, None X, var = X if just_variance: return var #print(Time() - t0) topt = Time() - t0 #print('X:', X) if len(X) == 0: return 0, X, 0. logverb('X: len', len(X), '; non-zero entries:', np.count_nonzero(X)) logverb('Finding optimal step size...') t0 = Time() (dlogprob, alpha) = self.tryUpdates(tractor, X, alphas=alphas) tstep = Time() - t0 logverb('Finished opt2.') logverb(' alpha =', alpha) logverb(' Tderiv', tderivs) logverb(' Topt ', topt) logverb(' Tstep ', tstep) if variance: return dlogprob, X, alpha, var return dlogprob, X, alpha
def imap_unordered(self, func, iterable, chunksize=None, wrap=False): # So, this is a bit strange, tracking parallel vs serial time # for an async object, via the ImapTracker & callback to # _imap_finished. tstart = Time() self.serial.append((self.t0, tstart)) #res = super(MyMultiproc, self).imap_unordered(*args, **kwargs) cs = chunksize if cs is None: cs = self.map_chunksize if self.pool is None: import itertools return itertools.imap(func, iterable) if wrap or self.wrap_all: func = funcwrapper(func) res = self.pool.imap_unordered(func, iterable, chunksize=cs) return ImapTracker(res, self, tstart)
if __name__ == '__main__': import optparse parser = optparse.OptionParser(usage='%prog <decam-image-filename> <decam-HDU> <catalog.fits or "DR1"> <output-catalog.fits>') parser.add_option('--zoom', type=int, nargs=4, help='Set target image extent (default "0 2046 0 4094")') parser.add_option('--no-ceres', action='store_false', default=True, dest='ceres', help='Do not use Ceres optimiziation engine (use scipy)') parser.add_option('--catalog-path', default='dr1', help='Path to DECaLS DR1 catalogs; default %default, eg, /project/projectdirs/cosmo/data/legacysurvey/dr1') parser.add_option('--plots', default=None, help='Create plots; specify a base filename for the plots') opt,args = parser.parse_args() if len(args) != 4: parser.print_help() sys.exit(-1) Time.add_measurement(MemMeas) t0 = Time() filename = args[0] hdu = int(args[1]) catfn = args[2] outfn = args[3] if os.path.exists(outfn): print 'Ouput file exists:', outfn sys.exit(0) zoomslice = None if opt.zoom is not None: (x0,x1,y0,y1) = opt.zoom
def stage1(T=None, coimgs=None, cons=None, detmaps=None, detivs=None, targetrd=None, pixscale=None, targetwcs=None, W=None, H=None, bands=None, tims=None, ps=None, brick=None, cat=None): orig_wcsxy0 = [tim.wcs.getX0Y0() for tim in tims] hot = np.zeros((H, W), np.float32) for band in bands: detmap = detmaps[band] / np.maximum(1e-16, detivs[band]) detsn = detmap * np.sqrt(detivs[band]) hot = np.maximum(hot, detsn) detmaps[band] = detmap ### FIXME -- ugri for sedname, sed in [('Flat', (1., 1., 1.)), ('Red', (2.5, 1.0, 0.4))]: sedmap = np.zeros((H, W), np.float32) sediv = np.zeros((H, W), np.float32) for iband, band in enumerate(bands): # We convert the detmap to canonical band via # detmap * w # And the corresponding change to sig1 is # sig1 * w # So the invvar-weighted sum is # (detmap * w) / (sig1**2 * w**2) # = detmap / (sig1**2 * w) sedmap += detmaps[band] * detivs[band] / sed[iband] sediv += detivs[band] / sed[iband]**2 sedmap /= np.maximum(1e-16, sediv) sedsn = sedmap * np.sqrt(sediv) hot = np.maximum(hot, sedsn) plt.clf() dimshow(np.round(sedsn), vmin=0, vmax=10, cmap='hot') plt.title('SED-matched detection filter: %s' % sedname) ps.savefig() peaks = (hot > 4) blobs, nblobs = label(peaks) print('N detected blobs:', nblobs) blobslices = find_objects(blobs) # Un-set catalog blobs for x, y in zip(T.itx, T.ity): # blob number bb = blobs[y, x] if bb == 0: continue # un-set 'peaks' within this blob slc = blobslices[bb - 1] peaks[slc][blobs[slc] == bb] = 0 # Now, after having removed catalog sources, crank up the detection threshold peaks &= (hot > 5) # zero out the edges(?) peaks[0, :] = peaks[:, 0] = 0 peaks[-1, :] = peaks[:, -1] = 0 peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[0:-2, 1:-1]) peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[2:, 1:-1]) peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[1:-1, 0:-2]) peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[1:-1, 2:]) # These are our peaks pki = np.flatnonzero(peaks) peaky, peakx = np.unravel_index(pki, peaks.shape) print(len(peaky), 'peaks') crossa = dict(ms=10, mew=1.5) plt.clf() dimshow(get_rgb(coimgs, bands)) ax = plt.axis() plt.plot(T.tx, T.ty, 'r+', **crossa) plt.plot(peakx, peaky, '+', color=green, **crossa) plt.axis(ax) plt.title('SDSS + SED-matched detections') ps.savefig() ### HACK -- high threshold again # Segment, and record which sources fall into each blob blobs, nblobs = label((hot > 20)) print('N detected blobs:', nblobs) blobslices = find_objects(blobs) T.blob = blobs[T.ity, T.itx] blobsrcs = [] blobflux = [] fluximg = coimgs[1] for blob in range(1, nblobs + 1): blobsrcs.append(np.flatnonzero(T.blob == blob)) bslc = blobslices[blob - 1] blobflux.append(np.sum(fluximg[bslc][blobs[bslc] == blob])) # Fit the SDSS sources for tim in tims: tim.psfex.fitSavedData(*tim.psfex.splinedata) tim.psf = tim.psfex # How far down to render model profiles minsigma = 0.1 for tim in tims: tim.modelMinval = minsigma * tim.sig1 srcvariances = [[] for src in cat] # Fit in order of flux for blobnumber, iblob in enumerate(np.argsort(-np.array(blobflux))): bslc = blobslices[iblob] Isrcs = blobsrcs[iblob] if len(Isrcs) == 0: continue print() print('Blob', blobnumber, 'of', len(blobflux), ':', len(Isrcs), 'sources') print('Source indices:', Isrcs) print() # blob bbox in target coords sy, sx = bslc by0, by1 = sy.start, sy.stop bx0, bx1 = sx.start, sx.stop blobh, blobw = by1 - by0, bx1 - bx0 rr, dd = targetwcs.pixelxy2radec([bx0, bx0, bx1, bx1], [by0, by1, by1, by0]) alphas = [0.1, 0.3, 1.0] subtims = [] for itim, tim in enumerate(tims): h, w = tim.shape ok, x, y = tim.subwcs.radec2pixelxy(rr, dd) sx0, sx1 = x.min(), x.max() sy0, sy1 = y.min(), y.max() if sx1 < 0 or sy1 < 0 or sx1 > w or sy1 > h: continue sx0 = np.clip(int(np.floor(sx0)), 0, w - 1) sx1 = np.clip(int(np.ceil(sx1)), 0, w - 1) + 1 sy0 = np.clip(int(np.floor(sy0)), 0, h - 1) sy1 = np.clip(int(np.ceil(sy1)), 0, h - 1) + 1 subslc = slice(sy0, sy1), slice(sx0, sx1) subimg = tim.getImage()[subslc] subie = tim.getInvError()[subslc] subwcs = tim.getWcs().copy() ox0, oy0 = orig_wcsxy0[itim] subwcs.setX0Y0(ox0 + sx0, oy0 + sy0) # Mask out inverr for pixels that are not within the blob. subtarget = targetwcs.get_subimage(bx0, by0, blobw, blobh) subsubwcs = tim.subwcs.get_subimage(int(sx0), int(sy0), int(sx1 - sx0), int(sy1 - sy0)) try: Yo, Xo, Yi, Xi, rims = resample_with_wcs( subsubwcs, subtarget, [], 2) except OverlapError: print('No overlap') continue if len(Yo) == 0: continue subie2 = np.zeros_like(subie) I = np.flatnonzero(blobs[bslc][Yi, Xi] == (iblob + 1)) subie2[Yo[I], Xo[I]] = subie[Yo[I], Xo[I]] subie = subie2 # If the subimage (blob) is small enough, instantiate a # constant PSF model in the center. if sy1 - sy0 < 100 and sx1 - sx0 < 100: subpsf = tim.psf.mogAt(ox0 + (sx0 + sx1) / 2., oy0 + (sy0 + sy1) / 2.) else: # Otherwise, instantiate a (shifted) spatially-varying # PsfEx model. subpsf = ShiftedPsf(tim.psf, ox0 + sx0, oy0 + sy0) subtim = Image(data=subimg, inverr=subie, wcs=subwcs, psf=subpsf, photocal=tim.getPhotoCal(), sky=tim.getSky(), name=tim.name) subtim.band = tim.band subtim.sig1 = tim.sig1 subtim.modelMinval = tim.modelMinval subtims.append(subtim) subcat = Catalog(*[cat[i] for i in Isrcs]) subtr = Tractor(subtims, subcat) subtr.freezeParam('images') # Optimize individual sources in order of flux fluxes = [] for src in subcat: # HACK -- here we just *sum* the nanomaggies in each band. Bogus! br = src.getBrightness() flux = sum([br.getFlux(band) for band in bands]) fluxes.append(flux) Ibright = np.argsort(-np.array(fluxes)) if len(Ibright) >= 5: # -Remember the original subtim images # -Compute initial models for each source (in each tim) # -Subtract initial models from images # -During fitting, for each source: # -add back in the source's initial model (to each tim) # -fit, with Catalog([src]) # -subtract final model (from each tim) # -Replace original subtim images # # --Might want to omit newly-added detection-filter sources, since their # fluxes are bogus. # Remember original tim images orig_timages = [tim.getImage().copy() for tim in subtims] initial_models = [] # Create initial models for each tim x each source for tim in subtims: mods = [] for src in subcat: mod = src.getModelPatch(tim) mods.append(mod) if mod is not None: if not np.all(np.isfinite(mod.patch)): print('Non-finite mod patch') print('source:', src) print('tim:', tim) print('PSF:', tim.getPsf()) assert (np.all(np.isfinite(mod.patch))) mod.addTo(tim.getImage(), scale=-1) initial_models.append(mods) # For sources in decreasing order of brightness for numi, i in enumerate(Ibright): tsrc = Time() print('Fitting source', i, '(%i of %i in blob)' % (numi, len(Ibright))) src = subcat[i] print(src) srctractor = Tractor(subtims, [src]) srctractor.freezeParams('images') # Add this source's initial model back in. for tim, mods in zip(subtims, initial_models): mod = mods[i] if mod is not None: mod.addTo(tim.getImage()) print('Optimizing:', srctractor) srctractor.printThawedParams() for step in range(50): dlnp, X, alpha = srctractor.optimize(priors=False, shared_params=False, alphas=alphas) print('dlnp:', dlnp, 'src', src) if dlnp < 0.1: break for tim in subtims: mod = src.getModelPatch(tim) if mod is not None: mod.addTo(tim.getImage(), scale=-1) for tim, img in zip(subtims, orig_timages): tim.data = img del orig_timages del initial_models else: # Fit sources one at a time, but don't subtract other models subcat.freezeAllParams() for numi, i in enumerate(Ibright): tsrc = Time() print('Fitting source', i, '(%i of %i in blob)' % (numi, len(Ibright))) print(subcat[i]) subcat.freezeAllBut(i) print('Optimizing:', subtr) subtr.printThawedParams() for step in range(10): dlnp, X, alpha = subtr.optimize(priors=False, shared_params=False, alphas=alphas) print('dlnp:', dlnp) if dlnp < 0.1: break print('Fitting source took', Time() - tsrc) print(subcat[i]) if len(Isrcs) > 1 and len(Isrcs) <= 10: tfit = Time() # Optimize all at once? subcat.thawAllParams() print('Optimizing:', subtr) subtr.printThawedParams() for step in range(20): dlnp, X, alpha = subtr.optimize(priors=False, shared_params=False, alphas=alphas) print('dlnp:', dlnp) if dlnp < 0.1: break # Variances subcat.thawAllRecursive() subcat.freezeAllParams() for isub, srci in enumerate(Isrcs): print('Variances for source', srci) subcat.thawParam(isub) src = subcat[isub] print('Source', src) print('Params:', src.getParamNames()) if isinstance(src, (DevGalaxy, ExpGalaxy)): src.shape = EllipseE.fromEllipseESoft(src.shape) elif isinstance(src, FixedCompositeGalaxy): src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp) src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev) print('Converted ellipse:', src) allderivs = subtr.getDerivs() for iparam, derivs in enumerate(allderivs): dchisq = 0 for deriv, tim in derivs: h, w = tim.shape deriv.clipTo(w, h) ie = tim.getInvError() slc = deriv.getSlice(ie) chi = deriv.patch * ie[slc] dchisq += (chi**2).sum() if dchisq == 0.: v = np.nan else: v = 1. / dchisq srcvariances[srci].append(v) assert (len(srcvariances[srci]) == subcat[isub].numberOfParams()) subcat.freezeParam(isub) cat.thawAllRecursive() for i, src in enumerate(cat): print('Source', i, src) print('variances:', srcvariances[i]) print(len(srcvariances[i]), 'vs', src.numberOfParams()) if len(srcvariances[i]) != src.numberOfParams(): # This can happen for sources outside the brick bounds: they never get optimized? print('Warning: zeroing variances for source', src) srcvariances[i] = [0] * src.numberOfParams() if isinstance(src, (DevGalaxy, ExpGalaxy)): src.shape = EllipseE.fromEllipseESoft(src.shape) elif isinstance(src, FixedCompositeGalaxy): src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp) src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev) assert (len(srcvariances[i]) == src.numberOfParams()) variances = np.hstack(srcvariances) assert (len(variances) == cat.numberOfParams()) return dict(cat=cat, variances=variances)
import runbrick if __name__ == '__main__': parser = optparse.OptionParser(usage='%prog [options] brick-number') parser.add_option('--threads', type=int, help='Run multi-threaded') parser.add_option('--no-ceres', action='store_true', help='Do not use Ceres') parser.add_option('--stamp', action='store_true', help='Run a tiny postage-stamp') opt,args = parser.parse_args() if len(args) != 1: parser.print_help() sys.exit(-1) brick = int(args[0], 10) Time.add_measurement(MemMeas) lvl = logging.WARNING logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) if opt.threads and opt.threads > 1: from astrometry.util.multiproc import multiproc if True: mp = multiproc(opt.threads, init=runbrick_global_init, initargs=()) else: from utils.debugpool import DebugPool, DebugPoolMeas dpool = DebugPool(opt.threads, taskqueuesize=2*opt.threads, initializer=runbrick_global_init) mp = multiproc(pool=dpool)
def forced_photometry( self, tractor, alphas=None, damp=0, priors=False, minsb=0., mindlnp=1., rois=None, sky=False, minFlux=None, fitstats=False, fitstat_extras=None, justims0=False, variance=False, skyvariance=False, shared_params=True, nonneg=False, nilcounts=-1e30, wantims=True, negfluxval=None, ): from basics import LinearPhotoCal, ShiftedWcs result = OptResult() assert (not priors) scales = [] imgs = tractor.getImages() for img in imgs: assert (isinstance(img.getPhotoCal(), LinearPhotoCal)) scales.append(img.getPhotoCal().getScale()) if rois is not None: assert (len(rois) == len(imgs)) # HACK -- if sky=True, assume we are fitting the sky in ALL images. # We could ask which ones are thawed... if sky: for img in imgs: # FIXME -- would be nice to allow multi-param linear sky models assert (img.getSky().numberOfParams() == 1) Nsourceparams = tractor.catalog.numberOfParams() srcs = list(tractor.catalog.getThawedSources()) # Render unit-flux models for each source. t0 = Time() (umodels, umodtosource, umodsforsource) = self._get_umodels(tractor, srcs, imgs, minsb, rois) for umods in umodels: assert (len(umods) == Nsourceparams) tmods = Time() - t0 logverb('forced phot: getting unit-flux models:', tmods) subimgs = [] if rois is not None: for i, img in enumerate(imgs): roi = rois[i] y0 = roi[0].start x0 = roi[1].start subwcs = img.wcs.shift(x0, y0) subimg = Image(data=img.data[roi], inverr=img.inverr[roi], psf=img.psf, wcs=subwcs, sky=img.sky, photocal=img.photocal, name=img.name) subimgs.append(subimg) imlist = subimgs else: imlist = imgs t0 = Time() fsrcs = list(tractor.catalog.getFrozenSources()) mod0 = [] for img in imlist: # "sky = not sky": I'm not just being contrary :) # If we're fitting sky, we'll do a setParams() and get the # sky models to render themselves when evaluating lnProbs, # rather than pre-computing the nominal value here and # then computing derivatives. mod0.append( tractor.getModelImage(img, fsrcs, minsb=minsb, sky=not sky)) tmod = Time() - t0 logverb('forced phot: getting frozen-source model:', tmod) skyderivs = None if sky: t0 = Time() # build the derivative list as required by getUpdateDirection: # (param0) -> [ (deriv, img), (deriv, img), ... ], ... ], skyderivs = [] for img in imlist: dskys = img.getSky().getParamDerivatives(tractor, img, None) for dsky in dskys: skyderivs.append([(dsky, img)]) Nsky = len(skyderivs) assert (Nsky == tractor.images.numberOfParams()) assert (Nsky + Nsourceparams == tractor.numberOfParams()) logverb('forced phot: sky derivs', Time() - t0) else: Nsky = 0 wantims0 = wantims1 = wantims if fitstats: wantims1 = True self._optimize_forcedphot_core(tractor, result, umodels, imlist, mod0, scales, skyderivs, minFlux, nonneg=nonneg, wantims0=wantims0, wantims1=wantims1, negfluxval=negfluxval, rois=rois, priors=priors, sky=sky, justims0=justims0, subimgs=subimgs, damp=damp, alphas=alphas, Nsky=Nsky, mindlnp=mindlnp, shared_params=shared_params) if variance: # Inverse variance t0 = Time() result.IV = self._get_iv(sky, skyvariance, Nsky, skyderivs, srcs, imlist, umodels, scales) logverb('forced phot: variance:', Time() - t0) imsBest = getattr(result, 'ims1', None) if fitstats and imsBest is None: print 'Warning: fit stats not computed because imsBest is None' result.fitstats = None elif fitstats: t0 = Time() result.fitstats = self._get_fitstats(tractor.catalog, imsBest, srcs, imlist, umodsforsource, umodels, scales, nilcounts, extras=fitstat_extras) logverb('forced phot: fit stats:', Time() - t0) return result
def run_one_ccd(survey, catsurvey_north, catsurvey_south, resolve_dec, ccd, opt, zoomslice, ps): from functools import reduce from legacypipe.bits import DQ_BITS tlast = Time() #print('Opt:', opt) im = survey.get_image_object(ccd) print('Run_one_ccd: checking cache', survey.cache_dir) if survey.cache_dir is not None: im.check_for_cached_files(survey) if opt.do_calib: im.run_calibs(splinesky=True) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, constant_invvar=opt.constant_invvar, hybridPsf=opt.hybrid_psf, normalizePsf=opt.normalize_psf, old_calibs_ok=True, trim_edges=False) print('Got tim:', tim) #, 'x0,y0', tim.x0, tim.y0) chipwcs = tim.subwcs H, W = tim.shape tnow = Time() print('Read image:', tnow - tlast) tlast = tnow if ccd.camera == 'decam': # Halo subtraction from legacypipe.halos import subtract_one from legacypipe.reference import mask_radius_for_mag, read_gaia ref_margin = mask_radius_for_mag(0.) mpix = int(np.ceil(ref_margin * 3600. / chipwcs.pixel_scale())) marginwcs = chipwcs.get_subimage(-mpix, -mpix, W + 2 * mpix, H + 2 * mpix) gaia = read_gaia(marginwcs, None) keeprad = np.ceil(gaia.keep_radius * 3600. / chipwcs.pixel_scale()).astype(int) _, xx, yy = chipwcs.radec2pixelxy(gaia.ra, gaia.dec) # cut to those touching the chip gaia.cut((xx > -keeprad) * (xx < W + keeprad) * (yy > -keeprad) * (yy < H + keeprad)) Igaia, = np.nonzero(gaia.isgaia * gaia.pointsource) halostars = gaia[Igaia] print('Got', len(gaia), 'Gaia stars,', len(halostars), 'for halo subtraction') moffat = True halos = subtract_one((tim, halostars, moffat)) tim.data -= halos # The "north" and "south" directories often don't have # 'survey-bricks" files of their own -- use the 'survey' one # instead. if catsurvey_south is not None: try: catsurvey_south.get_bricks_readonly() except: catsurvey_south.bricks = survey.get_bricks_readonly() if catsurvey_north is not None: try: catsurvey_north.get_bricks_readonly() except: catsurvey_north.bricks = survey.get_bricks_readonly() # Apply outlier masks outlier_header = None outlier_mask = None posneg_mask = None if opt.outlier_mask is not None: posneg_mask = np.zeros(tim.shape, np.uint8) # Outliers masks are computed within a survey (north/south for dr9), and are stored # in a brick-oriented way, in the results directories. north_ccd = (ccd.camera.strip() != 'decam') catsurvey = catsurvey_north if not north_ccd and catsurvey_south is not None: catsurvey = catsurvey_south bricks = bricks_touching_wcs(chipwcs, survey=catsurvey) for b in bricks: print( 'Reading outlier mask for brick', b.brickname, ':', catsurvey.find_file('outliers_mask', brick=b.brickname, output=False)) ok = read_outlier_mask_file(catsurvey, [tim], b.brickname, pos_neg_mask=posneg_mask, subimage=False, output=False, ps=ps) if not ok: print('WARNING: failed to read outliers mask file for brick', b.brickname) if opt.outlier_mask is not None: outlier_mask = np.zeros((ccd.height, ccd.width), np.uint8) outlier_mask[tim.y0:tim.y0 + H, tim.x0:tim.x0 + W] = posneg_mask del posneg_mask # Grab original image headers (including WCS) im = survey.get_image_object(ccd) imhdr = im.read_image_header() imhdr['CAMERA'] = ccd.camera imhdr['EXPNUM'] = ccd.expnum imhdr['CCDNAME'] = ccd.ccdname imhdr['IMGFILE'] = ccd.image_filename.strip() outlier_header = imhdr if opt.catalog: T = fits_table(opt.catalog) else: chipwcs = tim.subwcs T = get_catalog_in_wcs(chipwcs, survey, catsurvey_north, catsurvey_south=catsurvey_south, resolve_dec=resolve_dec) if T is None: print('No sources to photometer.') return None if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) surveydir = survey.get_survey_dir() del survey if opt.move_gaia: # Gaia stars: move RA,Dec to the epoch of this image. I = np.flatnonzero(T.ref_epoch > 0) if len(I): print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd()) ra, dec = radec_at_mjd(T.ra[I], T.dec[I], T.ref_epoch[I].astype(float), T.pmra[I], T.pmdec[I], T.parallax[I], tim.time.toMjd()) T.ra[I] = ra T.dec[I] = dec tnow = Time() print('Read catalog:', tnow - tlast) tlast = tnow # Find SGA galaxies outside this chip and subtract them before we begin. chipwcs = tim.subwcs _, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec) W, H = chipwcs.get_width(), chipwcs.get_height() sga_out = (T.ref_cat == 'L3') * np.logical_not( (xx >= 1) * (xx <= W) * (yy >= 1) * (yy <= H)) I = np.flatnonzero(sga_out) if len(I): print(len(I), 'SGA galaxies are outside the image. Subtracting...') cat = read_fits_catalog(T[I], bands=[tim.band]) tr = Tractor([tim], cat) mod = tr.getModelImage(0) tim.data -= mod I = np.flatnonzero(np.logical_not(sga_out)) T.cut(I) cat = read_fits_catalog(T, bands='r') # Replace the brightness (which will be a NanoMaggies with g,r,z) # with a NanoMaggies with this image's band only. for src in cat: src.brightness = NanoMaggies(**{tim.band: 1.}) tnow = Time() print('Parse catalog:', tnow - tlast) tlast = tnow print('Forced photom...') F = run_forced_phot(cat, tim, ceres=opt.ceres, derivs=opt.derivs, fixed_also=True, agn=opt.agn, do_forced=opt.forced, do_apphot=opt.apphot, get_model=opt.save_model, ps=ps, timing=True, ceres_threads=opt.ceres_threads) if opt.save_model: # unpack results F, model_img = F F.release = T.release F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.camera = np.array([ccd.camera] * len(F)) F.expnum = np.array([im.expnum] * len(F), dtype=np.int64) F.ccdname = np.array([im.ccdname] * len(F)) # "Denormalizing" F.filter = np.array([tim.band] * len(F)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(F)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(F), dtype=np.float32) F.psfsize = np.array([tim.psf_fwhm * tim.imobj.pixscale] * len(F), dtype=np.float32) F.ccd_cuts = np.array([ccd.ccd_cuts] * len(F)) F.airmass = np.array([ccd.airmass] * len(F), dtype=np.float32) ### --> also add units to the dict below so the FITS headers have units F.sky = np.array([tim.midsky / tim.zpscale / tim.imobj.pixscale**2] * len(F), dtype=np.float32) # in the same units as the depth maps -- flux inverse-variance. F.psfdepth = np.array([(1. / (tim.sig1 / tim.psfnorm)**2)] * len(F), dtype=np.float32) F.galdepth = np.array([(1. / (tim.sig1 / tim.galnorm)**2)] * len(F), dtype=np.float32) F.fwhm = np.array([tim.psf_fwhm] * len(F), dtype=np.float32) F.skyrms = np.array([ccd.skyrms] * len(F), dtype=np.float32) F.ccdzpt = np.array([ccd.ccdzpt] * len(F), dtype=np.float32) F.ccdrarms = np.array([ccd.ccdrarms] * len(F), dtype=np.float32) F.ccddecrms = np.array([ccd.ccddecrms] * len(F), dtype=np.float32) F.ccdphrms = np.array([ccd.ccdphrms] * len(F), dtype=np.float32) if opt.derivs: cosdec = np.cos(np.deg2rad(T.dec)) with np.errstate(divide='ignore', invalid='ignore'): F.dra = (F.flux_dra / F.flux) * 3600. / cosdec F.ddec = (F.flux_ddec / F.flux) * 3600. F.dra[F.flux == 0] = 0. F.ddec[F.flux == 0] = 0. F.dra_ivar = F.flux_dra_ivar * (F.flux / 3600. * cosdec)**2 F.ddec_ivar = F.flux_ddec_ivar * (F.flux / 3600.)**2 F.delete_column('flux_dra') F.delete_column('flux_ddec') F.delete_column('flux_dra_ivar') F.delete_column('flux_ddec_ivar') F.flux = F.flux_fixed F.flux_ivar = F.flux_fixed_ivar F.delete_column('flux_fixed') F.delete_column('flux_fixed_ivar') for c in ['dra', 'ddec', 'dra_ivar', 'ddec_ivar', 'flux', 'flux_ivar']: F.set(c, F.get(c).astype(np.float32)) F.ra = T.ra F.dec = T.dec _, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x - 1).astype(np.float32) F.y = (y - 1).astype(np.float32) h, w = tim.shape ix = np.round(F.x).astype(int) iy = np.round(F.y).astype(int) F.dqmask = tim.dq[np.clip(iy, 0, h - 1), np.clip(ix, 0, w - 1)] # Set an OUT-OF-BOUNDS bit. F.dqmask[reduce(np.logical_or, [ix < 0, ix >= w, iy < 0, iy >= h])] |= DQ_BITS['edge2'] program_name = sys.argv[0] ## FIXME -- from catalog? release = 9999 version_hdr = get_version_header(program_name, surveydir, release) filename = getattr(ccd, 'image_filename') if filename is None: # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn.strip()) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) filename = os.path.join(d2, d1, fname) print('Trimmed filename to', filename) version_hdr.add_record( dict(name='CPFILE', value=filename, comment='CP file')) version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext')) version_hdr.add_record( dict(name='CAMERA', value=ccd.camera, comment='Camera')) version_hdr.add_record( dict(name='EXPNUM', value=im.expnum, comment='Exposure num')) version_hdr.add_record( dict(name='CCDNAME', value=im.ccdname, comment='CCD name')) version_hdr.add_record( dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record( dict(name='PLVER', value=ccd.plver, comment='CP pipeline version')) version_hdr.add_record( dict(name='PLPROCID', value=ccd.plprocid, comment='CP pipeline id')) version_hdr.add_record( dict(name='PROCDATE', value=ccd.procdate, comment='CP image DATE')) keys = [ 'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME' ] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) if opt.save_model or opt.save_data: hdr = fitsio.FITSHDR() tim.getWcs().wcs.add_to_header(hdr) if opt.save_model: fitsio.write(opt.save_model, model_img, header=hdr, clobber=True) print('Wrote', opt.save_model) if opt.save_data: fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True) print('Wrote', opt.save_data) tnow = Time() print('Forced phot:', tnow - tlast) return F, version_hdr, outlier_mask, outlier_header
print(e.message) else: print(e) print() rtn = -1 print('Shutting down MPI pool...') pool.shutdown() print('Shut down MPI pool') return rtn if __name__ == '__main__': from astrometry.util.ttime import Time, MemMeas Time.add_measurement(MemMeas) sys.exit(main()) # salloc -N 2 -C haswell -q interactive -t 04:00:00 --ntasks-per-node=32 --cpus-per-task=2 # module unload cray-mpich # module load openmpi # mpirun -n 64 --map-by core --rank-by node python -m mpi4py.futures legacypipe/mpi-runbrick.py --no-wise-ceres --brick 0715m657 --zoom 100 300 100 300 --run south --outdir $CSCRATCH/mpi --stage wise_forced # cray-mpich version: # srun -n 64 --distribution cyclic:cyclic python -m mpi4py.futures legacypipe/mpi-runbrick.py --no-wise-ceres --brick 0715m657 --zoom 100 300 100 300 --run south --outdir $CSCRATCH/mpi --stage wise_forced # # # mpi4py setup: # cray-mpich module, PrgEnv-intel/6.0.5 # # mpi.cfg:
def main(decals=None, opt=None): '''Driver function for forced photometry of individual DECam images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0,x1,y0,y1) = opt.zoom zoomslice = (slice(y0,y1), slice(x0,x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.filename) opt.filename = None except: # make this 'None' for decals.find_ccds() expnum = None # Try parsing HDU number try: opt.hdu = int(opt.hdu) ccdname = None except: ccdname = opt.hdu opt.hdu = -1 if decals is None: decals = Decals() if opt.filename is not None and opt.hdu >= 0: # Read metadata from file T = exposure_metadata([opt.filename], hdus=[opt.hdu]) print('Metadata:') T.about() else: # Read metadata from decals-ccds.fits table T = decals.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if opt.hdu >= 0: T.cut(T.image_hdu == opt.hdu) print(len(T), 'with HDU', opt.hdu) if opt.filename is not None: T.cut(np.array([f.strip() == opt.filename for f in T.image_filename])) print(len(T), 'with filename', opt.filename) assert(len(T) == 1) im = decals.get_image_object(T[0]) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True) print('Got tim:', tim) if opt.catfn in ['DR1', 'DR2']: if opt.catalog_path is None: opt.catalog_path = opt.catfn.lower() margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, decals=decals) for b in bricks: # there is some overlap with this brick... read the catalog. fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3], 'tractor-%s.fits' % b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec) W,H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) * (yy >= -margin) * (yy <= (H+margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') T.cut((T.out_of_bounds == False) * (T.left_blob == False)) print('Cut to', len(T), 'on out_of_bounds and left_blob') TT.append(T) T = merge_tables(TT) T._header = TT[0]._header del TT # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE) # print('Got cat:', cat) print('Forced photom...') opti = None if opt.ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 opti = CeresOptimizer(BW=B, BH=B) tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') for src in cat: src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) F = fits_table() F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)) ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x-1).astype(np.float32) F.y = (y-1).astype(np.float32) if opt.apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() print('Apertures:', apertures, 'pixels') for rad in apertures: aper = photutils.CircularAperture(apxy, rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = ap ap = 1./(np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux_ivar = ap if opt.forced: kwa = {} if opt.plots is None: kwa.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, **kwa) if opt.plots: (data,mod,ie,chi,roi) = R.ims1[0] ima = tim.ima imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5) plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() F.flux = np.array([src.getBrightness().getFlux(tim.band) for src in cat]).astype(np.float32) F.flux_ivar = R.IV.astype(np.float32) F.fracflux = R.fitstats.profracflux.astype(np.float32) F.rchi2 = R.fitstats.prochi2 .astype(np.float32) program_name = sys.argv[0] version_hdr = get_version_header(program_name, decals.decals_dir) # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) fname = os.path.join(d2, d1, fname) print('Trimmed filename to', fname) #version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file')) version_hdr.add_record(dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file')) version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext')) version_hdr.add_record(dict(name='CAMERA', value='DECam', comment='Dark Energy Camera')) version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num')) version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name')) version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record(dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image')) keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV', 'INSTRUME'] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = {'mjd':'sec', 'exptime':'sec', 'flux':'nanomaggy', 'flux_ivar':'1/nanomaggy^2'} columns = F.get_columns() for i,col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) print('Finished forced phot:', Time()-t0) return 0
def get_queryset(self): print('get_queryset called') t0 = Time() req = self.request self.form = CatalogSearchForm(req.GET) if not self.form.is_valid(): print('FORM NOT VALID!') return [] desc = '' spatial = str(self.form.cleaned_data['spatial']) print('Spatial search type: "%s"' % spatial) print('type', type(spatial)) if spatial == 'cone': ra, dec = parse_coord(self.form.cleaned_data['coord']) rad = self.form.cleaned_data['radius'] if rad is None: rad = 0. rad = max(0., rad) # 1 degree max! # rad = min(rad, 1.) self.radecradius = (ra, dec, rad) print('q3c radial query:', ra, dec, rad) cat = Tractor.objects.extra(where=[ 'q3c_radial_query(tractor.ra, tractor.dec, %.4f, %.4f, %g)' % (ra, dec, rad) ]) desc += 'near RA,Dec = (%.4f, %.4f) radius %f degrees' % (ra, dec, rad) elif spatial == 'allsky': cat = Tractor.objects.all() desc += 'all sky' else: print('Invalid spatial type "%s"' % spatial) return [] terms = [] types = self.form.cleaned_data['sourcetypes'] print('Search for types:', types) if len(types): tt = [] for t in types: if len(t) == 3: t = t + ' ' tt.append(str(t)) if len(tt) == 1: terms.append('Type is %s' % tt[0]) cat = cat.filter(type=tt[0]) else: terms.append('Type in %s' % tt) cat = cat.filter(type__in=tt) for k in ['g', 'r', 'z', 'w1']: #, 'gmr', 'rmz', 'zmw1']: gt = self.form.cleaned_data[k + '_gt'] lt = self.form.cleaned_data[k + '_lt'] #print('Limits for', k, ':', gt, lt) if gt is not None: cat = cat.filter(**{k + '__gt': gt}) if lt is not None: cat = cat.filter(**{k + '__lt': lt}) if gt is not None and lt is not None: terms.append(k + ' between %g and %g' % (gt, lt)) elif gt is not None: terms.append(k + ' > %g' % gt) elif lt is not None: terms.append(k + ' < %g' % lt) for k in ['gmr', 'rmz', 'zmw1']: gt = self.form.cleaned_data[k + '_gt'] lt = self.form.cleaned_data[k + '_lt'] k1, k2 = k.split('m') # DR2 -> DR3 -- "gmr" -> "g_r" if k == 'gmr': dbk = 'g_r' if gt is not None: cat = cat.filter(**{dbk + '__gt': gt}) if lt is not None: cat = cat.filter(**{dbk + '__lt': lt}) else: if gt is not None: cat = cat.extra(where=['(%s - %s) > %f' % (k1, k2, gt)]) if lt is not None: cat = cat.extra(where=['(%s - %s) < %f' % (k1, k2, lt)]) k = '(%s-%s)' % (k1, k2) if gt is not None and lt is not None: terms.append(k + ' between %g and %g' % (gt, lt)) elif gt is not None: terms.append(k + ' > %g' % gt) elif lt is not None: terms.append(k + ' < %g' % lt) if len(terms): desc += ' where ' + ' and '.join(terms) cat = cat[:1000] self.querydesc = desc #print('Got:', cat.count(), 'hits') print('SQL:', cat.query) print('Set query description:', desc) print('Finished get_queryset in', Time() - t0) return cat
def make_coadds(tims, bands, targetwcs, mods=None, xy=None, apertures=None, apxy=None, ngood=False, detmaps=False, psfsize=False, callback=None, callback_args=[], plots=False, ps=None, lanczos=True, mp=None): from astrometry.util.ttime import Time t0 = Time() class Duck(object): pass C = Duck() W = int(targetwcs.get_width()) H = int(targetwcs.get_height()) # always, for patching SATUR, etc pixels? unweighted = True if not xy: psfsize = False C.coimgs = [] if detmaps: C.galdetivs = [] C.detivs = [] if mods is not None: C.comods = [] C.coresids = [] if apertures is not None: unweighted = True C.AP = fits_table() if xy: ix, iy = xy C.T = fits_table() C.T.nobs = np.zeros((len(ix), len(bands)), np.uint8) C.T.anymask = np.zeros((len(ix), len(bands)), np.int16) C.T.allmask = np.zeros((len(ix), len(bands)), np.int16) if psfsize: C.T.psfsize = np.zeros((len(ix), len(bands)), np.float32) if detmaps: C.T.depth = np.zeros((len(ix), len(bands)), np.float32) C.T.galdepth = np.zeros((len(ix), len(bands)), np.float32) if lanczos: print('Doing Lanczos resampling') for tim in tims: # surface-brightness correction tim.sbscale = (targetwcs.pixel_scale() / tim.subwcs.pixel_scale())**2 # We create one iterator per band to do the tim resampling. These all run in # parallel when multi-processing. imaps = [] for band in bands: args = [] for itim, tim in enumerate(tims): if tim.band != band: continue if mods is None: mo = None else: mo = mods[itim] args.append((itim, tim, mo, lanczos, targetwcs)) if mp is not None: imaps.append(mp.imap_unordered(_resample_one, args)) else: import itertools imaps.append(itertools.imap(_resample_one, args)) # Args for aperture photometry apargs = [] tinyw = 1e-30 for iband, (band, timiter) in enumerate(zip(bands, imaps)): print('Computing coadd for band', band) # coadded weight map (moo) cow = np.zeros((H, W), np.float32) # coadded weighted image map cowimg = np.zeros((H, W), np.float32) kwargs = dict(cowimg=cowimg, cow=cow) if detmaps: # detection map inverse-variance (depth map) detiv = np.zeros((H, W), np.float32) C.detivs.append(detiv) kwargs.update(detiv=detiv) # galaxy detection map inverse-variance (galdepth map) galdetiv = np.zeros((H, W), np.float32) C.galdetivs.append(galdetiv) kwargs.update(galdetiv=galdetiv) if mods is not None: # model image cowmod = np.zeros((H, W), np.float32) # chi-squared image cochi2 = np.zeros((H, W), np.float32) kwargs.update(cowmod=cowmod, cochi2=cochi2) if unweighted: # unweighted image coimg = np.zeros((H, W), np.float32) if mods is not None: # unweighted model comod = np.zeros((H, W), np.float32) # number of exposures con = np.zeros((H, W), np.uint8) # inverse-variance coiv = np.zeros((H, W), np.float32) kwargs.update(coimg=coimg, coiv=coiv) # Note that we have 'congood' as well as 'nobs': # * 'congood' is used for the 'nexp' *image*. # * 'nobs' is used for the per-source measurements # # (you want to know the number of observations within the # source footprint, not just the peak pixel which may be # saturated, etc.) if ngood: congood = np.zeros((H, W), np.uint8) kwargs.update(congood=congood) if xy: # These match the type of the "DQ" images. # "any" mask ormask = np.zeros((H, W), np.int16) # "all" mask andmask = np.empty((H, W), np.int16) allbits = reduce(np.bitwise_or, CP_DQ_BITS.values()) andmask[:, :] = allbits # number of observations nobs = np.zeros((H, W), np.uint8) kwargs.update(ormask=ormask, andmask=andmask, nobs=nobs) if psfsize: psfsizemap = np.zeros((H, W), np.float32) for R in timiter: if R is None: continue itim, Yo, Xo, iv, im, mo, dq = R #print('timiter Yo,Xo,im.shape=',Yo,Xo,im.shape) tim = tims[itim] # invvar-weighted image cowimg[Yo, Xo] += iv * im cow[Yo, Xo] += iv if unweighted: if dq is None: goodpix = 1 else: # include BLEED, SATUR, INTERP pixels if no other # pixels exists (do this by eliminating all other CP # flags) badbits = 0 for bitname in ['badpix', 'cr', 'trans', 'edge', 'edge2']: badbits |= CP_DQ_BITS[bitname] goodpix = ((dq & badbits) == 0) coimg[Yo, Xo] += goodpix * im con[Yo, Xo] += goodpix coiv[Yo, Xo] += goodpix * 1. / (tim.sig1 * tim.sbscale)**2 # ...ish if xy: if dq is not None: ormask[Yo, Xo] |= dq andmask[Yo, Xo] &= dq # raw exposure count nobs[Yo, Xo] += 1 if psfsize: # psfnorm is in units of 1/pixels. # (eg, psfnorm for a gaussian is ~ 1/psf_sigma) # Neff is in pixels**2 neff = 1. / tim.psfnorm**2 # Narcsec is in arcsec**2 narcsec = neff * tim.wcs.pixel_scale()**2 psfsizemap[Yo, Xo] += iv * (1. / narcsec) if detmaps: # point-source depth detsig1 = tim.sig1 / tim.psfnorm detiv[Yo, Xo] += (iv > 0) * (1. / detsig1**2) # Galaxy detection map gdetsig1 = tim.sig1 / tim.galnorm galdetiv[Yo, Xo] += (iv > 0) * (1. / gdetsig1**2) if ngood: congood[Yo, Xo] += (iv > 0) if mods is not None: # straight-up comod[Yo, Xo] += goodpix * mo # invvar-weighted cowmod[Yo, Xo] += iv * mo # chi-squared cochi2[Yo, Xo] += iv * (im - mo)**2 del mo del goodpix del Yo, Xo, im, iv # END of loop over tims # Per-band: cowimg /= np.maximum(cow, tinyw) C.coimgs.append(cowimg) if mods is not None: cowmod /= np.maximum(cow, tinyw) C.comods.append(cowmod) coresid = cowimg - cowmod coresid[cow == 0] = 0. C.coresids.append(coresid) if unweighted: coimg /= np.maximum(con, 1) del con cowimg[cow == 0] = coimg[cow == 0] if mods is not None: cowmod[cow == 0] = comod[cow == 0] if xy: C.T.nobs[:, iband] = nobs[iy, ix] C.T.anymask[:, iband] = ormask[iy, ix] C.T.allmask[:, iband] = andmask[iy, ix] # unless there were no images there... C.T.allmask[nobs[iy, ix] == 0, iband] = 0 if detmaps: C.T.depth[:, iband] = detiv[iy, ix] C.T.galdepth[:, iband] = galdetiv[iy, ix] if psfsize: wt = cow[iy, ix] # psfsizemap is in units of iv * (1 / arcsec**2) sz = psfsizemap[iy, ix] sz /= np.maximum(wt, tinyw) sz[wt == 0] = 0. # Back to units of linear arcsec. sz = 1. / np.sqrt(sz) sz[wt == 0] = 0. # Correction factor to get back to equivalent of Gaussian sigma sz /= (2. * np.sqrt(np.pi)) # Conversion factor to FWHM (2.35) sz *= 2. * np.sqrt(2. * np.log(2.)) C.T.psfsize[:, iband] = sz del psfsizemap if apertures is not None: # Aperture photometry, using the unweighted "coimg" and # "coiv" arrays. with np.errstate(divide='ignore'): imsigma = 1.0 / np.sqrt(coiv) imsigma[coiv == 0] = 0 for irad, rad in enumerate(apertures): apargs.append((irad, band, rad, coimg, imsigma, True, apxy)) if mods is not None: apargs.append( (irad, band, rad, coresid, None, False, apxy)) if callback is not None: callback(band, *callback_args, **kwargs) # END of loop over bands t2 = Time() print('coadds: images:', t2 - t0) if apertures is not None: # Aperture phot, in parallel if mp is not None: apresults = mp.map(_apphot_one, apargs) else: apresults = map(_apphot_one, apargs) del apargs apresults = iter(apresults) for iband, band in enumerate(bands): apimg = [] apimgerr = [] if mods is not None: apres = [] for irad, rad in enumerate(apertures): (airad, aband, isimg, ap_img, ap_err) = apresults.next() assert (airad == irad) assert (aband == band) assert (isimg) apimg.append(ap_img) apimgerr.append(ap_err) if mods is not None: (airad, aband, isimg, ap_img, ap_err) = apresults.next() assert (airad == irad) assert (aband == band) assert (not isimg) apres.append(ap_img) assert (ap_err is None) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. C.AP.set('apflux_img_%s' % band, ap) ap = 1. / (np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. C.AP.set('apflux_img_ivar_%s' % band, ap) if mods is not None: ap = np.vstack(apres).T ap[np.logical_not(np.isfinite(ap))] = 0. C.AP.set('apflux_resid_%s' % band, ap) t3 = Time() print('coadds apphot:', t3 - t2) return C
def __init__(self, *args, **kwargs): super(MyMultiproc, self).__init__(*args, **kwargs) self.t0 = Time() self.serial = [] self.parallel = [] self.phases = []
def ptime(text,t0): '''Timer''' tnow=Time() print('TIMING:%s ' % text,tnow-t0) return tnow
def run_one_ccd(survey, catsurvey_north, catsurvey_south, resolve_dec, ccd, opt, zoomslice, ps): tlast = Time() im = survey.get_image_object(ccd) if opt.do_calib: im.run_calibs(splinesky=True) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True, constant_invvar=opt.constant_invvar, hybridPsf=opt.hybrid_psf, normalizePsf=opt.normalize_psf, old_calibs_ok=True) print('Got tim:', tim, 'x0,y0', tim.x0, tim.y0) tnow = Time() print('Read image:', tnow - tlast) tlast = tnow # Apply outlier masks if True: # Outliers masks are computed within a survey (north/south for dr8), and are stored # in a brick-oriented way, in the results directories. north_ccd = (ccd.camera.strip() != 'decam') catsurvey = catsurvey_north if not north_ccd and catsurvey_south is not None: catsurvey = catsurvey_south chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, survey=catsurvey) for b in bricks: from legacypipe.outliers import read_outlier_mask_file print('Reading outlier mask for brick', b.brickname) ok = read_outlier_mask_file(catsurvey, [tim], b.brickname, subimage=False, output=False, ps=ps) if not ok: print('WARNING: failed to read outliers mask file for brick', b.brickname) if opt.catalog: T = fits_table(opt.catalog) else: chipwcs = tim.subwcs T = get_catalog_in_wcs(chipwcs, catsurvey_north, catsurvey_south=catsurvey_south, resolve_dec=resolve_dec) if T is None: print('No sources to photometer.') return None if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) surveydir = survey.get_survey_dir() del survey if opt.move_gaia: # Gaia stars: move RA,Dec to the epoch of this image. I = np.flatnonzero(T.ref_epoch > 0) if len(I): from legacypipe.survey import radec_at_mjd print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd()) ra, dec = radec_at_mjd(T.ra[I], T.dec[I], T.ref_epoch[I].astype(float), T.pmra[I], T.pmdec[I], T.parallax[I], tim.time.toMjd()) T.ra[I] = ra T.dec[I] = dec tnow = Time() print('Read catalog:', tnow - tlast) tlast = tnow cat = read_fits_catalog(T, bands='r') # Replace the brightness (which will be a NanoMaggies with g,r,z) # with a NanoMaggies with this image's band only. for src in cat: src.brightness = NanoMaggies(**{tim.band: 1.}) tnow = Time() print('Parse catalog:', tnow - tlast) tlast = tnow print('Forced photom...') F = run_forced_phot(cat, tim, ceres=opt.ceres, derivs=opt.derivs, fixed_also=True, agn=opt.agn, do_forced=opt.forced, do_apphot=opt.apphot, get_model=opt.save_model, ps=ps, timing=True, ceres_threads=opt.ceres_threads) if opt.save_model: # unpack results F, model_img = F F.release = T.release F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.camera = np.array([ccd.camera] * len(F)) F.expnum = np.array([im.expnum] * len(F)).astype(np.int64) F.ccdname = np.array([im.ccdname] * len(F)) # "Denormalizing" F.filter = np.array([tim.band] * len(F)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(F)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(F)).astype(np.float32) F.psfsize = np.array([tim.psf_fwhm * tim.imobj.pixscale] * len(F)).astype( np.float32) F.ccd_cuts = np.array([ccd.ccd_cuts] * len(F)) F.airmass = np.array([ccd.airmass] * len(F)) ### --> also add units to the dict below so the FITS headers have units F.sky = np.array([tim.midsky / tim.zpscale / tim.imobj.pixscale**2] * len(F)).astype(np.float32) # in the same units as the depth maps -- flux inverse-variance. F.psfdepth = np.array([(1. / (tim.sig1 / tim.psfnorm)**2)] * len(F)).astype(np.float32) F.galdepth = np.array([(1. / (tim.sig1 / tim.galnorm)**2)] * len(F)).astype(np.float32) # F.psfdepth = np.array([-2.5 * (np.log10(5. * tim.sig1 / tim.psfnorm) - 9)] * len(F)).astype(np.float32) # F.galdepth = np.array([-2.5 * (np.log10(5. * tim.sig1 / tim.galnorm) - 9)] * len(F)).astype(np.float32) # super units questions here if opt.derivs: cosdec = np.cos(np.deg2rad(T.dec)) F.dra = (F.flux_dra / F.flux) * 3600. / cosdec F.ddec = (F.flux_ddec / F.flux) * 3600. F.dra_ivar = F.flux_dra_ivar * (F.flux / 3600. * cosdec)**2 F.ddec_ivar = F.flux_ddec_ivar * (F.flux / 3600.)**2 F.delete_column('flux_dra') F.delete_column('flux_ddec') F.delete_column('flux_dra_ivar') F.delete_column('flux_ddec_ivar') F.flux = F.flux_fixed F.flux_ivar = F.flux_fixed_ivar F.delete_column('flux_fixed') F.delete_column('flux_fixed_ivar') for c in ['dra', 'ddec', 'dra_ivar', 'ddec_ivar', 'flux', 'flux_ivar']: F.set(c, F.get(c).astype(np.float32)) F.ra = T.ra F.dec = T.dec ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x - 1).astype(np.float32) F.y = (y - 1).astype(np.float32) h, w = tim.shape F.dqmask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h - 1), np.clip(np.round(F.x).astype(int), 0, w - 1)] program_name = sys.argv[0] ## FIXME -- from catalog? release = 8002 version_hdr = get_version_header(program_name, surveydir, release) filename = getattr(ccd, 'image_filename') if filename is None: # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn.strip()) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) filename = os.path.join(d2, d1, fname) print('Trimmed filename to', filename) version_hdr.add_record( dict(name='CPFILE', value=filename, comment='CP file')) version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext')) version_hdr.add_record( dict(name='CAMERA', value=ccd.camera, comment='Camera')) version_hdr.add_record( dict(name='EXPNUM', value=im.expnum, comment='Exposure num')) version_hdr.add_record( dict(name='CCDNAME', value=im.ccdname, comment='CCD name')) version_hdr.add_record( dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record( dict(name='PLVER', value=ccd.plver, comment='CP pipeline version')) version_hdr.add_record( dict(name='PLPROCID', value=ccd.plprocid, comment='CP pipeline id')) version_hdr.add_record( dict(name='PROCDATE', value=ccd.procdate, comment='CP image DATE')) keys = [ 'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME' ] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) if opt.save_model or opt.save_data: hdr = fitsio.FITSHDR() tim.getWcs().wcs.add_to_header(hdr) if opt.save_model: fitsio.write(opt.save_model, model_img, header=hdr, clobber=True) print('Wrote', opt.save_model) if opt.save_data: fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True) print('Wrote', opt.save_data) tnow = Time() print('Forced phot:', tnow - tlast) return F, version_hdr
def unwise_forcedphot(cat, tiles, bands=[1, 2, 3, 4], roiradecbox=None, unwise_dir='.', use_ceres=True, ceres_block=8, save_fits=False, ps=None, psf_broadening=None): ''' Given a list of tractor sources *cat* and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id) runs forced photometry, returning a FITS table the same length as *cat*. ''' # Severely limit sizes of models for src in cat: if isinstance(src, PointSource): src.fixedRadius = 10 else: src.halfsize = 10 wantims = ((ps is not None) or save_fits) wanyband = 'w' fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix', 'pronexp'] Nsrcs = len(cat) phot = fits_table() phot.tile = np.array([' '] * Nsrcs) ra = np.array([src.getPosition().ra for src in cat]) dec = np.array([src.getPosition().dec for src in cat]) for band in bands: print('Photometering WISE band', band) wband = 'w%i' % band # The tiles have some overlap, so for each source, keep the # fit in the tile whose center is closest to the source. tiledists = np.empty(Nsrcs) tiledists[:] = 1e100 flux_invvars = np.zeros(Nsrcs, np.float32) fitstats = dict([(k, np.zeros(Nsrcs, np.float32)) for k in fskeys]) nexp = np.zeros(Nsrcs, np.int16) mjd = np.zeros(Nsrcs, np.float64) for tile in tiles: print('Reading tile', tile.coadd_id) tim = get_unwise_tractor_image(unwise_dir, tile.coadd_id, band, bandname=wanyband, roiradecbox=roiradecbox) if tim is None: print('Actually, no overlap with tile', tile.coadd_id) continue if psf_broadening is not None: # psf_broadening is a factor by which the PSF FWHMs # should be scaled; the PSF is a little wider # post-reactivation. psf = tim.getPsf() from tractor import GaussianMixturePSF if isinstance(psf, GaussianMixturePSF): # print('Broadening PSF: from', psf) p0 = psf.getParams() #print('Params:', p0) pnames = psf.getParamNames() #print('Param names:', pnames) p1 = [p * psf_broadening**2 if 'var' in name else p for (p, name) in zip(p0, pnames)] #print('Broadened:', p1) psf.setParams(p1) print('Broadened PSF:', psf) else: print( 'WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf)) print('Read image with shape', tim.shape) # Select sources in play. wcs = tim.wcs.wcs H, W = tim.shape ok, x, y = wcs.radec2pixelxy(ra, dec) x = (x - 1.).astype(np.float32) y = (y - 1.).astype(np.float32) margin = 10. I = np.flatnonzero((x >= -margin) * (x < W + margin) * (y >= -margin) * (y < H + margin)) print(len(I), 'within the image + margin') inbox = ((x[I] >= -0.5) * (x[I] < (W - 0.5)) * (y[I] >= -0.5) * (y[I] < (H - 0.5))) print(sum(inbox), 'strictly within the image') # Compute L_inf distance to (full) tile center. tilewcs = unwise_tile_wcs(tile.ra, tile.dec) cx, cy = tilewcs.crpix ok, tx, ty = tilewcs.radec2pixelxy(ra[I], dec[I]) td = np.maximum(np.abs(tx - cx), np.abs(ty - cy)) closest = (td < tiledists[I]) tiledists[I[closest]] = td[closest] keep = inbox * closest # Source indices (in the full "cat") to keep (the fit values for) srci = I[keep] if not len(srci): print('No sources to be kept; skipping.') continue phot.tile[srci] = tile.coadd_id nexp[srci] = tim.nuims[np.clip(np.round(y[srci]).astype(int), 0, H - 1), np.clip(np.round(x[srci]).astype(int), 0, W - 1)] # Source indices in the margins margi = I[np.logical_not(keep)] # sources in the box -- at the start of the subcat list. subcat = [cat[i] for i in srci] # include *copies* of sources in the margins # (that way we automatically don't save the results) subcat.extend([cat[i].copy() for i in margi]) assert(len(subcat) == len(I)) # FIXME -- set source radii, ...? minsb = 0. fitsky = False # Look in image and set radius based on peak height?? tractor = Tractor([tim], subcat) if use_ceres: from tractor.ceres_optimizer import CeresOptimizer tractor.optimizer = CeresOptimizer(BW=ceres_block, BH=ceres_block) tractor.freezeParamsRecursive('*') tractor.thawPathsTo(wanyband) kwa = dict(fitstat_extras=[('pronexp', [tim.nims])]) t0 = Time() R = tractor.optimize_forced_photometry( minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True, variance=True, shared_params=False, wantims=wantims, **kwa) print('unWISE forced photometry took', Time() - t0) if use_ceres: term = R.ceres_status['termination'] print('Ceres termination status:', term) # Running out of memory can cause failure to converge # and term status = 2. # Fail completely in this case. if term != 0: raise RuntimeError( 'Ceres terminated with status %i' % term) if wantims: ims0 = R.ims0 ims1 = R.ims1 IV, fs = R.IV, R.fitstats if save_fits: import fitsio (dat, mod, ie, chi, roi) = ims1[0] wcshdr = fitsio.FITSHDR() tim.wcs.wcs.add_to_header(wcshdr) tag = 'fit-%s-w%i' % (tile.coadd_id, band) fitsio.write('%s-data.fits' % tag, dat, clobber=True, header=wcshdr) fitsio.write('%s-mod.fits' % tag, mod, clobber=True, header=wcshdr) fitsio.write('%s-chi.fits' % tag, chi, clobber=True, header=wcshdr) if ps: tag = '%s W%i' % (tile.coadd_id, band) (dat, mod, ie, chi, roi) = ims1[0] sig1 = tim.sig1 plt.clf() plt.imshow(dat, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() plt.title('%s: data' % tag) ps.savefig() plt.clf() plt.imshow(mod, interpolation='nearest', origin='lower', cmap='gray', vmin=-3 * sig1, vmax=10 * sig1) plt.colorbar() plt.title('%s: model' % tag) ps.savefig() plt.clf() plt.imshow(chi, interpolation='nearest', origin='lower', cmap='gray', vmin=-5, vmax=+5) plt.colorbar() plt.title('%s: chi' % tag) ps.savefig() # Save results for this tile. # the "keep" sources are at the beginning of the "subcat" list flux_invvars[srci] = IV[:len(srci)].astype(np.float32) if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'): mjd[srci] = (tim.mjdmin + tim.mjdmax) / 2. if fs is None: continue for k in fskeys: x = getattr(fs, k) # fitstats are returned only for un-frozen sources fitstats[k][srci] = np.array(x).astype(np.float32)[:len(srci)] # Note, this is *outside* the loop over tiles. # The fluxes are saved in the source objects, and will be set based on # the 'tiledists' logic above. nm = np.array([src.getBrightness().getBand(wanyband) for src in cat]) nm_ivar = flux_invvars # Sources out of bounds, eg, never change from their default # (1-sigma or whatever) initial fluxes. Zero them out instead. nm[nm_ivar == 0] = 0. phot.set(wband + '_nanomaggies', nm.astype(np.float32)) phot.set(wband + '_nanomaggies_ivar', nm_ivar) dnm = np.zeros(len(nm_ivar), np.float32) okiv = (nm_ivar > 0) dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32) okflux = (nm > 0) mag = np.zeros(len(nm), np.float32) mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux]) ).astype(np.float32) dmag = np.zeros(len(nm), np.float32) ok = (okiv * okflux) dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok]) ).astype(np.float32) mag[np.logical_not(okflux)] = np.nan dmag[np.logical_not(ok)] = np.nan phot.set(wband + '_mag', mag) phot.set(wband + '_mag_err', dmag) for k in fskeys: phot.set(wband + '_' + k, fitstats[k]) phot.set(wband + '_nexp', nexp) if not np.all(mjd == 0): phot.set(wband + '_mjd', mjd) return phot
def main(args=None): """Main routine which parses the optional inputs.""" t0= Time() # Command line options if args is None: # Read from cmd line parser= get_parser() args = parser.parse_args(args=args) else: # args is already a argparse.Namespace obj pass # Print calling sequence print('Args:', args) if args.do_more == 'yes': assert(not args.minid is None) # Setup loggers if args.verbose: lvl = logging.DEBUG else: lvl = logging.INFO logging.basicConfig(level=lvl, stream=sys.stdout) #,format='%(message)s') log = logging.getLogger('decals_sim') # Sort through args #log.info('decals_sim.py args={}'.format(args)) #max_nobj=500 #max_nchunk=1000 #if args.ith_chunk is not None: assert(args.ith_chunk <= max_nchunk-1) #assert(args.nchunk <= max_nchunk) #assert(args.nobj <= max_nobj) #if args.ith_chunk is not None: # assert(args.nchunk == 1) #if choose a chunk, only doing 1 chunk if args.nobj is None: parser.print_help() sys.exit(1) # Exit if expected output already exists rsdir= get_outdir_runbrick(args.outdir, args.brick,args.rowstart, do_skipids=args.do_skipids, do_more=args.do_more) rsdir= os.path.basename(rsdir) tractor_fn= os.path.join(args.outdir, 'tractor',args.brick[:3],args.brick, rsdir, 'tractor-%s.fits' % args.brick) if (os.path.exists(tractor_fn) & (not args.overwrite_if_exists)): print('Exiting, already finished %s' % tractor_fn) return 0 #sys.exit(0) brickname = args.brick objtype = args.objtype # Output dir decals_sim_dir = args.outdir #nchunk = args.nchunk #rand = np.random.RandomState(args.seed) # determines seed for all chunks #seeds = rand.random_integers(0,2**18, max_nchunk) log.info('Object type = {}'.format(objtype)) #log.info('Number of objects = {}'.format(nobj)) #log.info('Number of chunks = {}'.format(nchunk)) # Optionally zoom into a portion of the brick survey = LegacySurveyData(survey_dir=args.survey_dir) brickinfo= get_brickinfo_hack(survey,brickname) #brickinfo = survey.get_brick_by_name(brickname) #print(brickname) brickwcs = wcs_for_brick(brickinfo) W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale() log.info('Brick = {}'.format(brickname)) if args.zoom is not None: # See also runbrick.stage_tims() (x0, x1, y0, y1) = args.zoom W = x1 - x0 H = y1 - y0 brickwcs = brickwcs.get_subimage(x0, y0, W, H) log.info('Zoom (pixel boundaries) = {}'.format(args.zoom)) targetrd = np.array([brickwcs.pixelxy2radec(x, y) for x, y in [(1,1), (W,1), (W,H), (1,H), (1,1)]]) radec_center = brickwcs.radec_center() log.info('RA, Dec center = {}'.format(radec_center)) log.info('Brick = {}'.format(brickname)) t0= ptime('First part of Main()',t0) # SAMPLE table sample_kwargs= {"objtype":args.objtype, "brick":args.brick, "outdir":args.outdir, "randoms_db":args.randoms_db, "minid":args.minid, "do_skipids":args.do_skipids, "randoms_from_fits":args.randoms_from_fits, "dont_sort_sampleid":args.dont_sort_sampleid} Samp,seed= get_sample(**sample_kwargs) Samp= Samp[args.rowstart:args.rowstart + args.nobj] # Performance #if objtype in ['elg','lrg']: # Samp=Samp[np.argsort( Samp.get('%s_n' % objtype) )] print('Max sample size=%d, actual sample size=%d' % (args.nobj,len(Samp))) assert(len(Samp) <= args.nobj) t0= ptime('Got randoms sample',t0) # Store args in dict for easy func passing kwargs=dict(Samp=Samp,\ brickname=brickname, \ checkpoint=args.checkpoint, \ seed= seed, decals_sim_dir= decals_sim_dir,\ brickwcs= brickwcs, \ objtype=objtype,\ nobj=len(Samp),\ maxobjs=args.nobj,\ rowst=args.rowstart,\ do_skipids=args.do_skipids,\ do_more=args.do_more,\ minid=args.minid,\ survey_dir=args.survey_dir,\ args=args) # Stop if starting row exceeds length of radec,color table if len(Samp) == 0: fn= get_outdir_runbrick(kwargs['decals_sim_dir'], kwargs['brickname'],kwargs['rowst'], do_skipids=kwargs['do_skipids'],do_more=kwargs['do_more']) fn+= '_exceeded.txt' junk= os.system('touch %s' % fn) print('Wrote %s' % fn) #we want not to add any sample -- obiwan #raise ValueError('starting row=%d exceeds number of artificial sources, quit' % args.rowstart) # Create simulated catalogues and run Tractor create_metadata(kwargs=kwargs) t0= ptime('create_metadata',t0) # do chunks #for ith_chunk in chunk_list: #log.info('Working on chunk {:02d}/{:02d}'.format(ith_chunk,kwargs['nchunk']-1)) # Random ra,dec and source properties create_ith_simcat(d=kwargs) #log.info('HUI-TEST:::out of create_ith_simcat') t0= ptime('create_ith_simcat',t0) # Run tractor #log.info('HUI-TEST:::running tractor') do_one_chunk(d=kwargs) #log.info('HUI-TEST::: checkpoint3i') t0= ptime('do_one_chunk',t0) # Clean up output if args.no_cleanup == False: do_ith_cleanup(d=kwargs) #log.info('HUI-TEST::: checkpoint3j') t0= ptime('do_ith_cleanup',t0) log.info('All done!') #log.info('HUI-TEST::: checkpoint3k') return 0
def main(survey=None, opt=None, args=None): '''Driver function for forced photometry of individual Legacy Survey images. ''' if args is None: args = sys.argv[1:] print('forced_photom.py', ' '.join(args)) if opt is None: parser = get_parser() opt = parser.parse_args(args) import logging if opt.verbose == 0: lvl = logging.INFO else: lvl = logging.DEBUG logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout) # tractor logging is *soooo* chatty logging.getLogger('tractor.engine').setLevel(lvl + 10) t0 = Time() if survey is None: survey = LegacySurveyData(survey_dir=opt.survey_dir, cache_dir=opt.cache_dir, output_dir=opt.out_dir) if opt.skip: if opt.out is not None: outfn = opt.out else: outfn = survey.find_file('forced', output=True, camera=opt.camera, expnum=opt.expnum) if os.path.exists(outfn): print('Ouput file exists:', outfn) return 0 if opt.derivs and opt.agn: print('Sorry, can\'t do --derivs AND --agn') return -1 if opt.out is None and opt.out_dir is None: print('Must supply either --out or --out-dir') return -1 if opt.expnum is None and opt.out is None: print('If no --expnum is given, must supply --out filename') return -1 if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0, x1, y0, y1) = opt.zoom zoomslice = (slice(y0, y1), slice(x0, x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Cache CCDs files before the find_ccds call... # Copy required files into the cache? if opt.pre_cache: def copy_files_to_cache(fns): for fn in fns: cachefn = fn.replace(survey.survey_dir, survey.cache_dir) if not cachefn.startswith(survey.cache_dir): print('Skipping', fn) continue outdir = os.path.dirname(cachefn) trymakedirs(outdir) print('Copy', fn) print(' to', cachefn) shutil.copyfile(fn, cachefn) assert (survey.cache_dir is not None) fnset = set() fn = survey.find_file('bricks') fnset.add(fn) fns = survey.find_file('ccd-kds') fnset.update(fns) copy_files_to_cache(fnset) # Read metadata from survey-ccds.fits table ccds = survey.find_ccds(camera=opt.camera, expnum=opt.expnum, ccdname=opt.ccdname) print(len(ccds), 'with camera', opt.camera, 'and expnum', opt.expnum, 'and ccdname', opt.ccdname) # sort CCDs ccds.cut(np.lexsort((ccds.ccdname, ccds.expnum, ccds.camera))) # If there is only one catalog survey_dir, we pass it to get_catalog_in_wcs # as the northern survey. catsurvey_north = survey catsurvey_south = None if opt.catalog_dir_north is not None: assert (opt.catalog_dir_south is not None) assert (opt.catalog_resolve_dec_ngc is not None) catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north) catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south) elif opt.catalog_dir is not None: catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir) # Copy required CCD & calib files into the cache? if opt.pre_cache: assert (survey.cache_dir is not None) fnset = set() for ccd in ccds: im = survey.get_image_object(ccd) for key in im.get_cacheable_filename_variables(): fn = getattr(im, key) if fn is None or not (os.path.exists(fn)): continue fnset.add(fn) copy_files_to_cache(fnset) args = [] for ccd in ccds: args.append((survey, catsurvey_north, catsurvey_south, opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps)) if opt.threads: from astrometry.util.multiproc import multiproc from astrometry.util.timingpool import TimingPool, TimingPoolMeas pool = TimingPool(opt.threads) poolmeas = TimingPoolMeas(pool, pickleTraffic=False) Time.add_measurement(poolmeas) mp = multiproc(None, pool=pool) tm = Time() FF = mp.map(bounce_one_ccd, args) print('Multi-processing forced-phot:', Time() - tm) del mp Time.measurements.remove(poolmeas) del poolmeas pool.close() pool.join() del pool else: FF = map(bounce_one_ccd, args) FF = [F for F in FF if F is not None] if len(FF) == 0: print('No photometry results to write.') return 0 # Keep only the first header _, version_hdr, _, _ = FF[0] # unpack results outlier_masks = [m for _, _, m, _ in FF] outlier_hdrs = [h for _, _, _, h in FF] FF = [F for F, _, _, _ in FF] F = merge_tables(FF) if len(ccds): version_hdr.delete('CPHDU') version_hdr.delete('CCDNAME') from legacypipe.utils import add_bits from legacypipe.bits import DQ_BITS add_bits(version_hdr, DQ_BITS, 'DQMASK', 'DQ', 'D') from legacyzpts.psfzpt_cuts import CCD_CUT_BITS add_bits(version_hdr, CCD_CUT_BITS, 'CCD_CUTS', 'CC', 'C') for i, ap in enumerate(apertures_arcsec): version_hdr.add_record( dict(name='APRAD%i' % i, value=ap, comment='(optical) Aperture radius, in arcsec')) unitmap = { 'exptime': 'sec', 'flux': 'nanomaggy', 'flux_ivar': '1/nanomaggy^2', 'apflux': 'nanomaggy', 'apflux_ivar': '1/nanomaggy^2', 'psfdepth': '1/nanomaggy^2', 'galdepth': '1/nanomaggy^2', 'sky': 'nanomaggy/arcsec^2', 'psfsize': 'arcsec', 'fwhm': 'pixels', 'ccdrarms': 'arcsec', 'ccddecrms': 'arcsec', 'ra': 'deg', 'dec': 'deg', 'skyrms': 'counts/sec', 'dra': 'arcsec', 'ddec': 'arcsec', 'dra_ivar': '1/arcsec^2', 'ddec_ivar': '1/arcsec^2' } columns = F.get_columns() order = [ 'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum', 'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'fwhm', 'ccd_cuts', 'airmass', 'sky', 'skyrms', 'psfdepth', 'galdepth', 'ccdzpt', 'ccdrarms', 'ccddecrms', 'ccdphrms', 'ra', 'dec', 'flux', 'flux_ivar', 'fracflux', 'rchisq', 'fracmasked', 'fracin', 'apflux', 'apflux_ivar', 'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar', 'ddec_ivar' ] columns = [c for c in order if c in columns] units = [unitmap.get(c, '') for c in columns] if opt.out is not None: outdir = os.path.dirname(opt.out) if len(outdir): trymakedirs(outdir) tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.out)) fitsio.write(tmpfn, None, header=version_hdr, clobber=True) F.writeto(tmpfn, units=units, append=True, columns=columns) os.rename(tmpfn, opt.out) print('Wrote', opt.out) else: with survey.write_output('forced', camera=opt.camera, expnum=opt.expnum) as out: F.writeto(None, fits_object=out.fits, primheader=version_hdr, units=units, columns=columns) print('Wrote', out.real_fn) if opt.outlier_mask is not None: # Add outlier bit meanings to the primary header version_hdr.add_record( dict(name='COMMENT', value='Outlier mask bit meanings')) version_hdr.add_record( dict(name='OUTL_POS', value=1, comment='Outlier mask bit for Positive outlier')) version_hdr.add_record( dict(name='OUTL_NEG', value=2, comment='Outlier mask bit for Negative outlier')) if opt.outlier_mask == 'default': outdir = os.path.join(opt.out_dir, 'outlier-masks') camexp = set(zip(ccds.camera, ccds.expnum)) for c, e in camexp: I = np.flatnonzero((ccds.camera == c) * (ccds.expnum == e)) ccd = ccds[I[0]] imfn = ccd.image_filename.strip() outfn = os.path.join(outdir, imfn.replace('.fits', '-outlier.fits')) trymakedirs(outfn, dir=True) tempfn = outfn.replace('.fits', '-tmp.fits') with fitsio.FITS(tempfn, 'rw', clobber=True) as fits: fits.write(None, header=version_hdr) for i in I: mask = outlier_masks[i] _, _, _, meth, tile = survey.get_compression_args( 'outliers_mask', shape=mask.shape) fits.write(mask, header=outlier_hdrs[i], extname=ccds.ccdname[i], compress=meth, tile_dims=tile) os.rename(tempfn, outfn) print('Wrote', outfn) elif opt.outlier_mask is not None: with fitsio.FITS(opt.outlier_mask, 'rw', clobber=True) as F: F.write(None, header=version_hdr) for i, (hdr, mask) in enumerate(zip(outlier_hdrs, outlier_masks)): _, _, _, meth, tile = survey.get_compression_args( 'outliers_mask', shape=mask.shape) F.write(mask, header=hdr, extname=ccds.ccdname[i], compress=meth, tile_dims=tile) print('Wrote', opt.outlier_mask) tnow = Time() print('Total:', tnow - t0) return 0
def get_tractor_image(self, **kwargs): #t0 = time_builtin.clock() tim = super(SimImage, self).get_tractor_image(**kwargs) #debug: setting image to 0 by hui #tim.data = np.zeros(tim.data.shape) #print('get_tractor_image:time'+str(time_builtin.clock()-t0)) if tim is None: # this can be None when the edge of a CCD overlaps return tim # Seed #if 'SEED' in self.survey.metacat.columns: # seed = self.survey.metacat['SEED'] #else: # seed = None t1=time_builtin.clock() objtype = self.survey.metacat.get('objtype')[0] objstamp = BuildStamp(tim, seed=self.survey.seed, camera=self.t.camera, gain=self.t.gain,exptime=self.t.exptime) # ids make it onto a ccd (geometry cut) tim.ids_added=[] # Grab the data and inverse variance images [nanomaggies!] tim_image = galsim.Image(tim.getImage()) tim_invvar = galsim.Image(tim.getInvvar()) tim_dq = galsim.Image(tim.dq) # Also store galaxy sims and sims invvar sims_image = tim_image.copy() sims_image.fill(0.0) sims_ivar = sims_image.copy() # Store simulated galaxy images in tim object # Loop on each object. for ii, obj in enumerate(self.survey.simcat): # Print timing t0= Time() if objtype in ['lrg','elg']: strin= 'Drawing 1 %s: n=%.2f, rhalf=%.2f, e1=%.2f, e2=%.2f' % \ (objtype.upper(), obj.n,obj.rhalf,obj.e1,obj.e2) print(strin) if objtype == 'star': stamp = objstamp.star(obj) elif objtype == 'elg': stamp = objstamp.elg(obj) elif objtype == 'lrg': stamp = objstamp.lrg(obj) elif objtype == 'qso': stamp = objstamp.qso(obj) t0= ptime('Finished Drawing %s: id=%d band=%s dbflux=%f addedflux=%f' % (objtype.upper(), obj.id,objstamp.band, obj.get(objstamp.band+'flux'),stamp.array.sum()), t0) stamp_nonoise= stamp.copy() if self.survey.add_sim_noise: stamp += noise_for_galaxy(stamp,objstamp.nano2e) ivarstamp= ivar_for_galaxy(stamp,objstamp.nano2e) # Add source if EVEN 1 pix falls on the CCD overlap = stamp.bounds & tim_image.bounds if overlap.area() > 0: print('Stamp overlaps tim: id=%d band=%s' % (obj.id,objstamp.band)) tim.ids_added.append(obj.id) stamp = stamp[overlap] ivarstamp = ivarstamp[overlap] stamp_nonoise= stamp_nonoise[overlap] # Zero out invvar where bad pixel mask is flagged (> 0) keep = np.ones(tim_dq[overlap].array.shape) keep[ tim_dq[overlap].array > 0 ] = 0. ivarstamp *= keep # Add stamp to image back= tim_image[overlap].copy() tim_image[overlap] += stamp #= back.copy() + stamp.copy() # Add variances back_ivar= tim_invvar[overlap].copy() tot_ivar= get_srcimg_invvar(ivarstamp, back_ivar) tim_invvar[overlap] = tot_ivar.copy() #Extra sims_image[overlap] += stamp.copy() sims_ivar[overlap] += ivarstamp.copy() if np.min(sims_ivar.array) < 0: log.warning('Negative invvar!') import pdb ; pdb.set_trace() tim.sims_image = sims_image.array tim.sims_inverr = np.sqrt(sims_ivar.array) # Can set image=model, ivar=1/model for testing if self.survey.image_eq_model: tim.data = sims_image.array.copy() tim.inverr = np.zeros(tim.data.shape) tim.inverr[sims_image.array > 0.] = np.sqrt(1./sims_image.array.copy()[sims_image.array > 0.]) else: tim.data = tim_image.array tim.inverr = np.sqrt(tim_invvar.array) sys.stdout.flush() #print('get_tractor_image in obiwan :time '+str(time_builtin.clock()-t1)) return tim
def _ceres_forced_photom(self, tractor, result, umodels, imlist, mods0, scales, skyderivs, minFlux, nonneg=False, wantims0=True, wantims1=True, negfluxval=None, verbose=False, **kwargs): ''' negfluxval: when 'nonneg' is set, the flux value to give sources that went negative in an unconstrained fit. ''' from tractor.ceres import ceres_forced_phot t0 = Time() blocks = [] blockstart = {} usedParamMap = {} nextparam = 0 # umodels[ imagei, srci ] = Patch Nsky = 0 Z = [] if skyderivs is not None: # skyderivs = [ (param0:)[ (deriv,img), ], (param1:)[ (deriv,img), ], ...] # Reorg them to be in img-major order skymods = [[] for im in imlist] for skyd in skyderivs: for (deriv, img) in skyd: imi = imlist.index(img) skymods[imi].append(deriv) for mods, im, mod0 in zip(skymods, imlist, mods0): Z.append((mods, im, 1., mod0, Nsky)) Nsky += len(mods) Z.extend( zip(umodels, imlist, scales, mods0, np.zeros(len(imlist), int) + Nsky)) sky = (skyderivs is not None) for zi, (umods, img, scale, mod0, paramoffset) in enumerate(Z): H, W = img.shape if img in blockstart: (b0, nbw, nbh) = blockstart[img] else: # Dice up the image nbw = int(np.ceil(W / float(self.BW))) nbh = int(np.ceil(H / float(self.BH))) b0 = len(blocks) blockstart[img] = (b0, nbw, nbh) for iy in range(nbh): for ix in range(nbw): x0 = ix * self.BW y0 = iy * self.BH slc = (slice(y0, min(y0 + self.BH, H)), slice(x0, min(x0 + self.BW, W))) data = (x0, y0, img.getImage()[slc].astype(self.ceresType), mod0[slc].astype(self.ceresType), img.getInvError()[slc].astype(self.ceresType)) blocks.append((data, [])) for modi, umod in enumerate(umods): if umod is None: continue # DEBUG if len(umod.shape) != 2: print('zi', zi) print('modi', modi) print('umod', umod) umod.clipTo(W, H) umod.trimToNonZero() if umod.patch is None: continue # Dice up the model ph, pw = umod.shape bx0 = np.clip(int(np.floor(umod.x0 / float(self.BW))), 0, nbw - 1) bx1 = np.clip(int(np.ceil((umod.x0 + pw) / float(self.BW))), 0, nbw - 1) by0 = np.clip(int(np.floor(umod.y0 / float(self.BH))), 0, nbh - 1) by1 = np.clip(int(np.ceil((umod.y0 + ph) / float(self.BH))), 0, nbh - 1) parami = paramoffset + modi if parami in usedParamMap: ceresparam = usedParamMap[parami] else: usedParamMap[parami] = nextparam ceresparam = nextparam nextparam += 1 cmod = (umod.patch * scale).astype(self.ceresType) for by in range(by0, by1 + 1): for bx in range(bx0, bx1 + 1): bi = by * nbw + bx # if type(umod.x0) != int or type(umod.y0) != int: # print('umod:', umod.x0, umod.y0, type(umod.x0), type(umod.y0)) # print('umod:', umod) dd = (ceresparam, int(umod.x0), int(umod.y0), cmod) blocks[b0 + bi][1].append(dd) logverb('forced phot: dicing up', Time() - t0) if wantims0: t0 = Time() params = tractor.getParams() result.ims0 = self._getims(params, imlist, umodels, mods0, scales, sky, minFlux, None) logverb('forced phot: ims0', Time() - t0) t0 = Time() fluxes = np.zeros(len(usedParamMap)) logverb('Ceres forced phot:') logverb(len(blocks), ('image blocks (%ix%i), %i params' % (self.BW, self.BH, len(fluxes)))) if len(blocks) == 0 or len(fluxes) == 0: logverb('Nothing to do!') return # init fluxes passed to ceres p0 = tractor.getParams() for i, k in usedParamMap.items(): fluxes[k] = p0[i] iverbose = 1 if verbose else 0 nonneg = int(nonneg) ithreads = 0 if self.threads is not None: ithreads = int(self.threads) if nonneg: # Initial run with nonneg=False, to get in the ballpark x = ceres_forced_phot(blocks, fluxes, 0, iverbose, ithreads) assert (x == 0) logverb('forced phot: ceres initial run', Time() - t0) t0 = Time() if negfluxval is not None: fluxes = np.maximum(fluxes, negfluxval) x = ceres_forced_phot(blocks, fluxes, nonneg, iverbose, ithreads) #print('Ceres forced phot:', x) logverb('forced phot: ceres', Time() - t0) t0 = Time() params = np.zeros(len(p0)) for i, k in usedParamMap.items(): params[i] = fluxes[k] tractor.setParams(params) logverb('forced phot: unmapping params:', Time() - t0) if wantims1: t0 = Time() result.ims1 = self._getims(params, imlist, umodels, mods0, scales, sky, minFlux, None) logverb('forced phot: ims1:', Time() - t0) return x
def main(survey=None, opt=None): print(' '.join(sys.argv)) '''Driver function for forced photometry of individual Legacy Survey images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = tlast = Time() if opt.skip and os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if opt.derivs and opt.agn: print('Sorry, can\'t do --derivs AND --agn') sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0, x1, y0, y1) = opt.zoom zoomslice = (slice(y0, y1), slice(x0, x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing first arg as exposure number (otherwise, it's a filename) try: expnum = int(opt.expnum) filename = None except: # make this 'None' for survey.find_ccds() expnum = None filename = opt.expnum # Try parsing HDU: "all" or HDU name or HDU number. all_hdus = (opt.ccdname == 'all') hdu = -1 ccdname = None if not all_hdus: try: hdu = int(opt.ccdname) except: ccdname = opt.ccdname if survey is None: survey = LegacySurveyData(survey_dir=opt.survey_dir) catsurvey_north = survey catsurvey_south = None if opt.catalog_dir_north is not None: assert (opt.catalog_dir_south is not None) assert (opt.catalog_resolve_dec_ngc is not None) catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north) catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south) if opt.catalog_dir is not None: catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir) if filename is not None and hdu >= 0: # FIXME -- try looking up in CCDs file? # Read metadata from file print('Warning: faking metadata from file contents') T = exposure_metadata([filename], hdus=[hdu]) print('Metadata:') T.about() if not 'ccdzpt' in T.columns(): phdr = fitsio.read_header(filename) T.ccdzpt = np.array([phdr['MAGZERO']]) print('WARNING: using header MAGZERO') T.ccdraoff = np.array([0.]) T.ccddecoff = np.array([0.]) print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.') else: # Read metadata from survey-ccds.fits table T = survey.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and ccdname', ccdname) if hdu >= 0: T.cut(T.image_hdu == hdu) print(len(T), 'with HDU', hdu) if filename is not None: T.cut(np.array([f.strip() == filename for f in T.image_filename])) print(len(T), 'with filename', filename) if opt.camera is not None: T.cut(T.camera == opt.camera) print(len(T), 'with camera', opt.camera) if not all_hdus: assert (len(T) == 1) args = [] for ccd in T: args.append((survey, catsurvey_north, catsurvey_south, opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps)) if opt.threads: from astrometry.util.multiproc import multiproc from astrometry.util.timingpool import TimingPool, TimingPoolMeas pool = TimingPool(opt.threads) poolmeas = TimingPoolMeas(pool, pickleTraffic=False) Time.add_measurement(poolmeas) mp = multiproc(None, pool=pool) tm = Time() FF = mp.map(bounce_one_ccd, args) print('Multi-processing forced-phot:', Time() - tm) else: FF = map(bounce_one_ccd, args) FF = [F for F in FF if F is not None] if len(FF) == 0: print('No photometry results to write.') return 0 # Keep only the first header _, version_hdr = FF[0] FF = [F for F, hdr in FF] F = merge_tables(FF) if all_hdus: version_hdr.delete('CPHDU') version_hdr.delete('CCDNAME') units = { 'exptime': 'sec', 'flux': 'nanomaggy', 'flux_ivar': '1/nanomaggy^2', 'apflux': 'nanomaggy', 'apflux_ivar': '1/nanomaggy^2', 'psfdepth': '1/nanomaggy^2', 'galdepth': '1/nanomaggy^2', 'sky': 'nanomaggy/arcsec^2', 'psfsize': 'arcsec' } if opt.derivs: units.update({ 'dra': 'arcsec', 'ddec': 'arcsec', 'dra_ivar': '1/arcsec^2', 'ddec_ivar': '1/arcsec^2' }) columns = F.get_columns() order = [ 'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum', 'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'ccd_cuts', 'airmass', 'sky', 'psfdepth', 'galdepth', 'ra', 'dec', 'flux', 'flux_ivar', 'fracflux', 'rchisq', 'fracmasked', 'apflux', 'apflux_ivar', 'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar', 'ddec_ivar' ] columns = [c for c in order if c in columns] # Set units headers (must happen after column ordering is set!) hdr = fitsio.FITSHDR() for i, col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.outfn)) fitsio.write(tmpfn, None, header=version_hdr, clobber=True) F.writeto(tmpfn, header=hdr, append=True, columns=columns) os.rename(tmpfn, opt.outfn) print('Wrote', opt.outfn) tnow = Time() print('Total:', tnow - t0) return 0
def main(): import optparse from astrometry.util.stages import CallGlobal, runstage parser = optparse.OptionParser() parser.add_option('-f', '--force-stage', dest='force', action='append', default=[], help="Force re-running the given stage(s) -- don't read from pickle.") parser.add_option('-s', '--stage', dest='stage', default=[], action='append', help="Run up to the given stage(s)") parser.add_option('-n', '--no-write', dest='write', default=True, action='store_false') parser.add_option('-P', '--pickle', dest='picklepat', help='Pickle filename pattern, with %i, default %default', default='pickles/tunebrick-%(brick)06i-%%(stage)s.pickle') parser.add_option('-b', '--brick', type=int, help='Brick ID to run: default %default', default=377306) parser.add_option('-p', '--plots', dest='plots', action='store_true') #parser.add_option('--stamp', action='store_true') parser.add_option('--zoom', type=int, nargs=4, help='Set target image extent (default "0 3600 0 3600")') parser.add_option('-W', type=int, default=3600, help='Target image width (default %default)') parser.add_option('-H', type=int, default=3600, help='Target image height (default %default)') parser.add_option('--bands', help='Bands to process; default "%default"', default='grz') parser.add_option('--plot-base', default='plot-%(brick)06i', #'tunebrick/coadd/plot-%(brick)06i', help='Plot filenames; default %default') parser.add_option('--threads', type=int, help='Run multi-threaded') parser.add_option('--base-dir', dest='basedir', default='tunebrick', help='Base output directory; default %default') parser.add_option('--mock-psf', dest='mock_psf', action='store_true', help='Use fake PSF?') opt,args = parser.parse_args() Time.add_measurement(MemMeas) stagefunc = CallGlobal('stage_%s', globals()) if len(opt.stage) == 0: opt.stage.append('writecat2') opt.force.extend(opt.stage) opt.picklepat = opt.picklepat % dict(brick=opt.brick) prereqs = {'tims': None, 'cat': 'tims', 'tune': 'cat', 'writecat2': 'tune', 'recoadd': 'tims', 'rergb': 'recoadd', 'primage': 'recoadd', } ps = PlotSequence(opt.plot_base % dict(brick=opt.brick)) initargs = dict(ps=ps) initargs.update(W=opt.W, H=opt.H, brickid=opt.brick, target_extent=opt.zoom, program_name = 'tunebrick.py', pipe=True, bands=opt.bands, mock_psf=opt.mock_psf) kwargs = {} kwargs.update(basedir=opt.basedir) if opt.threads and opt.threads > 1: from astrometry.util.multiproc import multiproc mp = multiproc(opt.threads, init=runbrick_global_init, initargs=()) runbrick.mp = mp else: runbrick_global_init() t0 = Time() for stage in opt.stage: runstage(stage, opt.picklepat, stagefunc, force=opt.force, write=opt.write, prereqs=prereqs, initial_args=initargs, **kwargs) #tune(opt.brick, target_extent=opt.zoom) print 'Total:', Time()-t0
def run_forced_phot(cat, tim, ceres=True, derivs=False, agn=False, do_forced=True, do_apphot=True, get_model=False, ps=None, timing=False, fixed_also=False, ceres_threads=1): ''' fixed_also: if derivs=True, also run without derivatives and report that flux too? ''' if timing: tlast = Time() if ps is not None: import pylab as plt opti = None forced_kwargs = {} if ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 try: opti = CeresOptimizer(BW=B, BH=B, threads=ceres_threads) except: if ceres_threads > 1: raise RuntimeError( 'ceres_threads requested but not supported by tractor.ceres version' ) opti = CeresOptimizer(BW=B, BH=B) #forced_kwargs.update(verbose=True) # nsize = 0 for src in cat: # Limit sizes of huge models # from tractor.galaxy import ProfileGalaxy # if isinstance(src, ProfileGalaxy): # px,py = tim.wcs.positionToPixel(src.getPosition()) # h = src._getUnitFluxPatchSize(tim, px, py, tim.modelMinval) # MAXHALF = 128 # if h > MAXHALF: # #print('halfsize', h,'for',src,'-> setting to',MAXHALF) # nsize += 1 # src.halfsize = MAXHALF src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) #print('Limited the size of', nsize, 'large galaxy models') if derivs: realsrcs = [] derivsrcs = [] Iderivs = [] for i, src in enumerate(cat): from tractor import PointSource realsrcs.append(src) if not isinstance(src, PointSource): continue Iderivs.append(i) brightness_dra = src.getBrightness().copy() brightness_ddec = src.getBrightness().copy() brightness_dra.setParams(np.zeros(brightness_dra.numberOfParams())) brightness_ddec.setParams( np.zeros(brightness_ddec.numberOfParams())) brightness_dra.freezeAllBut(tim.band) brightness_ddec.freezeAllBut(tim.band) dsrc = SourceDerivatives(src, [brightness_dra, brightness_ddec], tim, ps) derivsrcs.append(dsrc) Iderivs = np.array(Iderivs) if fixed_also: pass else: # For convenience, put all the real sources at the front of # the list, so we can pull the IVs off the front of the list. cat = realsrcs + derivsrcs if agn: from tractor.galaxy import ExpGalaxy, DevGalaxy, FixedCompositeGalaxy from tractor import PointSource from legacypipe.survey import SimpleGalaxy, RexGalaxy realsrcs = [] agnsrcs = [] iagn = [] for i, src in enumerate(cat): realsrcs.append(src) ## ?? if isinstance(src, (SimpleGalaxy, RexGalaxy)): #print('Skipping SIMP or REX:', src) continue if isinstance(src, (ExpGalaxy, DevGalaxy, FixedCompositeGalaxy)): iagn.append(i) bright = src.getBrightness().copy() bright.setParams(np.zeros(bright.numberOfParams())) bright.freezeAllBut(tim.band) agn = PointSource(src.pos, bright) agn.freezeAllBut('brightness') #print('Adding "agn"', agn, 'to', src) #print('agn params:', agn.getParamNames()) agnsrcs.append(src) iagn = np.array(iagn) cat = realsrcs + agnsrcs print('Added AGN to', len(iagn), 'galaxies') tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') disable_galaxy_cache() F = fits_table() if do_forced: if timing and (derivs or agn): t = Time() print('Setting up:', t - tlast) tlast = t if derivs: if fixed_also: print('Forced photom with fixed positions:') R = tr.optimize_forced_photometry(variance=True, fitstats=False, shared_params=False, priors=False, **forced_kwargs) F.flux_fixed = np.array([ src.getBrightness().getFlux(tim.band) for src in cat ]).astype(np.float32) N = len(cat) F.flux_fixed_ivar = R.IV[:N].astype(np.float32) if timing: t = Time() print('Forced photom with fixed positions finished:', t - tlast) tlast = t cat = realsrcs + derivsrcs tr.setCatalog(Catalog(*cat)) print('Forced photom with position derivatives:') if ps is None and not get_model: forced_kwargs.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, priors=False, **forced_kwargs) if ps is not None or get_model: (data, mod, ie, chi, roi) = R.ims1[0] if ps is not None: ima = dict(vmin=-2. * tim.sig1, vmax=5. * tim.sig1, interpolation='nearest', origin='lower', cmap='gray') imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5, cmap='RdBu') plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() if derivs: trx = Tractor([tim], realsrcs) trx.freezeParam('images') modx = trx.getModelImage(0) chix = (data - modx) * tim.getInvError() plt.clf() plt.imshow(modx, **ima) plt.title('Model without derivatives: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chix, **imchi) plt.title('Chi without derivatives: %s' % tim.name) ps.savefig() if derivs or agn: cat = realsrcs N = len(cat) F.flux = np.array([ src.getBrightness().getFlux(tim.band) for src in cat ]).astype(np.float32) F.flux_ivar = R.IV[:N].astype(np.float32) F.fracflux = R.fitstats.profracflux[:N].astype(np.float32) F.rchisq = R.fitstats.prochi2[:N].astype(np.float32) try: F.fracmasked = R.fitstats.promasked[:N].astype(np.float32) except: print( 'No "fracmasked" available (only in recent Tractor versions)') if derivs: F.flux_dra = np.zeros(len(F), np.float32) F.flux_ddec = np.zeros(len(F), np.float32) F.flux_dra[Iderivs] = np.array( [src.getParams()[0] for src in derivsrcs]).astype(np.float32) F.flux_ddec[Iderivs] = np.array( [src.getParams()[1] for src in derivsrcs]).astype(np.float32) F.flux_dra_ivar = np.zeros(len(F), np.float32) F.flux_ddec_ivar = np.zeros(len(F), np.float32) F.flux_dra_ivar[Iderivs] = R.IV[N::2].astype(np.float32) F.flux_ddec_ivar[Iderivs] = R.IV[N + 1::2].astype(np.float32) if agn: F.flux_agn = np.zeros(len(F), np.float32) F.flux_agn_ivar = np.zeros(len(F), np.float32) F.flux_agn[iagn] = np.array( [src.getParams()[0] for src in agnsrcs]) F.flux_agn_ivar[iagn] = R.IV[N:].astype(np.float32) if timing: t = Time() print('Forced photom:', t - tlast) tlast = t if do_apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack( [tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() #print('Apertures:', apertures, 'pixels') #print('apxy shape', apxy.shape) # --> (2,N) # The aperture photometry routine doesn't like pixel positions outside the image H, W = img.shape Iap = np.flatnonzero((apxy[0, :] >= 0) * (apxy[1, :] >= 0) * (apxy[0, :] <= W - 1) * (apxy[1, :] <= H - 1)) print('Aperture photometry for', len(Iap), 'of', len(apxy[0, :]), 'sources within image bounds') for rad in apertures: aper = photutils.CircularAperture(apxy[:, Iap], rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = np.zeros((len(F), len(apertures)), np.float32) F.apflux[Iap, :] = ap.astype(np.float32) apimgerr = np.vstack(apimgerr).T apiv = np.zeros(apimgerr.shape, np.float32) apiv[apimgerr != 0] = 1. / apimgerr[apimgerr != 0]**2 F.apflux_ivar = np.zeros((len(F), len(apertures)), np.float32) F.apflux_ivar[Iap, :] = apiv if timing: print('Aperture photom:', Time() - tlast) if get_model: return F, mod return F
def _imap_finished(self, tstart): tend = Time() self.parallel.append((tstart, tend)) self.t0 = tend