def get_context_data(self, **kwargs):
print('get_context_data called')
t0 = Time()
from viewer import settings
print('Using query description:', self.querydesc)
context = super(CatalogSearchList, self).get_context_data(**kwargs)
print('Got context data', context, 'in', Time() - t0)
context.update(
root_url=settings.ROOT_URL,
search_description=self.querydesc,
)
req = self.request
args = req.GET.copy()
args.pop('page', None)
qstring = '?' + '&'.join(
['%s=%s' % (k, v) for k, v in args.items() if len(v)])
context['myurl'] = req.path + qstring
context['fitsurl'] = reverse(fits_results) + qstring
context['viewurl'] = reverse(viewer_results) + qstring
pager = context.get('paginator')
context['total_items'] = pager.count
print('Done updating context:', Time() - t0)
return context
def map(self, *args, **kwargs):
tstart = Time()
res = super(MyMultiproc, self).map(*args, **kwargs)
tend = Time()
self.serial.append((self.t0, tstart))
self.parallel.append((tstart, tend))
self.t0 = tend
return res
def test_jumbo():
# Test jumbo (> 2 GB) args/results
import numpy as np
from astrometry.util.ttime import Time
with TimingPool(2) as pool:
Time.add_measurement(TimingPoolMeas(pool, pickleTraffic=True))
t0 = Time()
R = pool.map(test_func_4, [np.ones(int(2.1 * 1024 * 1024 * 1024 / 8))])
print('Jumbo:', np.sum(R))
print(Time() - t0)
def time_run_calibs(*args):
from astrometry.util.ttime import Time
t0 = Time()
rtn = run_calibs(*args)
t1 = Time()
print('Time run_calibs:', t1 - t0)
import sys
sys.stdout.flush()
sys.stderr.flush()
return rtn
def __call__(self, stage, **kwargs):
from astrometry.util.ttime import Time
from datetime import datetime
t0 = Time()
print 'Running stage', stage, 'at', datetime.now().isoformat()
rtn = super(CallGlobalTime, self).__call__(stage, **kwargs)
t1 = Time()
print 'Stage', stage, ':', t1 - t0
print 'Stage', stage, 'finished:', datetime.now().isoformat()
return rtn
def _optimize_forcedphot_core(self,
tractor,
result,
umodels,
imlist,
mod0,
scales,
skyderivs,
minFlux,
nonneg=None,
wantims0=None,
wantims1=None,
negfluxval=None,
rois=None,
priors=None,
sky=None,
justims0=None,
subimgs=None,
damp=None,
alphas=None,
Nsky=None,
mindlnp=None,
shared_params=None):
# print(len(umodels), 'umodels, lengths', [len(x) for x in umodels])
if len(umodels) == 0:
return
Nsourceparams = len(umodels[0])
imgs = tractor.images
t0 = Time()
derivs = [[] for i in range(Nsourceparams)]
for i, (tim, umods, scale) in enumerate(zip(imlist, umodels, scales)):
for um, dd in zip(umods, derivs):
if um is None:
continue
dd.append((um * scale, tim))
logverb('forced phot: derivs', Time() - t0)
if sky:
# Sky derivatives are part of the image derivatives, so go
# first in the derivative list.
derivs = skyderivs + derivs
assert (len(derivs) == tractor.numberOfParams())
self._lsqr_forced_photom(tractor, result, derivs, mod0, imgs, umodels,
rois, scales, priors, sky, minFlux, justims0,
subimgs, damp, alphas, Nsky, mindlnp,
shared_params)
def global_init(loglevel):
'''
Global initialization routine called by mpi4py when each worker is started.
'''
import socket
import os
from mpi4py import MPI
print('MPI4PY process starting on', socket.gethostname(), 'pid',
os.getpid(), 'MPI rank', MPI.COMM_WORLD.Get_rank())
import logging
logging.basicConfig(level=loglevel,
format='%(message)s',
stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(loglevel + 10)
from astrometry.util.ttime import Time, MemMeas
Time.add_measurement(MemMeas)
def start_subphase(self, name):
# push current state to stack
tstart = Time()
self.serial.append((self.t0, tstart))
self.t0 = tstart
self.phases.append((name, self.serial, self.parallel, self.t0))
# print('Starting subphase', name)
# print(' pushing serial', self.serial)
# print(' pushing parallel', self.parallel)
self.serial = []
self.parallel = []
def test():
#import logging
#multiprocessing.log_to_stderr().setLevel(logging.DEBUG)
# per multiprocess.Pool documentation (as of Python 3.8 at least), can't just
# let a Pool fall out of scope; must close or you risk the process hanging.
# This definitely happens on Linux!
with TimingPool(4) as pool:
R = pool.map(test_func, [10, 20, 30])
print('worker cpu:', pool.get_worker_cpu())
print('worker wall:', pool.get_worker_wall())
print('pickles:', pool.get_pickle_traffic_string())
import numpy as np
print('Creating second pool')
with TimingPool(4) as pool:
print('Using second pool')
R = pool.map(test_func_2,
[np.random.normal(size=1000000) for x in range(5)])
print('Got result from second pool')
print('worker cpu:', pool.get_worker_cpu())
print('worker wall:', pool.get_worker_wall())
print('pickles:', pool.get_pickle_traffic_string())
from astrometry.util.ttime import Time
with TimingPool(4, track_send_pickles=False,
track_recv_pickles=False) as pool:
m = TimingPoolMeas(pool, pickleTraffic=False)
Time.add_measurement(m)
t0 = Time()
R = pool.map(test_func, [20, 20, 20])
print(Time() - t0)
Time.remove_measurement(m)
with TimingPool(4) as pool:
Time.add_measurement(TimingPoolMeas(pool, pickleTraffic=True))
t0 = Time()
R = pool.map(test_func_3,
[np.random.normal(size=1000000) for x in range(5)])
print(Time() - t0)
def report(self, nthreads):
# Tally the serial time up to now
tend = Time()
self.serial.append((self.t0, tend))
self.t0 = tend
# Nasty... peek into Time members
scpu = 0.
swall = 0.
print('Serial:')
for t0, t1 in self.serial:
print(t1 - t0)
for m0, m1 in zip(t0.meas, t1.meas):
if isinstance(m0, CpuMeas):
scpu += m1.cpu_seconds_since(m0)
swall += m1.wall_seconds_since(m0)
#print ' total cpu', scpu, 'wall', swall
pworkercpu = 0.
pworkerwall = 0.
pwall = 0.
pcpu = 0.
print('Parallel:')
for t0, t1 in self.parallel:
print(t1 - t0)
for m0, m1 in zip(t0.meas, t1.meas):
if isinstance(m0, TimingPoolTimestamp):
mt0 = m0.t0
mt1 = m1.t0
pworkercpu += mt1['worker_cpu'] - mt0['worker_cpu']
pworkerwall += mt1['worker_wall'] - mt0['worker_wall']
elif isinstance(m0, CpuMeas):
pwall += m1.wall_seconds_since(m0)
pcpu += m1.cpu_seconds_since(m0)
print()
print('Total serial CPU ', scpu)
print('Total serial Wall ', swall)
print('Total worker CPU ', pworkercpu)
print('Total worker Wall ', pworkerwall)
print('Total parallel Wall', pwall)
print('Total parallel CPU ', pcpu)
print()
tcpu = scpu + pworkercpu + pcpu
twall = swall + pwall
if nthreads is None:
nthreads = 1
print('Grand total CPU: %.1f sec' % tcpu)
print('Grand total Wall: %.1f sec' % twall)
print('Grand total CPU utilization: %.2f cores' % (tcpu / twall))
print('Grand total efficiency: %.1f %%' % (100. * tcpu /
(twall * nthreads)))
print()
def optimize(self,
tractor,
alphas=None,
damp=0,
priors=True,
scale_columns=True,
shared_params=True,
variance=False,
just_variance=False,
**nil):
logverb(tractor.getName() + ': Finding derivs...')
t0 = Time()
allderivs = tractor.getDerivs()
tderivs = Time() - t0
#print(Time() - t0)
#print('allderivs:', allderivs)
# for d in allderivs:
# for (p,im) in d:
# print('patch mean', np.mean(p.patch))
logverb('Finding optimal update direction...')
t0 = Time()
X = self.getUpdateDirection(tractor,
allderivs,
damp=damp,
priors=priors,
scale_columns=scale_columns,
shared_params=shared_params,
variance=variance)
if X is None:
# Failure
return (0., None, 0.)
if variance:
if len(X) == 0:
return 0, X, 0, None
X, var = X
if just_variance:
return var
#print(Time() - t0)
topt = Time() - t0
#print('X:', X)
if len(X) == 0:
return 0, X, 0.
logverb('X: len', len(X), '; non-zero entries:', np.count_nonzero(X))
logverb('Finding optimal step size...')
t0 = Time()
(dlogprob, alpha) = self.tryUpdates(tractor, X, alphas=alphas)
tstep = Time() - t0
logverb('Finished opt2.')
logverb(' alpha =', alpha)
logverb(' Tderiv', tderivs)
logverb(' Topt ', topt)
logverb(' Tstep ', tstep)
if variance:
return dlogprob, X, alpha, var
return dlogprob, X, alpha
def imap_unordered(self, func, iterable, chunksize=None, wrap=False):
# So, this is a bit strange, tracking parallel vs serial time
# for an async object, via the ImapTracker & callback to
# _imap_finished.
tstart = Time()
self.serial.append((self.t0, tstart))
#res = super(MyMultiproc, self).imap_unordered(*args, **kwargs)
cs = chunksize
if cs is None:
cs = self.map_chunksize
if self.pool is None:
import itertools
return itertools.imap(func, iterable)
if wrap or self.wrap_all:
func = funcwrapper(func)
res = self.pool.imap_unordered(func, iterable, chunksize=cs)
return ImapTracker(res, self, tstart)
if __name__ == '__main__':
import optparse
parser = optparse.OptionParser(usage='%prog <decam-image-filename> <decam-HDU> <catalog.fits or "DR1"> <output-catalog.fits>')
parser.add_option('--zoom', type=int, nargs=4, help='Set target image extent (default "0 2046 0 4094")')
parser.add_option('--no-ceres', action='store_false', default=True, dest='ceres', help='Do not use Ceres optimiziation engine (use scipy)')
parser.add_option('--catalog-path', default='dr1',
help='Path to DECaLS DR1 catalogs; default %default, eg, /project/projectdirs/cosmo/data/legacysurvey/dr1')
parser.add_option('--plots', default=None, help='Create plots; specify a base filename for the plots')
opt,args = parser.parse_args()
if len(args) != 4:
parser.print_help()
sys.exit(-1)
Time.add_measurement(MemMeas)
t0 = Time()
filename = args[0]
hdu = int(args[1])
catfn = args[2]
outfn = args[3]
if os.path.exists(outfn):
print 'Ouput file exists:', outfn
sys.exit(0)
zoomslice = None
if opt.zoom is not None:
(x0,x1,y0,y1) = opt.zoom
def stage1(T=None,
coimgs=None,
cons=None,
detmaps=None,
detivs=None,
targetrd=None,
pixscale=None,
targetwcs=None,
W=None,
H=None,
bands=None,
tims=None,
ps=None,
brick=None,
cat=None):
orig_wcsxy0 = [tim.wcs.getX0Y0() for tim in tims]
hot = np.zeros((H, W), np.float32)
for band in bands:
detmap = detmaps[band] / np.maximum(1e-16, detivs[band])
detsn = detmap * np.sqrt(detivs[band])
hot = np.maximum(hot, detsn)
detmaps[band] = detmap
### FIXME -- ugri
for sedname, sed in [('Flat', (1., 1., 1.)), ('Red', (2.5, 1.0, 0.4))]:
sedmap = np.zeros((H, W), np.float32)
sediv = np.zeros((H, W), np.float32)
for iband, band in enumerate(bands):
# We convert the detmap to canonical band via
# detmap * w
# And the corresponding change to sig1 is
# sig1 * w
# So the invvar-weighted sum is
# (detmap * w) / (sig1**2 * w**2)
# = detmap / (sig1**2 * w)
sedmap += detmaps[band] * detivs[band] / sed[iband]
sediv += detivs[band] / sed[iband]**2
sedmap /= np.maximum(1e-16, sediv)
sedsn = sedmap * np.sqrt(sediv)
hot = np.maximum(hot, sedsn)
plt.clf()
dimshow(np.round(sedsn), vmin=0, vmax=10, cmap='hot')
plt.title('SED-matched detection filter: %s' % sedname)
ps.savefig()
peaks = (hot > 4)
blobs, nblobs = label(peaks)
print('N detected blobs:', nblobs)
blobslices = find_objects(blobs)
# Un-set catalog blobs
for x, y in zip(T.itx, T.ity):
# blob number
bb = blobs[y, x]
if bb == 0:
continue
# un-set 'peaks' within this blob
slc = blobslices[bb - 1]
peaks[slc][blobs[slc] == bb] = 0
# Now, after having removed catalog sources, crank up the detection threshold
peaks &= (hot > 5)
# zero out the edges(?)
peaks[0, :] = peaks[:, 0] = 0
peaks[-1, :] = peaks[:, -1] = 0
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[0:-2, 1:-1])
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[2:, 1:-1])
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[1:-1, 0:-2])
peaks[1:-1, 1:-1] &= (hot[1:-1, 1:-1] >= hot[1:-1, 2:])
# These are our peaks
pki = np.flatnonzero(peaks)
peaky, peakx = np.unravel_index(pki, peaks.shape)
print(len(peaky), 'peaks')
crossa = dict(ms=10, mew=1.5)
plt.clf()
dimshow(get_rgb(coimgs, bands))
ax = plt.axis()
plt.plot(T.tx, T.ty, 'r+', **crossa)
plt.plot(peakx, peaky, '+', color=green, **crossa)
plt.axis(ax)
plt.title('SDSS + SED-matched detections')
ps.savefig()
### HACK -- high threshold again
# Segment, and record which sources fall into each blob
blobs, nblobs = label((hot > 20))
print('N detected blobs:', nblobs)
blobslices = find_objects(blobs)
T.blob = blobs[T.ity, T.itx]
blobsrcs = []
blobflux = []
fluximg = coimgs[1]
for blob in range(1, nblobs + 1):
blobsrcs.append(np.flatnonzero(T.blob == blob))
bslc = blobslices[blob - 1]
blobflux.append(np.sum(fluximg[bslc][blobs[bslc] == blob]))
# Fit the SDSS sources
for tim in tims:
tim.psfex.fitSavedData(*tim.psfex.splinedata)
tim.psf = tim.psfex
# How far down to render model profiles
minsigma = 0.1
for tim in tims:
tim.modelMinval = minsigma * tim.sig1
srcvariances = [[] for src in cat]
# Fit in order of flux
for blobnumber, iblob in enumerate(np.argsort(-np.array(blobflux))):
bslc = blobslices[iblob]
Isrcs = blobsrcs[iblob]
if len(Isrcs) == 0:
continue
print()
print('Blob', blobnumber, 'of', len(blobflux), ':', len(Isrcs),
'sources')
print('Source indices:', Isrcs)
print()
# blob bbox in target coords
sy, sx = bslc
by0, by1 = sy.start, sy.stop
bx0, bx1 = sx.start, sx.stop
blobh, blobw = by1 - by0, bx1 - bx0
rr, dd = targetwcs.pixelxy2radec([bx0, bx0, bx1, bx1],
[by0, by1, by1, by0])
alphas = [0.1, 0.3, 1.0]
subtims = []
for itim, tim in enumerate(tims):
h, w = tim.shape
ok, x, y = tim.subwcs.radec2pixelxy(rr, dd)
sx0, sx1 = x.min(), x.max()
sy0, sy1 = y.min(), y.max()
if sx1 < 0 or sy1 < 0 or sx1 > w or sy1 > h:
continue
sx0 = np.clip(int(np.floor(sx0)), 0, w - 1)
sx1 = np.clip(int(np.ceil(sx1)), 0, w - 1) + 1
sy0 = np.clip(int(np.floor(sy0)), 0, h - 1)
sy1 = np.clip(int(np.ceil(sy1)), 0, h - 1) + 1
subslc = slice(sy0, sy1), slice(sx0, sx1)
subimg = tim.getImage()[subslc]
subie = tim.getInvError()[subslc]
subwcs = tim.getWcs().copy()
ox0, oy0 = orig_wcsxy0[itim]
subwcs.setX0Y0(ox0 + sx0, oy0 + sy0)
# Mask out inverr for pixels that are not within the blob.
subtarget = targetwcs.get_subimage(bx0, by0, blobw, blobh)
subsubwcs = tim.subwcs.get_subimage(int(sx0), int(sy0),
int(sx1 - sx0), int(sy1 - sy0))
try:
Yo, Xo, Yi, Xi, rims = resample_with_wcs(
subsubwcs, subtarget, [], 2)
except OverlapError:
print('No overlap')
continue
if len(Yo) == 0:
continue
subie2 = np.zeros_like(subie)
I = np.flatnonzero(blobs[bslc][Yi, Xi] == (iblob + 1))
subie2[Yo[I], Xo[I]] = subie[Yo[I], Xo[I]]
subie = subie2
# If the subimage (blob) is small enough, instantiate a
# constant PSF model in the center.
if sy1 - sy0 < 100 and sx1 - sx0 < 100:
subpsf = tim.psf.mogAt(ox0 + (sx0 + sx1) / 2.,
oy0 + (sy0 + sy1) / 2.)
else:
# Otherwise, instantiate a (shifted) spatially-varying
# PsfEx model.
subpsf = ShiftedPsf(tim.psf, ox0 + sx0, oy0 + sy0)
subtim = Image(data=subimg,
inverr=subie,
wcs=subwcs,
psf=subpsf,
photocal=tim.getPhotoCal(),
sky=tim.getSky(),
name=tim.name)
subtim.band = tim.band
subtim.sig1 = tim.sig1
subtim.modelMinval = tim.modelMinval
subtims.append(subtim)
subcat = Catalog(*[cat[i] for i in Isrcs])
subtr = Tractor(subtims, subcat)
subtr.freezeParam('images')
# Optimize individual sources in order of flux
fluxes = []
for src in subcat:
# HACK -- here we just *sum* the nanomaggies in each band. Bogus!
br = src.getBrightness()
flux = sum([br.getFlux(band) for band in bands])
fluxes.append(flux)
Ibright = np.argsort(-np.array(fluxes))
if len(Ibright) >= 5:
# -Remember the original subtim images
# -Compute initial models for each source (in each tim)
# -Subtract initial models from images
# -During fitting, for each source:
# -add back in the source's initial model (to each tim)
# -fit, with Catalog([src])
# -subtract final model (from each tim)
# -Replace original subtim images
#
# --Might want to omit newly-added detection-filter sources, since their
# fluxes are bogus.
# Remember original tim images
orig_timages = [tim.getImage().copy() for tim in subtims]
initial_models = []
# Create initial models for each tim x each source
for tim in subtims:
mods = []
for src in subcat:
mod = src.getModelPatch(tim)
mods.append(mod)
if mod is not None:
if not np.all(np.isfinite(mod.patch)):
print('Non-finite mod patch')
print('source:', src)
print('tim:', tim)
print('PSF:', tim.getPsf())
assert (np.all(np.isfinite(mod.patch)))
mod.addTo(tim.getImage(), scale=-1)
initial_models.append(mods)
# For sources in decreasing order of brightness
for numi, i in enumerate(Ibright):
tsrc = Time()
print('Fitting source', i,
'(%i of %i in blob)' % (numi, len(Ibright)))
src = subcat[i]
print(src)
srctractor = Tractor(subtims, [src])
srctractor.freezeParams('images')
# Add this source's initial model back in.
for tim, mods in zip(subtims, initial_models):
mod = mods[i]
if mod is not None:
mod.addTo(tim.getImage())
print('Optimizing:', srctractor)
srctractor.printThawedParams()
for step in range(50):
dlnp, X, alpha = srctractor.optimize(priors=False,
shared_params=False,
alphas=alphas)
print('dlnp:', dlnp, 'src', src)
if dlnp < 0.1:
break
for tim in subtims:
mod = src.getModelPatch(tim)
if mod is not None:
mod.addTo(tim.getImage(), scale=-1)
for tim, img in zip(subtims, orig_timages):
tim.data = img
del orig_timages
del initial_models
else:
# Fit sources one at a time, but don't subtract other models
subcat.freezeAllParams()
for numi, i in enumerate(Ibright):
tsrc = Time()
print('Fitting source', i,
'(%i of %i in blob)' % (numi, len(Ibright)))
print(subcat[i])
subcat.freezeAllBut(i)
print('Optimizing:', subtr)
subtr.printThawedParams()
for step in range(10):
dlnp, X, alpha = subtr.optimize(priors=False,
shared_params=False,
alphas=alphas)
print('dlnp:', dlnp)
if dlnp < 0.1:
break
print('Fitting source took', Time() - tsrc)
print(subcat[i])
if len(Isrcs) > 1 and len(Isrcs) <= 10:
tfit = Time()
# Optimize all at once?
subcat.thawAllParams()
print('Optimizing:', subtr)
subtr.printThawedParams()
for step in range(20):
dlnp, X, alpha = subtr.optimize(priors=False,
shared_params=False,
alphas=alphas)
print('dlnp:', dlnp)
if dlnp < 0.1:
break
# Variances
subcat.thawAllRecursive()
subcat.freezeAllParams()
for isub, srci in enumerate(Isrcs):
print('Variances for source', srci)
subcat.thawParam(isub)
src = subcat[isub]
print('Source', src)
print('Params:', src.getParamNames())
if isinstance(src, (DevGalaxy, ExpGalaxy)):
src.shape = EllipseE.fromEllipseESoft(src.shape)
elif isinstance(src, FixedCompositeGalaxy):
src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp)
src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev)
print('Converted ellipse:', src)
allderivs = subtr.getDerivs()
for iparam, derivs in enumerate(allderivs):
dchisq = 0
for deriv, tim in derivs:
h, w = tim.shape
deriv.clipTo(w, h)
ie = tim.getInvError()
slc = deriv.getSlice(ie)
chi = deriv.patch * ie[slc]
dchisq += (chi**2).sum()
if dchisq == 0.:
v = np.nan
else:
v = 1. / dchisq
srcvariances[srci].append(v)
assert (len(srcvariances[srci]) == subcat[isub].numberOfParams())
subcat.freezeParam(isub)
cat.thawAllRecursive()
for i, src in enumerate(cat):
print('Source', i, src)
print('variances:', srcvariances[i])
print(len(srcvariances[i]), 'vs', src.numberOfParams())
if len(srcvariances[i]) != src.numberOfParams():
# This can happen for sources outside the brick bounds: they never get optimized?
print('Warning: zeroing variances for source', src)
srcvariances[i] = [0] * src.numberOfParams()
if isinstance(src, (DevGalaxy, ExpGalaxy)):
src.shape = EllipseE.fromEllipseESoft(src.shape)
elif isinstance(src, FixedCompositeGalaxy):
src.shapeExp = EllipseE.fromEllipseESoft(src.shapeExp)
src.shapeDev = EllipseE.fromEllipseESoft(src.shapeDev)
assert (len(srcvariances[i]) == src.numberOfParams())
variances = np.hstack(srcvariances)
assert (len(variances) == cat.numberOfParams())
return dict(cat=cat, variances=variances)
import runbrick
if __name__ == '__main__':
parser = optparse.OptionParser(usage='%prog [options] brick-number')
parser.add_option('--threads', type=int, help='Run multi-threaded')
parser.add_option('--no-ceres', action='store_true', help='Do not use Ceres')
parser.add_option('--stamp', action='store_true', help='Run a tiny postage-stamp')
opt,args = parser.parse_args()
if len(args) != 1:
parser.print_help()
sys.exit(-1)
brick = int(args[0], 10)
Time.add_measurement(MemMeas)
lvl = logging.WARNING
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
if opt.threads and opt.threads > 1:
from astrometry.util.multiproc import multiproc
if True:
mp = multiproc(opt.threads, init=runbrick_global_init, initargs=())
else:
from utils.debugpool import DebugPool, DebugPoolMeas
dpool = DebugPool(opt.threads, taskqueuesize=2*opt.threads,
initializer=runbrick_global_init)
mp = multiproc(pool=dpool)
def forced_photometry(
self,
tractor,
alphas=None,
damp=0,
priors=False,
minsb=0.,
mindlnp=1.,
rois=None,
sky=False,
minFlux=None,
fitstats=False,
fitstat_extras=None,
justims0=False,
variance=False,
skyvariance=False,
shared_params=True,
nonneg=False,
nilcounts=-1e30,
wantims=True,
negfluxval=None,
):
from basics import LinearPhotoCal, ShiftedWcs
result = OptResult()
assert (not priors)
scales = []
imgs = tractor.getImages()
for img in imgs:
assert (isinstance(img.getPhotoCal(), LinearPhotoCal))
scales.append(img.getPhotoCal().getScale())
if rois is not None:
assert (len(rois) == len(imgs))
# HACK -- if sky=True, assume we are fitting the sky in ALL images.
# We could ask which ones are thawed...
if sky:
for img in imgs:
# FIXME -- would be nice to allow multi-param linear sky models
assert (img.getSky().numberOfParams() == 1)
Nsourceparams = tractor.catalog.numberOfParams()
srcs = list(tractor.catalog.getThawedSources())
# Render unit-flux models for each source.
t0 = Time()
(umodels, umodtosource,
umodsforsource) = self._get_umodels(tractor, srcs, imgs, minsb, rois)
for umods in umodels:
assert (len(umods) == Nsourceparams)
tmods = Time() - t0
logverb('forced phot: getting unit-flux models:', tmods)
subimgs = []
if rois is not None:
for i, img in enumerate(imgs):
roi = rois[i]
y0 = roi[0].start
x0 = roi[1].start
subwcs = img.wcs.shift(x0, y0)
subimg = Image(data=img.data[roi],
inverr=img.inverr[roi],
psf=img.psf,
wcs=subwcs,
sky=img.sky,
photocal=img.photocal,
name=img.name)
subimgs.append(subimg)
imlist = subimgs
else:
imlist = imgs
t0 = Time()
fsrcs = list(tractor.catalog.getFrozenSources())
mod0 = []
for img in imlist:
# "sky = not sky": I'm not just being contrary :)
# If we're fitting sky, we'll do a setParams() and get the
# sky models to render themselves when evaluating lnProbs,
# rather than pre-computing the nominal value here and
# then computing derivatives.
mod0.append(
tractor.getModelImage(img, fsrcs, minsb=minsb, sky=not sky))
tmod = Time() - t0
logverb('forced phot: getting frozen-source model:', tmod)
skyderivs = None
if sky:
t0 = Time()
# build the derivative list as required by getUpdateDirection:
# (param0) -> [ (deriv, img), (deriv, img), ... ], ... ],
skyderivs = []
for img in imlist:
dskys = img.getSky().getParamDerivatives(tractor, img, None)
for dsky in dskys:
skyderivs.append([(dsky, img)])
Nsky = len(skyderivs)
assert (Nsky == tractor.images.numberOfParams())
assert (Nsky + Nsourceparams == tractor.numberOfParams())
logverb('forced phot: sky derivs', Time() - t0)
else:
Nsky = 0
wantims0 = wantims1 = wantims
if fitstats:
wantims1 = True
self._optimize_forcedphot_core(tractor,
result,
umodels,
imlist,
mod0,
scales,
skyderivs,
minFlux,
nonneg=nonneg,
wantims0=wantims0,
wantims1=wantims1,
negfluxval=negfluxval,
rois=rois,
priors=priors,
sky=sky,
justims0=justims0,
subimgs=subimgs,
damp=damp,
alphas=alphas,
Nsky=Nsky,
mindlnp=mindlnp,
shared_params=shared_params)
if variance:
# Inverse variance
t0 = Time()
result.IV = self._get_iv(sky, skyvariance, Nsky, skyderivs, srcs,
imlist, umodels, scales)
logverb('forced phot: variance:', Time() - t0)
imsBest = getattr(result, 'ims1', None)
if fitstats and imsBest is None:
print 'Warning: fit stats not computed because imsBest is None'
result.fitstats = None
elif fitstats:
t0 = Time()
result.fitstats = self._get_fitstats(tractor.catalog,
imsBest,
srcs,
imlist,
umodsforsource,
umodels,
scales,
nilcounts,
extras=fitstat_extras)
logverb('forced phot: fit stats:', Time() - t0)
return result
def run_one_ccd(survey, catsurvey_north, catsurvey_south, resolve_dec, ccd,
opt, zoomslice, ps):
from functools import reduce
from legacypipe.bits import DQ_BITS
tlast = Time()
#print('Opt:', opt)
im = survey.get_image_object(ccd)
print('Run_one_ccd: checking cache', survey.cache_dir)
if survey.cache_dir is not None:
im.check_for_cached_files(survey)
if opt.do_calib:
im.run_calibs(splinesky=True)
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
constant_invvar=opt.constant_invvar,
hybridPsf=opt.hybrid_psf,
normalizePsf=opt.normalize_psf,
old_calibs_ok=True,
trim_edges=False)
print('Got tim:', tim) #, 'x0,y0', tim.x0, tim.y0)
chipwcs = tim.subwcs
H, W = tim.shape
tnow = Time()
print('Read image:', tnow - tlast)
tlast = tnow
if ccd.camera == 'decam':
# Halo subtraction
from legacypipe.halos import subtract_one
from legacypipe.reference import mask_radius_for_mag, read_gaia
ref_margin = mask_radius_for_mag(0.)
mpix = int(np.ceil(ref_margin * 3600. / chipwcs.pixel_scale()))
marginwcs = chipwcs.get_subimage(-mpix, -mpix, W + 2 * mpix,
H + 2 * mpix)
gaia = read_gaia(marginwcs, None)
keeprad = np.ceil(gaia.keep_radius * 3600. /
chipwcs.pixel_scale()).astype(int)
_, xx, yy = chipwcs.radec2pixelxy(gaia.ra, gaia.dec)
# cut to those touching the chip
gaia.cut((xx > -keeprad) * (xx < W + keeprad) * (yy > -keeprad) *
(yy < H + keeprad))
Igaia, = np.nonzero(gaia.isgaia * gaia.pointsource)
halostars = gaia[Igaia]
print('Got', len(gaia), 'Gaia stars,', len(halostars),
'for halo subtraction')
moffat = True
halos = subtract_one((tim, halostars, moffat))
tim.data -= halos
# The "north" and "south" directories often don't have
# 'survey-bricks" files of their own -- use the 'survey' one
# instead.
if catsurvey_south is not None:
try:
catsurvey_south.get_bricks_readonly()
except:
catsurvey_south.bricks = survey.get_bricks_readonly()
if catsurvey_north is not None:
try:
catsurvey_north.get_bricks_readonly()
except:
catsurvey_north.bricks = survey.get_bricks_readonly()
# Apply outlier masks
outlier_header = None
outlier_mask = None
posneg_mask = None
if opt.outlier_mask is not None:
posneg_mask = np.zeros(tim.shape, np.uint8)
# Outliers masks are computed within a survey (north/south for dr9), and are stored
# in a brick-oriented way, in the results directories.
north_ccd = (ccd.camera.strip() != 'decam')
catsurvey = catsurvey_north
if not north_ccd and catsurvey_south is not None:
catsurvey = catsurvey_south
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
for b in bricks:
print(
'Reading outlier mask for brick', b.brickname, ':',
catsurvey.find_file('outliers_mask',
brick=b.brickname,
output=False))
ok = read_outlier_mask_file(catsurvey, [tim],
b.brickname,
pos_neg_mask=posneg_mask,
subimage=False,
output=False,
ps=ps)
if not ok:
print('WARNING: failed to read outliers mask file for brick',
b.brickname)
if opt.outlier_mask is not None:
outlier_mask = np.zeros((ccd.height, ccd.width), np.uint8)
outlier_mask[tim.y0:tim.y0 + H, tim.x0:tim.x0 + W] = posneg_mask
del posneg_mask
# Grab original image headers (including WCS)
im = survey.get_image_object(ccd)
imhdr = im.read_image_header()
imhdr['CAMERA'] = ccd.camera
imhdr['EXPNUM'] = ccd.expnum
imhdr['CCDNAME'] = ccd.ccdname
imhdr['IMGFILE'] = ccd.image_filename.strip()
outlier_header = imhdr
if opt.catalog:
T = fits_table(opt.catalog)
else:
chipwcs = tim.subwcs
T = get_catalog_in_wcs(chipwcs,
survey,
catsurvey_north,
catsurvey_south=catsurvey_south,
resolve_dec=resolve_dec)
if T is None:
print('No sources to photometer.')
return None
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
surveydir = survey.get_survey_dir()
del survey
if opt.move_gaia:
# Gaia stars: move RA,Dec to the epoch of this image.
I = np.flatnonzero(T.ref_epoch > 0)
if len(I):
print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd())
ra, dec = radec_at_mjd(T.ra[I], T.dec[I],
T.ref_epoch[I].astype(float), T.pmra[I],
T.pmdec[I], T.parallax[I], tim.time.toMjd())
T.ra[I] = ra
T.dec[I] = dec
tnow = Time()
print('Read catalog:', tnow - tlast)
tlast = tnow
# Find SGA galaxies outside this chip and subtract them before we begin.
chipwcs = tim.subwcs
_, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
sga_out = (T.ref_cat == 'L3') * np.logical_not(
(xx >= 1) * (xx <= W) * (yy >= 1) * (yy <= H))
I = np.flatnonzero(sga_out)
if len(I):
print(len(I), 'SGA galaxies are outside the image. Subtracting...')
cat = read_fits_catalog(T[I], bands=[tim.band])
tr = Tractor([tim], cat)
mod = tr.getModelImage(0)
tim.data -= mod
I = np.flatnonzero(np.logical_not(sga_out))
T.cut(I)
cat = read_fits_catalog(T, bands='r')
# Replace the brightness (which will be a NanoMaggies with g,r,z)
# with a NanoMaggies with this image's band only.
for src in cat:
src.brightness = NanoMaggies(**{tim.band: 1.})
tnow = Time()
print('Parse catalog:', tnow - tlast)
tlast = tnow
print('Forced photom...')
F = run_forced_phot(cat,
tim,
ceres=opt.ceres,
derivs=opt.derivs,
fixed_also=True,
agn=opt.agn,
do_forced=opt.forced,
do_apphot=opt.apphot,
get_model=opt.save_model,
ps=ps,
timing=True,
ceres_threads=opt.ceres_threads)
if opt.save_model:
# unpack results
F, model_img = F
F.release = T.release
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.camera = np.array([ccd.camera] * len(F))
F.expnum = np.array([im.expnum] * len(F), dtype=np.int64)
F.ccdname = np.array([im.ccdname] * len(F))
# "Denormalizing"
F.filter = np.array([tim.band] * len(F))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(F))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(F), dtype=np.float32)
F.psfsize = np.array([tim.psf_fwhm * tim.imobj.pixscale] * len(F),
dtype=np.float32)
F.ccd_cuts = np.array([ccd.ccd_cuts] * len(F))
F.airmass = np.array([ccd.airmass] * len(F), dtype=np.float32)
### --> also add units to the dict below so the FITS headers have units
F.sky = np.array([tim.midsky / tim.zpscale / tim.imobj.pixscale**2] *
len(F),
dtype=np.float32)
# in the same units as the depth maps -- flux inverse-variance.
F.psfdepth = np.array([(1. / (tim.sig1 / tim.psfnorm)**2)] * len(F),
dtype=np.float32)
F.galdepth = np.array([(1. / (tim.sig1 / tim.galnorm)**2)] * len(F),
dtype=np.float32)
F.fwhm = np.array([tim.psf_fwhm] * len(F), dtype=np.float32)
F.skyrms = np.array([ccd.skyrms] * len(F), dtype=np.float32)
F.ccdzpt = np.array([ccd.ccdzpt] * len(F), dtype=np.float32)
F.ccdrarms = np.array([ccd.ccdrarms] * len(F), dtype=np.float32)
F.ccddecrms = np.array([ccd.ccddecrms] * len(F), dtype=np.float32)
F.ccdphrms = np.array([ccd.ccdphrms] * len(F), dtype=np.float32)
if opt.derivs:
cosdec = np.cos(np.deg2rad(T.dec))
with np.errstate(divide='ignore', invalid='ignore'):
F.dra = (F.flux_dra / F.flux) * 3600. / cosdec
F.ddec = (F.flux_ddec / F.flux) * 3600.
F.dra[F.flux == 0] = 0.
F.ddec[F.flux == 0] = 0.
F.dra_ivar = F.flux_dra_ivar * (F.flux / 3600. * cosdec)**2
F.ddec_ivar = F.flux_ddec_ivar * (F.flux / 3600.)**2
F.delete_column('flux_dra')
F.delete_column('flux_ddec')
F.delete_column('flux_dra_ivar')
F.delete_column('flux_ddec_ivar')
F.flux = F.flux_fixed
F.flux_ivar = F.flux_fixed_ivar
F.delete_column('flux_fixed')
F.delete_column('flux_fixed_ivar')
for c in ['dra', 'ddec', 'dra_ivar', 'ddec_ivar', 'flux', 'flux_ivar']:
F.set(c, F.get(c).astype(np.float32))
F.ra = T.ra
F.dec = T.dec
_, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
h, w = tim.shape
ix = np.round(F.x).astype(int)
iy = np.round(F.y).astype(int)
F.dqmask = tim.dq[np.clip(iy, 0, h - 1), np.clip(ix, 0, w - 1)]
# Set an OUT-OF-BOUNDS bit.
F.dqmask[reduce(np.logical_or,
[ix < 0, ix >= w, iy < 0, iy >= h])] |= DQ_BITS['edge2']
program_name = sys.argv[0]
## FIXME -- from catalog?
release = 9999
version_hdr = get_version_header(program_name, surveydir, release)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn.strip())
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(
dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(
dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='PLVER', value=ccd.plver, comment='CP pipeline version'))
version_hdr.add_record(
dict(name='PLPROCID', value=ccd.plprocid, comment='CP pipeline id'))
version_hdr.add_record(
dict(name='PROCDATE', value=ccd.procdate, comment='CP image DATE'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
fitsio.write(opt.save_model, model_img, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
tnow = Time()
print('Forced phot:', tnow - tlast)
return F, version_hdr, outlier_mask, outlier_header
print(e.message)
else:
print(e)
print()
rtn = -1
print('Shutting down MPI pool...')
pool.shutdown()
print('Shut down MPI pool')
return rtn
if __name__ == '__main__':
from astrometry.util.ttime import Time, MemMeas
Time.add_measurement(MemMeas)
sys.exit(main())
# salloc -N 2 -C haswell -q interactive -t 04:00:00 --ntasks-per-node=32 --cpus-per-task=2
# module unload cray-mpich
# module load openmpi
# mpirun -n 64 --map-by core --rank-by node python -m mpi4py.futures legacypipe/mpi-runbrick.py --no-wise-ceres --brick 0715m657 --zoom 100 300 100 300 --run south --outdir $CSCRATCH/mpi --stage wise_forced
# cray-mpich version:
# srun -n 64 --distribution cyclic:cyclic python -m mpi4py.futures legacypipe/mpi-runbrick.py --no-wise-ceres --brick 0715m657 --zoom 100 300 100 300 --run south --outdir $CSCRATCH/mpi --stage wise_forced
#
#
# mpi4py setup:
# cray-mpich module, PrgEnv-intel/6.0.5
#
# mpi.cfg:
def main(decals=None, opt=None):
'''Driver function for forced photometry of individual DECam images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0,x1,y0,y1) = opt.zoom
zoomslice = (slice(y0,y1), slice(x0,x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.filename)
opt.filename = None
except:
# make this 'None' for decals.find_ccds()
expnum = None
# Try parsing HDU number
try:
opt.hdu = int(opt.hdu)
ccdname = None
except:
ccdname = opt.hdu
opt.hdu = -1
if decals is None:
decals = Decals()
if opt.filename is not None and opt.hdu >= 0:
# Read metadata from file
T = exposure_metadata([opt.filename], hdus=[opt.hdu])
print('Metadata:')
T.about()
else:
# Read metadata from decals-ccds.fits table
T = decals.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if opt.hdu >= 0:
T.cut(T.image_hdu == opt.hdu)
print(len(T), 'with HDU', opt.hdu)
if opt.filename is not None:
T.cut(np.array([f.strip() == opt.filename for f in T.image_filename]))
print(len(T), 'with filename', opt.filename)
assert(len(T) == 1)
im = decals.get_image_object(T[0])
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True)
print('Got tim:', tim)
if opt.catfn in ['DR1', 'DR2']:
if opt.catalog_path is None:
opt.catalog_path = opt.catfn.lower()
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, decals=decals)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3],
'tractor-%s.fits' % b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W,H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) *
(yy >= -margin) * (yy <= (H+margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print('Cut to', len(T), 'on out_of_bounds and left_blob')
TT.append(T)
T = merge_tables(TT)
T._header = TT[0]._header
del TT
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T
T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T
cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE)
# print('Got cat:', cat)
print('Forced photom...')
opti = None
if opt.ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
opti = CeresOptimizer(BW=B, BH=B)
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
for src in cat:
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
F = fits_table()
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T))
ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x-1).astype(np.float32)
F.y = (y-1).astype(np.float32)
if opt.apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
print('Apertures:', apertures, 'pixels')
for rad in apertures:
aper = photutils.CircularAperture(apxy, rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = ap
ap = 1./(np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux_ivar = ap
if opt.forced:
kwa = {}
if opt.plots is None:
kwa.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True, fitstats=True,
shared_params=False, **kwa)
if opt.plots:
(data,mod,ie,chi,roi) = R.ims1[0]
ima = tim.ima
imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5)
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
F.flux = np.array([src.getBrightness().getFlux(tim.band)
for src in cat]).astype(np.float32)
F.flux_ivar = R.IV.astype(np.float32)
F.fracflux = R.fitstats.profracflux.astype(np.float32)
F.rchi2 = R.fitstats.prochi2 .astype(np.float32)
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, decals.decals_dir)
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
fname = os.path.join(d2, d1, fname)
print('Trimmed filename to', fname)
#version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file'))
version_hdr.add_record(dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext'))
version_hdr.add_record(dict(name='CAMERA', value='DECam', comment='Dark Energy Camera'))
version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num'))
version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name'))
version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image'))
keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV',
'INSTRUME']
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {'mjd':'sec', 'exptime':'sec', 'flux':'nanomaggy',
'flux_ivar':'1/nanomaggy^2'}
columns = F.get_columns()
for i,col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
print('Finished forced phot:', Time()-t0)
return 0
def get_queryset(self):
print('get_queryset called')
t0 = Time()
req = self.request
self.form = CatalogSearchForm(req.GET)
if not self.form.is_valid():
print('FORM NOT VALID!')
return []
desc = ''
spatial = str(self.form.cleaned_data['spatial'])
print('Spatial search type: "%s"' % spatial)
print('type', type(spatial))
if spatial == 'cone':
ra, dec = parse_coord(self.form.cleaned_data['coord'])
rad = self.form.cleaned_data['radius']
if rad is None:
rad = 0.
rad = max(0., rad)
# 1 degree max!
# rad = min(rad, 1.)
self.radecradius = (ra, dec, rad)
print('q3c radial query:', ra, dec, rad)
cat = Tractor.objects.extra(where=[
'q3c_radial_query(tractor.ra, tractor.dec, %.4f, %.4f, %g)' %
(ra, dec, rad)
])
desc += 'near RA,Dec = (%.4f, %.4f) radius %f degrees' % (ra, dec,
rad)
elif spatial == 'allsky':
cat = Tractor.objects.all()
desc += 'all sky'
else:
print('Invalid spatial type "%s"' % spatial)
return []
terms = []
types = self.form.cleaned_data['sourcetypes']
print('Search for types:', types)
if len(types):
tt = []
for t in types:
if len(t) == 3:
t = t + ' '
tt.append(str(t))
if len(tt) == 1:
terms.append('Type is %s' % tt[0])
cat = cat.filter(type=tt[0])
else:
terms.append('Type in %s' % tt)
cat = cat.filter(type__in=tt)
for k in ['g', 'r', 'z', 'w1']: #, 'gmr', 'rmz', 'zmw1']:
gt = self.form.cleaned_data[k + '_gt']
lt = self.form.cleaned_data[k + '_lt']
#print('Limits for', k, ':', gt, lt)
if gt is not None:
cat = cat.filter(**{k + '__gt': gt})
if lt is not None:
cat = cat.filter(**{k + '__lt': lt})
if gt is not None and lt is not None:
terms.append(k + ' between %g and %g' % (gt, lt))
elif gt is not None:
terms.append(k + ' > %g' % gt)
elif lt is not None:
terms.append(k + ' < %g' % lt)
for k in ['gmr', 'rmz', 'zmw1']:
gt = self.form.cleaned_data[k + '_gt']
lt = self.form.cleaned_data[k + '_lt']
k1, k2 = k.split('m')
# DR2 -> DR3 -- "gmr" -> "g_r"
if k == 'gmr':
dbk = 'g_r'
if gt is not None:
cat = cat.filter(**{dbk + '__gt': gt})
if lt is not None:
cat = cat.filter(**{dbk + '__lt': lt})
else:
if gt is not None:
cat = cat.extra(where=['(%s - %s) > %f' % (k1, k2, gt)])
if lt is not None:
cat = cat.extra(where=['(%s - %s) < %f' % (k1, k2, lt)])
k = '(%s-%s)' % (k1, k2)
if gt is not None and lt is not None:
terms.append(k + ' between %g and %g' % (gt, lt))
elif gt is not None:
terms.append(k + ' > %g' % gt)
elif lt is not None:
terms.append(k + ' < %g' % lt)
if len(terms):
desc += ' where ' + ' and '.join(terms)
cat = cat[:1000]
self.querydesc = desc
#print('Got:', cat.count(), 'hits')
print('SQL:', cat.query)
print('Set query description:', desc)
print('Finished get_queryset in', Time() - t0)
return cat
def make_coadds(tims,
bands,
targetwcs,
mods=None,
xy=None,
apertures=None,
apxy=None,
ngood=False,
detmaps=False,
psfsize=False,
callback=None,
callback_args=[],
plots=False,
ps=None,
lanczos=True,
mp=None):
from astrometry.util.ttime import Time
t0 = Time()
class Duck(object):
pass
C = Duck()
W = int(targetwcs.get_width())
H = int(targetwcs.get_height())
# always, for patching SATUR, etc pixels?
unweighted = True
if not xy:
psfsize = False
C.coimgs = []
if detmaps:
C.galdetivs = []
C.detivs = []
if mods is not None:
C.comods = []
C.coresids = []
if apertures is not None:
unweighted = True
C.AP = fits_table()
if xy:
ix, iy = xy
C.T = fits_table()
C.T.nobs = np.zeros((len(ix), len(bands)), np.uint8)
C.T.anymask = np.zeros((len(ix), len(bands)), np.int16)
C.T.allmask = np.zeros((len(ix), len(bands)), np.int16)
if psfsize:
C.T.psfsize = np.zeros((len(ix), len(bands)), np.float32)
if detmaps:
C.T.depth = np.zeros((len(ix), len(bands)), np.float32)
C.T.galdepth = np.zeros((len(ix), len(bands)), np.float32)
if lanczos:
print('Doing Lanczos resampling')
for tim in tims:
# surface-brightness correction
tim.sbscale = (targetwcs.pixel_scale() / tim.subwcs.pixel_scale())**2
# We create one iterator per band to do the tim resampling. These all run in
# parallel when multi-processing.
imaps = []
for band in bands:
args = []
for itim, tim in enumerate(tims):
if tim.band != band:
continue
if mods is None:
mo = None
else:
mo = mods[itim]
args.append((itim, tim, mo, lanczos, targetwcs))
if mp is not None:
imaps.append(mp.imap_unordered(_resample_one, args))
else:
import itertools
imaps.append(itertools.imap(_resample_one, args))
# Args for aperture photometry
apargs = []
tinyw = 1e-30
for iband, (band, timiter) in enumerate(zip(bands, imaps)):
print('Computing coadd for band', band)
# coadded weight map (moo)
cow = np.zeros((H, W), np.float32)
# coadded weighted image map
cowimg = np.zeros((H, W), np.float32)
kwargs = dict(cowimg=cowimg, cow=cow)
if detmaps:
# detection map inverse-variance (depth map)
detiv = np.zeros((H, W), np.float32)
C.detivs.append(detiv)
kwargs.update(detiv=detiv)
# galaxy detection map inverse-variance (galdepth map)
galdetiv = np.zeros((H, W), np.float32)
C.galdetivs.append(galdetiv)
kwargs.update(galdetiv=galdetiv)
if mods is not None:
# model image
cowmod = np.zeros((H, W), np.float32)
# chi-squared image
cochi2 = np.zeros((H, W), np.float32)
kwargs.update(cowmod=cowmod, cochi2=cochi2)
if unweighted:
# unweighted image
coimg = np.zeros((H, W), np.float32)
if mods is not None:
# unweighted model
comod = np.zeros((H, W), np.float32)
# number of exposures
con = np.zeros((H, W), np.uint8)
# inverse-variance
coiv = np.zeros((H, W), np.float32)
kwargs.update(coimg=coimg, coiv=coiv)
# Note that we have 'congood' as well as 'nobs':
# * 'congood' is used for the 'nexp' *image*.
# * 'nobs' is used for the per-source measurements
#
# (you want to know the number of observations within the
# source footprint, not just the peak pixel which may be
# saturated, etc.)
if ngood:
congood = np.zeros((H, W), np.uint8)
kwargs.update(congood=congood)
if xy:
# These match the type of the "DQ" images.
# "any" mask
ormask = np.zeros((H, W), np.int16)
# "all" mask
andmask = np.empty((H, W), np.int16)
allbits = reduce(np.bitwise_or, CP_DQ_BITS.values())
andmask[:, :] = allbits
# number of observations
nobs = np.zeros((H, W), np.uint8)
kwargs.update(ormask=ormask, andmask=andmask, nobs=nobs)
if psfsize:
psfsizemap = np.zeros((H, W), np.float32)
for R in timiter:
if R is None:
continue
itim, Yo, Xo, iv, im, mo, dq = R
#print('timiter Yo,Xo,im.shape=',Yo,Xo,im.shape)
tim = tims[itim]
# invvar-weighted image
cowimg[Yo, Xo] += iv * im
cow[Yo, Xo] += iv
if unweighted:
if dq is None:
goodpix = 1
else:
# include BLEED, SATUR, INTERP pixels if no other
# pixels exists (do this by eliminating all other CP
# flags)
badbits = 0
for bitname in ['badpix', 'cr', 'trans', 'edge', 'edge2']:
badbits |= CP_DQ_BITS[bitname]
goodpix = ((dq & badbits) == 0)
coimg[Yo, Xo] += goodpix * im
con[Yo, Xo] += goodpix
coiv[Yo, Xo] += goodpix * 1. / (tim.sig1 *
tim.sbscale)**2 # ...ish
if xy:
if dq is not None:
ormask[Yo, Xo] |= dq
andmask[Yo, Xo] &= dq
# raw exposure count
nobs[Yo, Xo] += 1
if psfsize:
# psfnorm is in units of 1/pixels.
# (eg, psfnorm for a gaussian is ~ 1/psf_sigma)
# Neff is in pixels**2
neff = 1. / tim.psfnorm**2
# Narcsec is in arcsec**2
narcsec = neff * tim.wcs.pixel_scale()**2
psfsizemap[Yo, Xo] += iv * (1. / narcsec)
if detmaps:
# point-source depth
detsig1 = tim.sig1 / tim.psfnorm
detiv[Yo, Xo] += (iv > 0) * (1. / detsig1**2)
# Galaxy detection map
gdetsig1 = tim.sig1 / tim.galnorm
galdetiv[Yo, Xo] += (iv > 0) * (1. / gdetsig1**2)
if ngood:
congood[Yo, Xo] += (iv > 0)
if mods is not None:
# straight-up
comod[Yo, Xo] += goodpix * mo
# invvar-weighted
cowmod[Yo, Xo] += iv * mo
# chi-squared
cochi2[Yo, Xo] += iv * (im - mo)**2
del mo
del goodpix
del Yo, Xo, im, iv
# END of loop over tims
# Per-band:
cowimg /= np.maximum(cow, tinyw)
C.coimgs.append(cowimg)
if mods is not None:
cowmod /= np.maximum(cow, tinyw)
C.comods.append(cowmod)
coresid = cowimg - cowmod
coresid[cow == 0] = 0.
C.coresids.append(coresid)
if unweighted:
coimg /= np.maximum(con, 1)
del con
cowimg[cow == 0] = coimg[cow == 0]
if mods is not None:
cowmod[cow == 0] = comod[cow == 0]
if xy:
C.T.nobs[:, iband] = nobs[iy, ix]
C.T.anymask[:, iband] = ormask[iy, ix]
C.T.allmask[:, iband] = andmask[iy, ix]
# unless there were no images there...
C.T.allmask[nobs[iy, ix] == 0, iband] = 0
if detmaps:
C.T.depth[:, iband] = detiv[iy, ix]
C.T.galdepth[:, iband] = galdetiv[iy, ix]
if psfsize:
wt = cow[iy, ix]
# psfsizemap is in units of iv * (1 / arcsec**2)
sz = psfsizemap[iy, ix]
sz /= np.maximum(wt, tinyw)
sz[wt == 0] = 0.
# Back to units of linear arcsec.
sz = 1. / np.sqrt(sz)
sz[wt == 0] = 0.
# Correction factor to get back to equivalent of Gaussian sigma
sz /= (2. * np.sqrt(np.pi))
# Conversion factor to FWHM (2.35)
sz *= 2. * np.sqrt(2. * np.log(2.))
C.T.psfsize[:, iband] = sz
del psfsizemap
if apertures is not None:
# Aperture photometry, using the unweighted "coimg" and
# "coiv" arrays.
with np.errstate(divide='ignore'):
imsigma = 1.0 / np.sqrt(coiv)
imsigma[coiv == 0] = 0
for irad, rad in enumerate(apertures):
apargs.append((irad, band, rad, coimg, imsigma, True, apxy))
if mods is not None:
apargs.append(
(irad, band, rad, coresid, None, False, apxy))
if callback is not None:
callback(band, *callback_args, **kwargs)
# END of loop over bands
t2 = Time()
print('coadds: images:', t2 - t0)
if apertures is not None:
# Aperture phot, in parallel
if mp is not None:
apresults = mp.map(_apphot_one, apargs)
else:
apresults = map(_apphot_one, apargs)
del apargs
apresults = iter(apresults)
for iband, band in enumerate(bands):
apimg = []
apimgerr = []
if mods is not None:
apres = []
for irad, rad in enumerate(apertures):
(airad, aband, isimg, ap_img, ap_err) = apresults.next()
assert (airad == irad)
assert (aband == band)
assert (isimg)
apimg.append(ap_img)
apimgerr.append(ap_err)
if mods is not None:
(airad, aband, isimg, ap_img, ap_err) = apresults.next()
assert (airad == irad)
assert (aband == band)
assert (not isimg)
apres.append(ap_img)
assert (ap_err is None)
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
C.AP.set('apflux_img_%s' % band, ap)
ap = 1. / (np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
C.AP.set('apflux_img_ivar_%s' % band, ap)
if mods is not None:
ap = np.vstack(apres).T
ap[np.logical_not(np.isfinite(ap))] = 0.
C.AP.set('apflux_resid_%s' % band, ap)
t3 = Time()
print('coadds apphot:', t3 - t2)
return C
def __init__(self, *args, **kwargs):
super(MyMultiproc, self).__init__(*args, **kwargs)
self.t0 = Time()
self.serial = []
self.parallel = []
self.phases = []
def ptime(text,t0):
'''Timer'''
tnow=Time()
print('TIMING:%s ' % text,tnow-t0)
return tnow
def run_one_ccd(survey, catsurvey_north, catsurvey_south, resolve_dec, ccd,
opt, zoomslice, ps):
tlast = Time()
im = survey.get_image_object(ccd)
if opt.do_calib:
im.run_calibs(splinesky=True)
tim = im.get_tractor_image(slc=zoomslice,
pixPsf=True,
splinesky=True,
constant_invvar=opt.constant_invvar,
hybridPsf=opt.hybrid_psf,
normalizePsf=opt.normalize_psf,
old_calibs_ok=True)
print('Got tim:', tim, 'x0,y0', tim.x0, tim.y0)
tnow = Time()
print('Read image:', tnow - tlast)
tlast = tnow
# Apply outlier masks
if True:
# Outliers masks are computed within a survey (north/south for dr8), and are stored
# in a brick-oriented way, in the results directories.
north_ccd = (ccd.camera.strip() != 'decam')
catsurvey = catsurvey_north
if not north_ccd and catsurvey_south is not None:
catsurvey = catsurvey_south
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
for b in bricks:
from legacypipe.outliers import read_outlier_mask_file
print('Reading outlier mask for brick', b.brickname)
ok = read_outlier_mask_file(catsurvey, [tim],
b.brickname,
subimage=False,
output=False,
ps=ps)
if not ok:
print('WARNING: failed to read outliers mask file for brick',
b.brickname)
if opt.catalog:
T = fits_table(opt.catalog)
else:
chipwcs = tim.subwcs
T = get_catalog_in_wcs(chipwcs,
catsurvey_north,
catsurvey_south=catsurvey_south,
resolve_dec=resolve_dec)
if T is None:
print('No sources to photometer.')
return None
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
surveydir = survey.get_survey_dir()
del survey
if opt.move_gaia:
# Gaia stars: move RA,Dec to the epoch of this image.
I = np.flatnonzero(T.ref_epoch > 0)
if len(I):
from legacypipe.survey import radec_at_mjd
print('Moving', len(I), 'Gaia stars to MJD', tim.time.toMjd())
ra, dec = radec_at_mjd(T.ra[I], T.dec[I],
T.ref_epoch[I].astype(float), T.pmra[I],
T.pmdec[I], T.parallax[I], tim.time.toMjd())
T.ra[I] = ra
T.dec[I] = dec
tnow = Time()
print('Read catalog:', tnow - tlast)
tlast = tnow
cat = read_fits_catalog(T, bands='r')
# Replace the brightness (which will be a NanoMaggies with g,r,z)
# with a NanoMaggies with this image's band only.
for src in cat:
src.brightness = NanoMaggies(**{tim.band: 1.})
tnow = Time()
print('Parse catalog:', tnow - tlast)
tlast = tnow
print('Forced photom...')
F = run_forced_phot(cat,
tim,
ceres=opt.ceres,
derivs=opt.derivs,
fixed_also=True,
agn=opt.agn,
do_forced=opt.forced,
do_apphot=opt.apphot,
get_model=opt.save_model,
ps=ps,
timing=True,
ceres_threads=opt.ceres_threads)
if opt.save_model:
# unpack results
F, model_img = F
F.release = T.release
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.camera = np.array([ccd.camera] * len(F))
F.expnum = np.array([im.expnum] * len(F)).astype(np.int64)
F.ccdname = np.array([im.ccdname] * len(F))
# "Denormalizing"
F.filter = np.array([tim.band] * len(F))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(F))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(F)).astype(np.float32)
F.psfsize = np.array([tim.psf_fwhm * tim.imobj.pixscale] * len(F)).astype(
np.float32)
F.ccd_cuts = np.array([ccd.ccd_cuts] * len(F))
F.airmass = np.array([ccd.airmass] * len(F))
### --> also add units to the dict below so the FITS headers have units
F.sky = np.array([tim.midsky / tim.zpscale / tim.imobj.pixscale**2] *
len(F)).astype(np.float32)
# in the same units as the depth maps -- flux inverse-variance.
F.psfdepth = np.array([(1. / (tim.sig1 / tim.psfnorm)**2)] *
len(F)).astype(np.float32)
F.galdepth = np.array([(1. / (tim.sig1 / tim.galnorm)**2)] *
len(F)).astype(np.float32)
# F.psfdepth = np.array([-2.5 * (np.log10(5. * tim.sig1 / tim.psfnorm) - 9)] * len(F)).astype(np.float32)
# F.galdepth = np.array([-2.5 * (np.log10(5. * tim.sig1 / tim.galnorm) - 9)] * len(F)).astype(np.float32)
# super units questions here
if opt.derivs:
cosdec = np.cos(np.deg2rad(T.dec))
F.dra = (F.flux_dra / F.flux) * 3600. / cosdec
F.ddec = (F.flux_ddec / F.flux) * 3600.
F.dra_ivar = F.flux_dra_ivar * (F.flux / 3600. * cosdec)**2
F.ddec_ivar = F.flux_ddec_ivar * (F.flux / 3600.)**2
F.delete_column('flux_dra')
F.delete_column('flux_ddec')
F.delete_column('flux_dra_ivar')
F.delete_column('flux_ddec_ivar')
F.flux = F.flux_fixed
F.flux_ivar = F.flux_fixed_ivar
F.delete_column('flux_fixed')
F.delete_column('flux_fixed_ivar')
for c in ['dra', 'ddec', 'dra_ivar', 'ddec_ivar', 'flux', 'flux_ivar']:
F.set(c, F.get(c).astype(np.float32))
F.ra = T.ra
F.dec = T.dec
ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
h, w = tim.shape
F.dqmask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h - 1),
np.clip(np.round(F.x).astype(int), 0, w - 1)]
program_name = sys.argv[0]
## FIXME -- from catalog?
release = 8002
version_hdr = get_version_header(program_name, surveydir, release)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn.strip())
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(
dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(
dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='PLVER', value=ccd.plver, comment='CP pipeline version'))
version_hdr.add_record(
dict(name='PLPROCID', value=ccd.plprocid, comment='CP pipeline id'))
version_hdr.add_record(
dict(name='PROCDATE', value=ccd.procdate, comment='CP image DATE'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
fitsio.write(opt.save_model, model_img, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
tnow = Time()
print('Forced phot:', tnow - tlast)
return F, version_hdr
def unwise_forcedphot(cat, tiles, bands=[1, 2, 3, 4], roiradecbox=None,
unwise_dir='.',
use_ceres=True, ceres_block=8,
save_fits=False, ps=None,
psf_broadening=None):
'''
Given a list of tractor sources *cat*
and a list of unWISE tiles *tiles* (a fits_table with RA,Dec,coadd_id)
runs forced photometry, returning a FITS table the same length as *cat*.
'''
# Severely limit sizes of models
for src in cat:
if isinstance(src, PointSource):
src.fixedRadius = 10
else:
src.halfsize = 10
wantims = ((ps is not None) or save_fits)
wanyband = 'w'
fskeys = ['prochi2', 'pronpix', 'profracflux', 'proflux', 'npix',
'pronexp']
Nsrcs = len(cat)
phot = fits_table()
phot.tile = np.array([' '] * Nsrcs)
ra = np.array([src.getPosition().ra for src in cat])
dec = np.array([src.getPosition().dec for src in cat])
for band in bands:
print('Photometering WISE band', band)
wband = 'w%i' % band
# The tiles have some overlap, so for each source, keep the
# fit in the tile whose center is closest to the source.
tiledists = np.empty(Nsrcs)
tiledists[:] = 1e100
flux_invvars = np.zeros(Nsrcs, np.float32)
fitstats = dict([(k, np.zeros(Nsrcs, np.float32)) for k in fskeys])
nexp = np.zeros(Nsrcs, np.int16)
mjd = np.zeros(Nsrcs, np.float64)
for tile in tiles:
print('Reading tile', tile.coadd_id)
tim = get_unwise_tractor_image(unwise_dir, tile.coadd_id, band,
bandname=wanyband, roiradecbox=roiradecbox)
if tim is None:
print('Actually, no overlap with tile', tile.coadd_id)
continue
if psf_broadening is not None:
# psf_broadening is a factor by which the PSF FWHMs
# should be scaled; the PSF is a little wider
# post-reactivation.
psf = tim.getPsf()
from tractor import GaussianMixturePSF
if isinstance(psf, GaussianMixturePSF):
#
print('Broadening PSF: from', psf)
p0 = psf.getParams()
#print('Params:', p0)
pnames = psf.getParamNames()
#print('Param names:', pnames)
p1 = [p * psf_broadening**2 if 'var' in name else p
for (p, name) in zip(p0, pnames)]
#print('Broadened:', p1)
psf.setParams(p1)
print('Broadened PSF:', psf)
else:
print(
'WARNING: cannot apply psf_broadening to WISE PSF of type', type(psf))
print('Read image with shape', tim.shape)
# Select sources in play.
wcs = tim.wcs.wcs
H, W = tim.shape
ok, x, y = wcs.radec2pixelxy(ra, dec)
x = (x - 1.).astype(np.float32)
y = (y - 1.).astype(np.float32)
margin = 10.
I = np.flatnonzero((x >= -margin) * (x < W + margin) *
(y >= -margin) * (y < H + margin))
print(len(I), 'within the image + margin')
inbox = ((x[I] >= -0.5) * (x[I] < (W - 0.5)) *
(y[I] >= -0.5) * (y[I] < (H - 0.5)))
print(sum(inbox), 'strictly within the image')
# Compute L_inf distance to (full) tile center.
tilewcs = unwise_tile_wcs(tile.ra, tile.dec)
cx, cy = tilewcs.crpix
ok, tx, ty = tilewcs.radec2pixelxy(ra[I], dec[I])
td = np.maximum(np.abs(tx - cx), np.abs(ty - cy))
closest = (td < tiledists[I])
tiledists[I[closest]] = td[closest]
keep = inbox * closest
# Source indices (in the full "cat") to keep (the fit values for)
srci = I[keep]
if not len(srci):
print('No sources to be kept; skipping.')
continue
phot.tile[srci] = tile.coadd_id
nexp[srci] = tim.nuims[np.clip(np.round(y[srci]).astype(int), 0, H - 1),
np.clip(np.round(x[srci]).astype(int), 0, W - 1)]
# Source indices in the margins
margi = I[np.logical_not(keep)]
# sources in the box -- at the start of the subcat list.
subcat = [cat[i] for i in srci]
# include *copies* of sources in the margins
# (that way we automatically don't save the results)
subcat.extend([cat[i].copy() for i in margi])
assert(len(subcat) == len(I))
# FIXME -- set source radii, ...?
minsb = 0.
fitsky = False
# Look in image and set radius based on peak height??
tractor = Tractor([tim], subcat)
if use_ceres:
from tractor.ceres_optimizer import CeresOptimizer
tractor.optimizer = CeresOptimizer(BW=ceres_block,
BH=ceres_block)
tractor.freezeParamsRecursive('*')
tractor.thawPathsTo(wanyband)
kwa = dict(fitstat_extras=[('pronexp', [tim.nims])])
t0 = Time()
R = tractor.optimize_forced_photometry(
minsb=minsb, mindlnp=1., sky=fitsky, fitstats=True,
variance=True, shared_params=False,
wantims=wantims, **kwa)
print('unWISE forced photometry took', Time() - t0)
if use_ceres:
term = R.ceres_status['termination']
print('Ceres termination status:', term)
# Running out of memory can cause failure to converge
# and term status = 2.
# Fail completely in this case.
if term != 0:
raise RuntimeError(
'Ceres terminated with status %i' % term)
if wantims:
ims0 = R.ims0
ims1 = R.ims1
IV, fs = R.IV, R.fitstats
if save_fits:
import fitsio
(dat, mod, ie, chi, roi) = ims1[0]
wcshdr = fitsio.FITSHDR()
tim.wcs.wcs.add_to_header(wcshdr)
tag = 'fit-%s-w%i' % (tile.coadd_id, band)
fitsio.write('%s-data.fits' %
tag, dat, clobber=True, header=wcshdr)
fitsio.write('%s-mod.fits' % tag, mod,
clobber=True, header=wcshdr)
fitsio.write('%s-chi.fits' % tag, chi,
clobber=True, header=wcshdr)
if ps:
tag = '%s W%i' % (tile.coadd_id, band)
(dat, mod, ie, chi, roi) = ims1[0]
sig1 = tim.sig1
plt.clf()
plt.imshow(dat, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
plt.colorbar()
plt.title('%s: data' % tag)
ps.savefig()
plt.clf()
plt.imshow(mod, interpolation='nearest', origin='lower',
cmap='gray', vmin=-3 * sig1, vmax=10 * sig1)
plt.colorbar()
plt.title('%s: model' % tag)
ps.savefig()
plt.clf()
plt.imshow(chi, interpolation='nearest', origin='lower',
cmap='gray', vmin=-5, vmax=+5)
plt.colorbar()
plt.title('%s: chi' % tag)
ps.savefig()
# Save results for this tile.
# the "keep" sources are at the beginning of the "subcat" list
flux_invvars[srci] = IV[:len(srci)].astype(np.float32)
if hasattr(tim, 'mjdmin') and hasattr(tim, 'mjdmax'):
mjd[srci] = (tim.mjdmin + tim.mjdmax) / 2.
if fs is None:
continue
for k in fskeys:
x = getattr(fs, k)
# fitstats are returned only for un-frozen sources
fitstats[k][srci] = np.array(x).astype(np.float32)[:len(srci)]
# Note, this is *outside* the loop over tiles.
# The fluxes are saved in the source objects, and will be set based on
# the 'tiledists' logic above.
nm = np.array([src.getBrightness().getBand(wanyband) for src in cat])
nm_ivar = flux_invvars
# Sources out of bounds, eg, never change from their default
# (1-sigma or whatever) initial fluxes. Zero them out instead.
nm[nm_ivar == 0] = 0.
phot.set(wband + '_nanomaggies', nm.astype(np.float32))
phot.set(wband + '_nanomaggies_ivar', nm_ivar)
dnm = np.zeros(len(nm_ivar), np.float32)
okiv = (nm_ivar > 0)
dnm[okiv] = (1. / np.sqrt(nm_ivar[okiv])).astype(np.float32)
okflux = (nm > 0)
mag = np.zeros(len(nm), np.float32)
mag[okflux] = (NanoMaggies.nanomaggiesToMag(nm[okflux])
).astype(np.float32)
dmag = np.zeros(len(nm), np.float32)
ok = (okiv * okflux)
dmag[ok] = (np.abs((-2.5 / np.log(10.)) * dnm[ok] / nm[ok])
).astype(np.float32)
mag[np.logical_not(okflux)] = np.nan
dmag[np.logical_not(ok)] = np.nan
phot.set(wband + '_mag', mag)
phot.set(wband + '_mag_err', dmag)
for k in fskeys:
phot.set(wband + '_' + k, fitstats[k])
phot.set(wband + '_nexp', nexp)
if not np.all(mjd == 0):
phot.set(wband + '_mjd', mjd)
return phot
def main(args=None):
"""Main routine which parses the optional inputs."""
t0= Time()
# Command line options
if args is None:
# Read from cmd line
parser= get_parser()
args = parser.parse_args(args=args)
else:
# args is already a argparse.Namespace obj
pass
# Print calling sequence
print('Args:', args)
if args.do_more == 'yes':
assert(not args.minid is None)
# Setup loggers
if args.verbose:
lvl = logging.DEBUG
else:
lvl = logging.INFO
logging.basicConfig(level=lvl, stream=sys.stdout) #,format='%(message)s')
log = logging.getLogger('decals_sim')
# Sort through args
#log.info('decals_sim.py args={}'.format(args))
#max_nobj=500
#max_nchunk=1000
#if args.ith_chunk is not None: assert(args.ith_chunk <= max_nchunk-1)
#assert(args.nchunk <= max_nchunk)
#assert(args.nobj <= max_nobj)
#if args.ith_chunk is not None:
# assert(args.nchunk == 1) #if choose a chunk, only doing 1 chunk
if args.nobj is None:
parser.print_help()
sys.exit(1)
# Exit if expected output already exists
rsdir= get_outdir_runbrick(args.outdir,
args.brick,args.rowstart,
do_skipids=args.do_skipids,
do_more=args.do_more)
rsdir= os.path.basename(rsdir)
tractor_fn= os.path.join(args.outdir,
'tractor',args.brick[:3],args.brick,
rsdir,
'tractor-%s.fits' % args.brick)
if (os.path.exists(tractor_fn) &
(not args.overwrite_if_exists)):
print('Exiting, already finished %s' % tractor_fn)
return 0 #sys.exit(0)
brickname = args.brick
objtype = args.objtype
# Output dir
decals_sim_dir = args.outdir
#nchunk = args.nchunk
#rand = np.random.RandomState(args.seed) # determines seed for all chunks
#seeds = rand.random_integers(0,2**18, max_nchunk)
log.info('Object type = {}'.format(objtype))
#log.info('Number of objects = {}'.format(nobj))
#log.info('Number of chunks = {}'.format(nchunk))
# Optionally zoom into a portion of the brick
survey = LegacySurveyData(survey_dir=args.survey_dir)
brickinfo= get_brickinfo_hack(survey,brickname)
#brickinfo = survey.get_brick_by_name(brickname)
#print(brickname)
brickwcs = wcs_for_brick(brickinfo)
W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale()
log.info('Brick = {}'.format(brickname))
if args.zoom is not None: # See also runbrick.stage_tims()
(x0, x1, y0, y1) = args.zoom
W = x1 - x0
H = y1 - y0
brickwcs = brickwcs.get_subimage(x0, y0, W, H)
log.info('Zoom (pixel boundaries) = {}'.format(args.zoom))
targetrd = np.array([brickwcs.pixelxy2radec(x, y) for x, y in
[(1,1), (W,1), (W,H), (1,H), (1,1)]])
radec_center = brickwcs.radec_center()
log.info('RA, Dec center = {}'.format(radec_center))
log.info('Brick = {}'.format(brickname))
t0= ptime('First part of Main()',t0)
# SAMPLE table
sample_kwargs= {"objtype":args.objtype,
"brick":args.brick,
"outdir":args.outdir,
"randoms_db":args.randoms_db,
"minid":args.minid,
"do_skipids":args.do_skipids,
"randoms_from_fits":args.randoms_from_fits,
"dont_sort_sampleid":args.dont_sort_sampleid}
Samp,seed= get_sample(**sample_kwargs)
Samp= Samp[args.rowstart:args.rowstart + args.nobj]
# Performance
#if objtype in ['elg','lrg']:
# Samp=Samp[np.argsort( Samp.get('%s_n' % objtype) )]
print('Max sample size=%d, actual sample size=%d' % (args.nobj,len(Samp)))
assert(len(Samp) <= args.nobj)
t0= ptime('Got randoms sample',t0)
# Store args in dict for easy func passing
kwargs=dict(Samp=Samp,\
brickname=brickname, \
checkpoint=args.checkpoint, \
seed= seed,
decals_sim_dir= decals_sim_dir,\
brickwcs= brickwcs, \
objtype=objtype,\
nobj=len(Samp),\
maxobjs=args.nobj,\
rowst=args.rowstart,\
do_skipids=args.do_skipids,\
do_more=args.do_more,\
minid=args.minid,\
survey_dir=args.survey_dir,\
args=args)
# Stop if starting row exceeds length of radec,color table
if len(Samp) == 0:
fn= get_outdir_runbrick(kwargs['decals_sim_dir'],
kwargs['brickname'],kwargs['rowst'],
do_skipids=kwargs['do_skipids'],do_more=kwargs['do_more'])
fn+= '_exceeded.txt'
junk= os.system('touch %s' % fn)
print('Wrote %s' % fn)
#we want not to add any sample -- obiwan
#raise ValueError('starting row=%d exceeds number of artificial sources, quit' % args.rowstart)
# Create simulated catalogues and run Tractor
create_metadata(kwargs=kwargs)
t0= ptime('create_metadata',t0)
# do chunks
#for ith_chunk in chunk_list:
#log.info('Working on chunk {:02d}/{:02d}'.format(ith_chunk,kwargs['nchunk']-1))
# Random ra,dec and source properties
create_ith_simcat(d=kwargs)
#log.info('HUI-TEST:::out of create_ith_simcat')
t0= ptime('create_ith_simcat',t0)
# Run tractor
#log.info('HUI-TEST:::running tractor')
do_one_chunk(d=kwargs)
#log.info('HUI-TEST::: checkpoint3i')
t0= ptime('do_one_chunk',t0)
# Clean up output
if args.no_cleanup == False:
do_ith_cleanup(d=kwargs)
#log.info('HUI-TEST::: checkpoint3j')
t0= ptime('do_ith_cleanup',t0)
log.info('All done!')
#log.info('HUI-TEST::: checkpoint3k')
return 0
def main(survey=None, opt=None, args=None):
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if args is None:
args = sys.argv[1:]
print('forced_photom.py', ' '.join(args))
if opt is None:
parser = get_parser()
opt = parser.parse_args(args)
import logging
if opt.verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
t0 = Time()
if survey is None:
survey = LegacySurveyData(survey_dir=opt.survey_dir,
cache_dir=opt.cache_dir,
output_dir=opt.out_dir)
if opt.skip:
if opt.out is not None:
outfn = opt.out
else:
outfn = survey.find_file('forced',
output=True,
camera=opt.camera,
expnum=opt.expnum)
if os.path.exists(outfn):
print('Ouput file exists:', outfn)
return 0
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
return -1
if opt.out is None and opt.out_dir is None:
print('Must supply either --out or --out-dir')
return -1
if opt.expnum is None and opt.out is None:
print('If no --expnum is given, must supply --out filename')
return -1
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Cache CCDs files before the find_ccds call...
# Copy required files into the cache?
if opt.pre_cache:
def copy_files_to_cache(fns):
for fn in fns:
cachefn = fn.replace(survey.survey_dir, survey.cache_dir)
if not cachefn.startswith(survey.cache_dir):
print('Skipping', fn)
continue
outdir = os.path.dirname(cachefn)
trymakedirs(outdir)
print('Copy', fn)
print(' to', cachefn)
shutil.copyfile(fn, cachefn)
assert (survey.cache_dir is not None)
fnset = set()
fn = survey.find_file('bricks')
fnset.add(fn)
fns = survey.find_file('ccd-kds')
fnset.update(fns)
copy_files_to_cache(fnset)
# Read metadata from survey-ccds.fits table
ccds = survey.find_ccds(camera=opt.camera,
expnum=opt.expnum,
ccdname=opt.ccdname)
print(len(ccds), 'with camera', opt.camera, 'and expnum', opt.expnum,
'and ccdname', opt.ccdname)
# sort CCDs
ccds.cut(np.lexsort((ccds.ccdname, ccds.expnum, ccds.camera)))
# If there is only one catalog survey_dir, we pass it to get_catalog_in_wcs
# as the northern survey.
catsurvey_north = survey
catsurvey_south = None
if opt.catalog_dir_north is not None:
assert (opt.catalog_dir_south is not None)
assert (opt.catalog_resolve_dec_ngc is not None)
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)
elif opt.catalog_dir is not None:
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)
# Copy required CCD & calib files into the cache?
if opt.pre_cache:
assert (survey.cache_dir is not None)
fnset = set()
for ccd in ccds:
im = survey.get_image_object(ccd)
for key in im.get_cacheable_filename_variables():
fn = getattr(im, key)
if fn is None or not (os.path.exists(fn)):
continue
fnset.add(fn)
copy_files_to_cache(fnset)
args = []
for ccd in ccds:
args.append((survey, catsurvey_north, catsurvey_south,
opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))
if opt.threads:
from astrometry.util.multiproc import multiproc
from astrometry.util.timingpool import TimingPool, TimingPoolMeas
pool = TimingPool(opt.threads)
poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
Time.add_measurement(poolmeas)
mp = multiproc(None, pool=pool)
tm = Time()
FF = mp.map(bounce_one_ccd, args)
print('Multi-processing forced-phot:', Time() - tm)
del mp
Time.measurements.remove(poolmeas)
del poolmeas
pool.close()
pool.join()
del pool
else:
FF = map(bounce_one_ccd, args)
FF = [F for F in FF if F is not None]
if len(FF) == 0:
print('No photometry results to write.')
return 0
# Keep only the first header
_, version_hdr, _, _ = FF[0]
# unpack results
outlier_masks = [m for _, _, m, _ in FF]
outlier_hdrs = [h for _, _, _, h in FF]
FF = [F for F, _, _, _ in FF]
F = merge_tables(FF)
if len(ccds):
version_hdr.delete('CPHDU')
version_hdr.delete('CCDNAME')
from legacypipe.utils import add_bits
from legacypipe.bits import DQ_BITS
add_bits(version_hdr, DQ_BITS, 'DQMASK', 'DQ', 'D')
from legacyzpts.psfzpt_cuts import CCD_CUT_BITS
add_bits(version_hdr, CCD_CUT_BITS, 'CCD_CUTS', 'CC', 'C')
for i, ap in enumerate(apertures_arcsec):
version_hdr.add_record(
dict(name='APRAD%i' % i,
value=ap,
comment='(optical) Aperture radius, in arcsec'))
unitmap = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2',
'apflux': 'nanomaggy',
'apflux_ivar': '1/nanomaggy^2',
'psfdepth': '1/nanomaggy^2',
'galdepth': '1/nanomaggy^2',
'sky': 'nanomaggy/arcsec^2',
'psfsize': 'arcsec',
'fwhm': 'pixels',
'ccdrarms': 'arcsec',
'ccddecrms': 'arcsec',
'ra': 'deg',
'dec': 'deg',
'skyrms': 'counts/sec',
'dra': 'arcsec',
'ddec': 'arcsec',
'dra_ivar': '1/arcsec^2',
'ddec_ivar': '1/arcsec^2'
}
columns = F.get_columns()
order = [
'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'fwhm', 'ccd_cuts',
'airmass', 'sky', 'skyrms', 'psfdepth', 'galdepth', 'ccdzpt',
'ccdrarms', 'ccddecrms', 'ccdphrms', 'ra', 'dec', 'flux', 'flux_ivar',
'fracflux', 'rchisq', 'fracmasked', 'fracin', 'apflux', 'apflux_ivar',
'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar', 'ddec_ivar'
]
columns = [c for c in order if c in columns]
units = [unitmap.get(c, '') for c in columns]
if opt.out is not None:
outdir = os.path.dirname(opt.out)
if len(outdir):
trymakedirs(outdir)
tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.out))
fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
F.writeto(tmpfn, units=units, append=True, columns=columns)
os.rename(tmpfn, opt.out)
print('Wrote', opt.out)
else:
with survey.write_output('forced',
camera=opt.camera,
expnum=opt.expnum) as out:
F.writeto(None,
fits_object=out.fits,
primheader=version_hdr,
units=units,
columns=columns)
print('Wrote', out.real_fn)
if opt.outlier_mask is not None:
# Add outlier bit meanings to the primary header
version_hdr.add_record(
dict(name='COMMENT', value='Outlier mask bit meanings'))
version_hdr.add_record(
dict(name='OUTL_POS',
value=1,
comment='Outlier mask bit for Positive outlier'))
version_hdr.add_record(
dict(name='OUTL_NEG',
value=2,
comment='Outlier mask bit for Negative outlier'))
if opt.outlier_mask == 'default':
outdir = os.path.join(opt.out_dir, 'outlier-masks')
camexp = set(zip(ccds.camera, ccds.expnum))
for c, e in camexp:
I = np.flatnonzero((ccds.camera == c) * (ccds.expnum == e))
ccd = ccds[I[0]]
imfn = ccd.image_filename.strip()
outfn = os.path.join(outdir, imfn.replace('.fits',
'-outlier.fits'))
trymakedirs(outfn, dir=True)
tempfn = outfn.replace('.fits', '-tmp.fits')
with fitsio.FITS(tempfn, 'rw', clobber=True) as fits:
fits.write(None, header=version_hdr)
for i in I:
mask = outlier_masks[i]
_, _, _, meth, tile = survey.get_compression_args(
'outliers_mask', shape=mask.shape)
fits.write(mask,
header=outlier_hdrs[i],
extname=ccds.ccdname[i],
compress=meth,
tile_dims=tile)
os.rename(tempfn, outfn)
print('Wrote', outfn)
elif opt.outlier_mask is not None:
with fitsio.FITS(opt.outlier_mask, 'rw', clobber=True) as F:
F.write(None, header=version_hdr)
for i, (hdr, mask) in enumerate(zip(outlier_hdrs, outlier_masks)):
_, _, _, meth, tile = survey.get_compression_args(
'outliers_mask', shape=mask.shape)
F.write(mask,
header=hdr,
extname=ccds.ccdname[i],
compress=meth,
tile_dims=tile)
print('Wrote', opt.outlier_mask)
tnow = Time()
print('Total:', tnow - t0)
return 0
def get_tractor_image(self, **kwargs):
#t0 = time_builtin.clock()
tim = super(SimImage, self).get_tractor_image(**kwargs)
#debug: setting image to 0 by hui
#tim.data = np.zeros(tim.data.shape)
#print('get_tractor_image:time'+str(time_builtin.clock()-t0))
if tim is None: # this can be None when the edge of a CCD overlaps
return tim
# Seed
#if 'SEED' in self.survey.metacat.columns:
# seed = self.survey.metacat['SEED']
#else:
# seed = None
t1=time_builtin.clock()
objtype = self.survey.metacat.get('objtype')[0]
objstamp = BuildStamp(tim, seed=self.survey.seed,
camera=self.t.camera,
gain=self.t.gain,exptime=self.t.exptime)
# ids make it onto a ccd (geometry cut)
tim.ids_added=[]
# Grab the data and inverse variance images [nanomaggies!]
tim_image = galsim.Image(tim.getImage())
tim_invvar = galsim.Image(tim.getInvvar())
tim_dq = galsim.Image(tim.dq)
# Also store galaxy sims and sims invvar
sims_image = tim_image.copy()
sims_image.fill(0.0)
sims_ivar = sims_image.copy()
# Store simulated galaxy images in tim object
# Loop on each object.
for ii, obj in enumerate(self.survey.simcat):
# Print timing
t0= Time()
if objtype in ['lrg','elg']:
strin= 'Drawing 1 %s: n=%.2f, rhalf=%.2f, e1=%.2f, e2=%.2f' % \
(objtype.upper(), obj.n,obj.rhalf,obj.e1,obj.e2)
print(strin)
if objtype == 'star':
stamp = objstamp.star(obj)
elif objtype == 'elg':
stamp = objstamp.elg(obj)
elif objtype == 'lrg':
stamp = objstamp.lrg(obj)
elif objtype == 'qso':
stamp = objstamp.qso(obj)
t0= ptime('Finished Drawing %s: id=%d band=%s dbflux=%f addedflux=%f' %
(objtype.upper(), obj.id,objstamp.band,
obj.get(objstamp.band+'flux'),stamp.array.sum()), t0)
stamp_nonoise= stamp.copy()
if self.survey.add_sim_noise:
stamp += noise_for_galaxy(stamp,objstamp.nano2e)
ivarstamp= ivar_for_galaxy(stamp,objstamp.nano2e)
# Add source if EVEN 1 pix falls on the CCD
overlap = stamp.bounds & tim_image.bounds
if overlap.area() > 0:
print('Stamp overlaps tim: id=%d band=%s' % (obj.id,objstamp.band))
tim.ids_added.append(obj.id)
stamp = stamp[overlap]
ivarstamp = ivarstamp[overlap]
stamp_nonoise= stamp_nonoise[overlap]
# Zero out invvar where bad pixel mask is flagged (> 0)
keep = np.ones(tim_dq[overlap].array.shape)
keep[ tim_dq[overlap].array > 0 ] = 0.
ivarstamp *= keep
# Add stamp to image
back= tim_image[overlap].copy()
tim_image[overlap] += stamp #= back.copy() + stamp.copy()
# Add variances
back_ivar= tim_invvar[overlap].copy()
tot_ivar= get_srcimg_invvar(ivarstamp, back_ivar)
tim_invvar[overlap] = tot_ivar.copy()
#Extra
sims_image[overlap] += stamp.copy()
sims_ivar[overlap] += ivarstamp.copy()
if np.min(sims_ivar.array) < 0:
log.warning('Negative invvar!')
import pdb ; pdb.set_trace()
tim.sims_image = sims_image.array
tim.sims_inverr = np.sqrt(sims_ivar.array)
# Can set image=model, ivar=1/model for testing
if self.survey.image_eq_model:
tim.data = sims_image.array.copy()
tim.inverr = np.zeros(tim.data.shape)
tim.inverr[sims_image.array > 0.] = np.sqrt(1./sims_image.array.copy()[sims_image.array > 0.])
else:
tim.data = tim_image.array
tim.inverr = np.sqrt(tim_invvar.array)
sys.stdout.flush()
#print('get_tractor_image in obiwan :time '+str(time_builtin.clock()-t1))
return tim
def _ceres_forced_photom(self,
tractor,
result,
umodels,
imlist,
mods0,
scales,
skyderivs,
minFlux,
nonneg=False,
wantims0=True,
wantims1=True,
negfluxval=None,
verbose=False,
**kwargs):
'''
negfluxval: when 'nonneg' is set, the flux value to give sources that went
negative in an unconstrained fit.
'''
from tractor.ceres import ceres_forced_phot
t0 = Time()
blocks = []
blockstart = {}
usedParamMap = {}
nextparam = 0
# umodels[ imagei, srci ] = Patch
Nsky = 0
Z = []
if skyderivs is not None:
# skyderivs = [ (param0:)[ (deriv,img), ], (param1:)[ (deriv,img), ], ...]
# Reorg them to be in img-major order
skymods = [[] for im in imlist]
for skyd in skyderivs:
for (deriv, img) in skyd:
imi = imlist.index(img)
skymods[imi].append(deriv)
for mods, im, mod0 in zip(skymods, imlist, mods0):
Z.append((mods, im, 1., mod0, Nsky))
Nsky += len(mods)
Z.extend(
zip(umodels, imlist, scales, mods0,
np.zeros(len(imlist), int) + Nsky))
sky = (skyderivs is not None)
for zi, (umods, img, scale, mod0, paramoffset) in enumerate(Z):
H, W = img.shape
if img in blockstart:
(b0, nbw, nbh) = blockstart[img]
else:
# Dice up the image
nbw = int(np.ceil(W / float(self.BW)))
nbh = int(np.ceil(H / float(self.BH)))
b0 = len(blocks)
blockstart[img] = (b0, nbw, nbh)
for iy in range(nbh):
for ix in range(nbw):
x0 = ix * self.BW
y0 = iy * self.BH
slc = (slice(y0,
min(y0 + self.BH,
H)), slice(x0, min(x0 + self.BW, W)))
data = (x0, y0,
img.getImage()[slc].astype(self.ceresType),
mod0[slc].astype(self.ceresType),
img.getInvError()[slc].astype(self.ceresType))
blocks.append((data, []))
for modi, umod in enumerate(umods):
if umod is None:
continue
# DEBUG
if len(umod.shape) != 2:
print('zi', zi)
print('modi', modi)
print('umod', umod)
umod.clipTo(W, H)
umod.trimToNonZero()
if umod.patch is None:
continue
# Dice up the model
ph, pw = umod.shape
bx0 = np.clip(int(np.floor(umod.x0 / float(self.BW))), 0,
nbw - 1)
bx1 = np.clip(int(np.ceil((umod.x0 + pw) / float(self.BW))), 0,
nbw - 1)
by0 = np.clip(int(np.floor(umod.y0 / float(self.BH))), 0,
nbh - 1)
by1 = np.clip(int(np.ceil((umod.y0 + ph) / float(self.BH))), 0,
nbh - 1)
parami = paramoffset + modi
if parami in usedParamMap:
ceresparam = usedParamMap[parami]
else:
usedParamMap[parami] = nextparam
ceresparam = nextparam
nextparam += 1
cmod = (umod.patch * scale).astype(self.ceresType)
for by in range(by0, by1 + 1):
for bx in range(bx0, bx1 + 1):
bi = by * nbw + bx
# if type(umod.x0) != int or type(umod.y0) != int:
# print('umod:', umod.x0, umod.y0, type(umod.x0), type(umod.y0))
# print('umod:', umod)
dd = (ceresparam, int(umod.x0), int(umod.y0), cmod)
blocks[b0 + bi][1].append(dd)
logverb('forced phot: dicing up', Time() - t0)
if wantims0:
t0 = Time()
params = tractor.getParams()
result.ims0 = self._getims(params, imlist, umodels, mods0, scales,
sky, minFlux, None)
logverb('forced phot: ims0', Time() - t0)
t0 = Time()
fluxes = np.zeros(len(usedParamMap))
logverb('Ceres forced phot:')
logverb(len(blocks), ('image blocks (%ix%i), %i params' %
(self.BW, self.BH, len(fluxes))))
if len(blocks) == 0 or len(fluxes) == 0:
logverb('Nothing to do!')
return
# init fluxes passed to ceres
p0 = tractor.getParams()
for i, k in usedParamMap.items():
fluxes[k] = p0[i]
iverbose = 1 if verbose else 0
nonneg = int(nonneg)
ithreads = 0
if self.threads is not None:
ithreads = int(self.threads)
if nonneg:
# Initial run with nonneg=False, to get in the ballpark
x = ceres_forced_phot(blocks, fluxes, 0, iverbose, ithreads)
assert (x == 0)
logverb('forced phot: ceres initial run', Time() - t0)
t0 = Time()
if negfluxval is not None:
fluxes = np.maximum(fluxes, negfluxval)
x = ceres_forced_phot(blocks, fluxes, nonneg, iverbose, ithreads)
#print('Ceres forced phot:', x)
logverb('forced phot: ceres', Time() - t0)
t0 = Time()
params = np.zeros(len(p0))
for i, k in usedParamMap.items():
params[i] = fluxes[k]
tractor.setParams(params)
logverb('forced phot: unmapping params:', Time() - t0)
if wantims1:
t0 = Time()
result.ims1 = self._getims(params, imlist, umodels, mods0, scales,
sky, minFlux, None)
logverb('forced phot: ims1:', Time() - t0)
return x
def main(survey=None, opt=None):
print(' '.join(sys.argv))
'''Driver function for forced photometry of individual Legacy
Survey images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = tlast = Time()
if opt.skip and os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if opt.derivs and opt.agn:
print('Sorry, can\'t do --derivs AND --agn')
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing first arg as exposure number (otherwise, it's a filename)
try:
expnum = int(opt.expnum)
filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
filename = opt.expnum
# Try parsing HDU: "all" or HDU name or HDU number.
all_hdus = (opt.ccdname == 'all')
hdu = -1
ccdname = None
if not all_hdus:
try:
hdu = int(opt.ccdname)
except:
ccdname = opt.ccdname
if survey is None:
survey = LegacySurveyData(survey_dir=opt.survey_dir)
catsurvey_north = survey
catsurvey_south = None
if opt.catalog_dir_north is not None:
assert (opt.catalog_dir_south is not None)
assert (opt.catalog_resolve_dec_ngc is not None)
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir_north)
catsurvey_south = LegacySurveyData(survey_dir=opt.catalog_dir_south)
if opt.catalog_dir is not None:
catsurvey_north = LegacySurveyData(survey_dir=opt.catalog_dir)
if filename is not None and hdu >= 0:
# FIXME -- try looking up in CCDs file?
# Read metadata from file
print('Warning: faking metadata from file contents')
T = exposure_metadata([filename], hdus=[hdu])
print('Metadata:')
T.about()
if not 'ccdzpt' in T.columns():
phdr = fitsio.read_header(filename)
T.ccdzpt = np.array([phdr['MAGZERO']])
print('WARNING: using header MAGZERO')
T.ccdraoff = np.array([0.])
T.ccddecoff = np.array([0.])
print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.')
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and ccdname', ccdname)
if hdu >= 0:
T.cut(T.image_hdu == hdu)
print(len(T), 'with HDU', hdu)
if filename is not None:
T.cut(np.array([f.strip() == filename for f in T.image_filename]))
print(len(T), 'with filename', filename)
if opt.camera is not None:
T.cut(T.camera == opt.camera)
print(len(T), 'with camera', opt.camera)
if not all_hdus:
assert (len(T) == 1)
args = []
for ccd in T:
args.append((survey, catsurvey_north, catsurvey_south,
opt.catalog_resolve_dec_ngc, ccd, opt, zoomslice, ps))
if opt.threads:
from astrometry.util.multiproc import multiproc
from astrometry.util.timingpool import TimingPool, TimingPoolMeas
pool = TimingPool(opt.threads)
poolmeas = TimingPoolMeas(pool, pickleTraffic=False)
Time.add_measurement(poolmeas)
mp = multiproc(None, pool=pool)
tm = Time()
FF = mp.map(bounce_one_ccd, args)
print('Multi-processing forced-phot:', Time() - tm)
else:
FF = map(bounce_one_ccd, args)
FF = [F for F in FF if F is not None]
if len(FF) == 0:
print('No photometry results to write.')
return 0
# Keep only the first header
_, version_hdr = FF[0]
FF = [F for F, hdr in FF]
F = merge_tables(FF)
if all_hdus:
version_hdr.delete('CPHDU')
version_hdr.delete('CCDNAME')
units = {
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2',
'apflux': 'nanomaggy',
'apflux_ivar': '1/nanomaggy^2',
'psfdepth': '1/nanomaggy^2',
'galdepth': '1/nanomaggy^2',
'sky': 'nanomaggy/arcsec^2',
'psfsize': 'arcsec'
}
if opt.derivs:
units.update({
'dra': 'arcsec',
'ddec': 'arcsec',
'dra_ivar': '1/arcsec^2',
'ddec_ivar': '1/arcsec^2'
})
columns = F.get_columns()
order = [
'release', 'brickid', 'brickname', 'objid', 'camera', 'expnum',
'ccdname', 'filter', 'mjd', 'exptime', 'psfsize', 'ccd_cuts',
'airmass', 'sky', 'psfdepth', 'galdepth', 'ra', 'dec', 'flux',
'flux_ivar', 'fracflux', 'rchisq', 'fracmasked', 'apflux',
'apflux_ivar', 'x', 'y', 'dqmask', 'dra', 'ddec', 'dra_ivar',
'ddec_ivar'
]
columns = [c for c in order if c in columns]
# Set units headers (must happen after column ordering is set!)
hdr = fitsio.FITSHDR()
for i, col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
tmpfn = os.path.join(outdir, 'tmp-' + os.path.basename(opt.outfn))
fitsio.write(tmpfn, None, header=version_hdr, clobber=True)
F.writeto(tmpfn, header=hdr, append=True, columns=columns)
os.rename(tmpfn, opt.outfn)
print('Wrote', opt.outfn)
tnow = Time()
print('Total:', tnow - t0)
return 0
def main():
import optparse
from astrometry.util.stages import CallGlobal, runstage
parser = optparse.OptionParser()
parser.add_option('-f', '--force-stage', dest='force', action='append', default=[],
help="Force re-running the given stage(s) -- don't read from pickle.")
parser.add_option('-s', '--stage', dest='stage', default=[], action='append',
help="Run up to the given stage(s)")
parser.add_option('-n', '--no-write', dest='write', default=True, action='store_false')
parser.add_option('-P', '--pickle', dest='picklepat', help='Pickle filename pattern, with %i, default %default',
default='pickles/tunebrick-%(brick)06i-%%(stage)s.pickle')
parser.add_option('-b', '--brick', type=int, help='Brick ID to run: default %default',
default=377306)
parser.add_option('-p', '--plots', dest='plots', action='store_true')
#parser.add_option('--stamp', action='store_true')
parser.add_option('--zoom', type=int, nargs=4, help='Set target image extent (default "0 3600 0 3600")')
parser.add_option('-W', type=int, default=3600, help='Target image width (default %default)')
parser.add_option('-H', type=int, default=3600, help='Target image height (default %default)')
parser.add_option('--bands', help='Bands to process; default "%default"', default='grz')
parser.add_option('--plot-base', default='plot-%(brick)06i', #'tunebrick/coadd/plot-%(brick)06i',
help='Plot filenames; default %default')
parser.add_option('--threads', type=int, help='Run multi-threaded')
parser.add_option('--base-dir', dest='basedir', default='tunebrick',
help='Base output directory; default %default')
parser.add_option('--mock-psf', dest='mock_psf', action='store_true',
help='Use fake PSF?')
opt,args = parser.parse_args()
Time.add_measurement(MemMeas)
stagefunc = CallGlobal('stage_%s', globals())
if len(opt.stage) == 0:
opt.stage.append('writecat2')
opt.force.extend(opt.stage)
opt.picklepat = opt.picklepat % dict(brick=opt.brick)
prereqs = {'tims': None,
'cat': 'tims',
'tune': 'cat',
'writecat2': 'tune',
'recoadd': 'tims',
'rergb': 'recoadd',
'primage': 'recoadd',
}
ps = PlotSequence(opt.plot_base % dict(brick=opt.brick))
initargs = dict(ps=ps)
initargs.update(W=opt.W, H=opt.H, brickid=opt.brick, target_extent=opt.zoom,
program_name = 'tunebrick.py', pipe=True,
bands=opt.bands,
mock_psf=opt.mock_psf)
kwargs = {}
kwargs.update(basedir=opt.basedir)
if opt.threads and opt.threads > 1:
from astrometry.util.multiproc import multiproc
mp = multiproc(opt.threads, init=runbrick_global_init, initargs=())
runbrick.mp = mp
else:
runbrick_global_init()
t0 = Time()
for stage in opt.stage:
runstage(stage, opt.picklepat, stagefunc, force=opt.force, write=opt.write,
prereqs=prereqs, initial_args=initargs, **kwargs)
#tune(opt.brick, target_extent=opt.zoom)
print 'Total:', Time()-t0
def run_forced_phot(cat,
tim,
ceres=True,
derivs=False,
agn=False,
do_forced=True,
do_apphot=True,
get_model=False,
ps=None,
timing=False,
fixed_also=False,
ceres_threads=1):
'''
fixed_also: if derivs=True, also run without derivatives and report
that flux too?
'''
if timing:
tlast = Time()
if ps is not None:
import pylab as plt
opti = None
forced_kwargs = {}
if ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
try:
opti = CeresOptimizer(BW=B, BH=B, threads=ceres_threads)
except:
if ceres_threads > 1:
raise RuntimeError(
'ceres_threads requested but not supported by tractor.ceres version'
)
opti = CeresOptimizer(BW=B, BH=B)
#forced_kwargs.update(verbose=True)
# nsize = 0
for src in cat:
# Limit sizes of huge models
# from tractor.galaxy import ProfileGalaxy
# if isinstance(src, ProfileGalaxy):
# px,py = tim.wcs.positionToPixel(src.getPosition())
# h = src._getUnitFluxPatchSize(tim, px, py, tim.modelMinval)
# MAXHALF = 128
# if h > MAXHALF:
# #print('halfsize', h,'for',src,'-> setting to',MAXHALF)
# nsize += 1
# src.halfsize = MAXHALF
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
#print('Limited the size of', nsize, 'large galaxy models')
if derivs:
realsrcs = []
derivsrcs = []
Iderivs = []
for i, src in enumerate(cat):
from tractor import PointSource
realsrcs.append(src)
if not isinstance(src, PointSource):
continue
Iderivs.append(i)
brightness_dra = src.getBrightness().copy()
brightness_ddec = src.getBrightness().copy()
brightness_dra.setParams(np.zeros(brightness_dra.numberOfParams()))
brightness_ddec.setParams(
np.zeros(brightness_ddec.numberOfParams()))
brightness_dra.freezeAllBut(tim.band)
brightness_ddec.freezeAllBut(tim.band)
dsrc = SourceDerivatives(src, [brightness_dra, brightness_ddec],
tim, ps)
derivsrcs.append(dsrc)
Iderivs = np.array(Iderivs)
if fixed_also:
pass
else:
# For convenience, put all the real sources at the front of
# the list, so we can pull the IVs off the front of the list.
cat = realsrcs + derivsrcs
if agn:
from tractor.galaxy import ExpGalaxy, DevGalaxy, FixedCompositeGalaxy
from tractor import PointSource
from legacypipe.survey import SimpleGalaxy, RexGalaxy
realsrcs = []
agnsrcs = []
iagn = []
for i, src in enumerate(cat):
realsrcs.append(src)
## ??
if isinstance(src, (SimpleGalaxy, RexGalaxy)):
#print('Skipping SIMP or REX:', src)
continue
if isinstance(src, (ExpGalaxy, DevGalaxy, FixedCompositeGalaxy)):
iagn.append(i)
bright = src.getBrightness().copy()
bright.setParams(np.zeros(bright.numberOfParams()))
bright.freezeAllBut(tim.band)
agn = PointSource(src.pos, bright)
agn.freezeAllBut('brightness')
#print('Adding "agn"', agn, 'to', src)
#print('agn params:', agn.getParamNames())
agnsrcs.append(src)
iagn = np.array(iagn)
cat = realsrcs + agnsrcs
print('Added AGN to', len(iagn), 'galaxies')
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
disable_galaxy_cache()
F = fits_table()
if do_forced:
if timing and (derivs or agn):
t = Time()
print('Setting up:', t - tlast)
tlast = t
if derivs:
if fixed_also:
print('Forced photom with fixed positions:')
R = tr.optimize_forced_photometry(variance=True,
fitstats=False,
shared_params=False,
priors=False,
**forced_kwargs)
F.flux_fixed = np.array([
src.getBrightness().getFlux(tim.band) for src in cat
]).astype(np.float32)
N = len(cat)
F.flux_fixed_ivar = R.IV[:N].astype(np.float32)
if timing:
t = Time()
print('Forced photom with fixed positions finished:',
t - tlast)
tlast = t
cat = realsrcs + derivsrcs
tr.setCatalog(Catalog(*cat))
print('Forced photom with position derivatives:')
if ps is None and not get_model:
forced_kwargs.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True,
fitstats=True,
shared_params=False,
priors=False,
**forced_kwargs)
if ps is not None or get_model:
(data, mod, ie, chi, roi) = R.ims1[0]
if ps is not None:
ima = dict(vmin=-2. * tim.sig1,
vmax=5. * tim.sig1,
interpolation='nearest',
origin='lower',
cmap='gray')
imchi = dict(interpolation='nearest',
origin='lower',
vmin=-5,
vmax=5,
cmap='RdBu')
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
if derivs:
trx = Tractor([tim], realsrcs)
trx.freezeParam('images')
modx = trx.getModelImage(0)
chix = (data - modx) * tim.getInvError()
plt.clf()
plt.imshow(modx, **ima)
plt.title('Model without derivatives: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chix, **imchi)
plt.title('Chi without derivatives: %s' % tim.name)
ps.savefig()
if derivs or agn:
cat = realsrcs
N = len(cat)
F.flux = np.array([
src.getBrightness().getFlux(tim.band) for src in cat
]).astype(np.float32)
F.flux_ivar = R.IV[:N].astype(np.float32)
F.fracflux = R.fitstats.profracflux[:N].astype(np.float32)
F.rchisq = R.fitstats.prochi2[:N].astype(np.float32)
try:
F.fracmasked = R.fitstats.promasked[:N].astype(np.float32)
except:
print(
'No "fracmasked" available (only in recent Tractor versions)')
if derivs:
F.flux_dra = np.zeros(len(F), np.float32)
F.flux_ddec = np.zeros(len(F), np.float32)
F.flux_dra[Iderivs] = np.array(
[src.getParams()[0] for src in derivsrcs]).astype(np.float32)
F.flux_ddec[Iderivs] = np.array(
[src.getParams()[1] for src in derivsrcs]).astype(np.float32)
F.flux_dra_ivar = np.zeros(len(F), np.float32)
F.flux_ddec_ivar = np.zeros(len(F), np.float32)
F.flux_dra_ivar[Iderivs] = R.IV[N::2].astype(np.float32)
F.flux_ddec_ivar[Iderivs] = R.IV[N + 1::2].astype(np.float32)
if agn:
F.flux_agn = np.zeros(len(F), np.float32)
F.flux_agn_ivar = np.zeros(len(F), np.float32)
F.flux_agn[iagn] = np.array(
[src.getParams()[0] for src in agnsrcs])
F.flux_agn_ivar[iagn] = R.IV[N:].astype(np.float32)
if timing:
t = Time()
print('Forced photom:', t - tlast)
tlast = t
if do_apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack(
[tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
#print('Apertures:', apertures, 'pixels')
#print('apxy shape', apxy.shape) # --> (2,N)
# The aperture photometry routine doesn't like pixel positions outside the image
H, W = img.shape
Iap = np.flatnonzero((apxy[0, :] >= 0) * (apxy[1, :] >= 0) *
(apxy[0, :] <= W - 1) * (apxy[1, :] <= H - 1))
print('Aperture photometry for', len(Iap), 'of', len(apxy[0, :]),
'sources within image bounds')
for rad in apertures:
aper = photutils.CircularAperture(apxy[:, Iap], rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = np.zeros((len(F), len(apertures)), np.float32)
F.apflux[Iap, :] = ap.astype(np.float32)
apimgerr = np.vstack(apimgerr).T
apiv = np.zeros(apimgerr.shape, np.float32)
apiv[apimgerr != 0] = 1. / apimgerr[apimgerr != 0]**2
F.apflux_ivar = np.zeros((len(F), len(apertures)), np.float32)
F.apflux_ivar[Iap, :] = apiv
if timing:
print('Aperture photom:', Time() - tlast)
if get_model:
return F, mod
return F
def _imap_finished(self, tstart):
tend = Time()
self.parallel.append((tstart, tend))
self.t0 = tend