def run_forced_phot(cat,
tim,
ceres=True,
derivs=False,
agn=False,
do_forced=True,
do_apphot=True,
get_model=False,
ps=None,
timing=False,
fixed_also=False,
ceres_threads=1):
'''
fixed_also: if derivs=True, also run without derivatives and report
that flux too?
'''
if timing:
tlast = Time()
if ps is not None:
import pylab as plt
opti = None
forced_kwargs = {}
if ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
try:
opti = CeresOptimizer(BW=B, BH=B, threads=ceres_threads)
except:
if ceres_threads > 1:
raise RuntimeError(
'ceres_threads requested but not supported by tractor.ceres version'
)
opti = CeresOptimizer(BW=B, BH=B)
#forced_kwargs.update(verbose=True)
# nsize = 0
for src in cat:
# Limit sizes of huge models
# from tractor.galaxy import ProfileGalaxy
# if isinstance(src, ProfileGalaxy):
# px,py = tim.wcs.positionToPixel(src.getPosition())
# h = src._getUnitFluxPatchSize(tim, px, py, tim.modelMinval)
# MAXHALF = 128
# if h > MAXHALF:
# #print('halfsize', h,'for',src,'-> setting to',MAXHALF)
# nsize += 1
# src.halfsize = MAXHALF
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
#print('Limited the size of', nsize, 'large galaxy models')
if derivs:
realsrcs = []
derivsrcs = []
Iderivs = []
for i, src in enumerate(cat):
from tractor import PointSource
realsrcs.append(src)
if not isinstance(src, PointSource):
continue
Iderivs.append(i)
brightness_dra = src.getBrightness().copy()
brightness_ddec = src.getBrightness().copy()
brightness_dra.setParams(np.zeros(brightness_dra.numberOfParams()))
brightness_ddec.setParams(
np.zeros(brightness_ddec.numberOfParams()))
brightness_dra.freezeAllBut(tim.band)
brightness_ddec.freezeAllBut(tim.band)
dsrc = SourceDerivatives(src, [brightness_dra, brightness_ddec],
tim, ps)
derivsrcs.append(dsrc)
Iderivs = np.array(Iderivs)
if fixed_also:
pass
else:
# For convenience, put all the real sources at the front of
# the list, so we can pull the IVs off the front of the list.
cat = realsrcs + derivsrcs
if agn:
from tractor.galaxy import ExpGalaxy, DevGalaxy, FixedCompositeGalaxy
from tractor import PointSource
from legacypipe.survey import SimpleGalaxy, RexGalaxy
realsrcs = []
agnsrcs = []
iagn = []
for i, src in enumerate(cat):
realsrcs.append(src)
## ??
if isinstance(src, (SimpleGalaxy, RexGalaxy)):
#print('Skipping SIMP or REX:', src)
continue
if isinstance(src, (ExpGalaxy, DevGalaxy, FixedCompositeGalaxy)):
iagn.append(i)
bright = src.getBrightness().copy()
bright.setParams(np.zeros(bright.numberOfParams()))
bright.freezeAllBut(tim.band)
agn = PointSource(src.pos, bright)
agn.freezeAllBut('brightness')
#print('Adding "agn"', agn, 'to', src)
#print('agn params:', agn.getParamNames())
agnsrcs.append(src)
iagn = np.array(iagn)
cat = realsrcs + agnsrcs
print('Added AGN to', len(iagn), 'galaxies')
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
disable_galaxy_cache()
F = fits_table()
if do_forced:
if timing and (derivs or agn):
t = Time()
print('Setting up:', t - tlast)
tlast = t
if derivs:
if fixed_also:
print('Forced photom with fixed positions:')
R = tr.optimize_forced_photometry(variance=True,
fitstats=False,
shared_params=False,
priors=False,
**forced_kwargs)
F.flux_fixed = np.array([
src.getBrightness().getFlux(tim.band) for src in cat
]).astype(np.float32)
N = len(cat)
F.flux_fixed_ivar = R.IV[:N].astype(np.float32)
if timing:
t = Time()
print('Forced photom with fixed positions finished:',
t - tlast)
tlast = t
cat = realsrcs + derivsrcs
tr.setCatalog(Catalog(*cat))
print('Forced photom with position derivatives:')
if ps is None and not get_model:
forced_kwargs.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True,
fitstats=True,
shared_params=False,
priors=False,
**forced_kwargs)
if ps is not None or get_model:
(data, mod, ie, chi, roi) = R.ims1[0]
if ps is not None:
ima = dict(vmin=-2. * tim.sig1,
vmax=5. * tim.sig1,
interpolation='nearest',
origin='lower',
cmap='gray')
imchi = dict(interpolation='nearest',
origin='lower',
vmin=-5,
vmax=5,
cmap='RdBu')
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
if derivs:
trx = Tractor([tim], realsrcs)
trx.freezeParam('images')
modx = trx.getModelImage(0)
chix = (data - modx) * tim.getInvError()
plt.clf()
plt.imshow(modx, **ima)
plt.title('Model without derivatives: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chix, **imchi)
plt.title('Chi without derivatives: %s' % tim.name)
ps.savefig()
if derivs or agn:
cat = realsrcs
N = len(cat)
F.flux = np.array([
src.getBrightness().getFlux(tim.band) for src in cat
]).astype(np.float32)
F.flux_ivar = R.IV[:N].astype(np.float32)
F.fracflux = R.fitstats.profracflux[:N].astype(np.float32)
F.rchisq = R.fitstats.prochi2[:N].astype(np.float32)
try:
F.fracmasked = R.fitstats.promasked[:N].astype(np.float32)
except:
print(
'No "fracmasked" available (only in recent Tractor versions)')
if derivs:
F.flux_dra = np.zeros(len(F), np.float32)
F.flux_ddec = np.zeros(len(F), np.float32)
F.flux_dra[Iderivs] = np.array(
[src.getParams()[0] for src in derivsrcs]).astype(np.float32)
F.flux_ddec[Iderivs] = np.array(
[src.getParams()[1] for src in derivsrcs]).astype(np.float32)
F.flux_dra_ivar = np.zeros(len(F), np.float32)
F.flux_ddec_ivar = np.zeros(len(F), np.float32)
F.flux_dra_ivar[Iderivs] = R.IV[N::2].astype(np.float32)
F.flux_ddec_ivar[Iderivs] = R.IV[N + 1::2].astype(np.float32)
if agn:
F.flux_agn = np.zeros(len(F), np.float32)
F.flux_agn_ivar = np.zeros(len(F), np.float32)
F.flux_agn[iagn] = np.array(
[src.getParams()[0] for src in agnsrcs])
F.flux_agn_ivar[iagn] = R.IV[N:].astype(np.float32)
if timing:
t = Time()
print('Forced photom:', t - tlast)
tlast = t
if do_apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack(
[tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
#print('Apertures:', apertures, 'pixels')
#print('apxy shape', apxy.shape) # --> (2,N)
# The aperture photometry routine doesn't like pixel positions outside the image
H, W = img.shape
Iap = np.flatnonzero((apxy[0, :] >= 0) * (apxy[1, :] >= 0) *
(apxy[0, :] <= W - 1) * (apxy[1, :] <= H - 1))
print('Aperture photometry for', len(Iap), 'of', len(apxy[0, :]),
'sources within image bounds')
for rad in apertures:
aper = photutils.CircularAperture(apxy[:, Iap], rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = np.zeros((len(F), len(apertures)), np.float32)
F.apflux[Iap, :] = ap.astype(np.float32)
apimgerr = np.vstack(apimgerr).T
apiv = np.zeros(apimgerr.shape, np.float32)
apiv[apimgerr != 0] = 1. / apimgerr[apimgerr != 0]**2
F.apflux_ivar = np.zeros((len(F), len(apertures)), np.float32)
F.apflux_ivar[Iap, :] = apiv
if timing:
print('Aperture photom:', Time() - tlast)
if get_model:
return F, mod
return F
def main():
import optparse
from astrometry.util.plotutils import PlotSequence
from astrometry.util.util import Tan
parser = optparse.OptionParser(usage='%prog [options] incat.fits out.fits')
parser.add_option('-r', '--ralo', dest='ralo', type=float,
help='Minimum RA')
parser.add_option('-R', '--rahi', dest='rahi', type=float,
help='Maximum RA')
parser.add_option('-d', '--declo', dest='declo', type=float,
help='Minimum Dec')
parser.add_option('-D', '--dechi', dest='dechi', type=float,
help='Maximum Dec')
parser.add_option('-b', '--band', dest='bands', action='append', type=int,
default=[], help='WISE band to photometer (default: 1,2)')
parser.add_option('-u', '--unwise', dest='unwise_dir',
default='unwise-coadds',
help='Directory containing unWISE coadds')
parser.add_option('--no-ceres', dest='ceres', action='store_false',
default=True,
help='Use scipy lsqr rather than Ceres Solver?')
parser.add_option('--ceres-block', '-B', dest='ceresblock', type=int,
default=8,
help='Ceres image block size (default: %default)')
parser.add_option('--plots', dest='plots',
default=False, action='store_true')
parser.add_option('--save-fits', dest='save_fits',
default=False, action='store_true')
# parser.add_option('--ellipses', action='store_true',
# help='Assume catalog shapes are ellipse descriptions (not r,ab,phi)')
# parser.add_option('--ra', help='Center RA')
# parser.add_option('--dec', help='Center Dec')
# parser.add_option('--width', help='Degrees width (in RA*cos(Dec))')
# parser.add_option('--height', help='Degrees height (Dec)')
opt, args = parser.parse_args()
if len(args) != 2:
parser.print_help()
sys.exit(-1)
if len(opt.bands) == 0:
opt.bands = [1, 2]
# Allow specifying bands like "123"
bb = []
for band in opt.bands:
for s in str(band):
bb.append(int(s))
opt.bands = bb
print('Bands', opt.bands)
ps = None
if opt.plots:
ps = PlotSequence('unwise')
infn, outfn = args
T = fits_table(infn)
print('Read', len(T), 'sources from', infn)
if opt.declo is not None:
T.cut(T.dec >= opt.declo)
if opt.dechi is not None:
T.cut(T.dec <= opt.dechi)
# Let's be a bit smart about RA wrap-around. Compute the 'center'
# of the RA points, use the cross product against that to define
# inequality (clockwise-of).
r = np.deg2rad(T.ra)
x = np.mean(np.cos(r))
y = np.mean(np.sin(r))
rr = np.hypot(x, y)
x /= rr
y /= rr
midra = np.rad2deg(np.arctan2(y, x))
midra += 360. * (midra < 0)
xx = np.cos(r)
yy = np.sin(r)
T.cross = x * yy - y * xx
minra = T.ra[np.argmin(T.cross)]
maxra = T.ra[np.argmax(T.cross)]
if opt.ralo is not None:
r = np.deg2rad(opt.ralo)
xx = np.cos(r)
yy = np.sin(r)
crosscut = x * yy - y * xx
T.cut(T.cross >= crosscut)
print('Cut to', len(T), 'with RA >', opt.ralo)
if opt.rahi is not None:
r = np.deg2rad(opt.rahi)
xx = np.cos(r)
yy = np.sin(r)
crosscut = x * yy - y * xx
T.cut(T.cross <= crosscut)
print('Cut to', len(T), 'with RA <', opt.rahi)
if opt.declo is None:
opt.declo = T.dec.min()
if opt.dechi is None:
opt.dechi = T.dec.max()
if opt.ralo is None:
opt.ralo = T.ra[np.argmin(T.cross)]
if opt.rahi is None:
opt.rahi = T.ra[np.argmax(T.cross)]
T.delete_column('cross')
print('RA range:', opt.ralo, opt.rahi)
print('Dec range:', opt.declo, opt.dechi)
x = np.mean([np.cos(np.deg2rad(r)) for r in (opt.ralo, opt.rahi)])
y = np.mean([np.sin(np.deg2rad(r)) for r in (opt.ralo, opt.rahi)])
midra = np.rad2deg(np.arctan2(y, x))
midra += 360. * (midra < 0)
middec = (opt.declo + opt.dechi) / 2.
print('RA,Dec center:', midra, middec)
pixscale = 2.75 / 3600.
H = (opt.dechi - opt.declo) / pixscale
dra = 2. * min(np.abs(midra - opt.ralo), np.abs(midra - opt.rahi))
W = dra * np.cos(np.deg2rad(middec)) / pixscale
margin = 5
W = int(W) + margin * 2
H = int(H) + margin * 2
print('W,H', W, H)
targetwcs = Tan(midra, middec, (W + 1) / 2., (H + 1) / 2.,
-pixscale, 0., 0., pixscale, float(W), float(H))
#print('Target WCS:', targetwcs)
ra0, dec0 = targetwcs.pixelxy2radec(0.5, 0.5)
ra1, dec1 = targetwcs.pixelxy2radec(W + 0.5, H + 0.5)
roiradecbox = [ra0, ra1, dec0, dec1]
#print('ROI RA,Dec box', roiradecbox)
tiles = unwise_tiles_touching_wcs(targetwcs)
print('Cut to', len(tiles), 'unWISE tiles')
disable_galaxy_cache()
cols = T.get_columns()
all_ptsrcs = not('type' in cols)
if not all_ptsrcs:
assert('shapeexp' in cols)
assert('shapedev' in cols)
assert('fracdev' in cols)
wanyband = 'w'
print('Creating Tractor catalog...')
cat = []
for i, t in enumerate(T):
pos = RaDecPos(t.ra, t.dec)
flux = NanoMaggies(**{wanyband: 1.})
if all_ptsrcs:
cat.append(PointSource(pos, flux))
continue
tt = t.type.strip()
if tt in ['PTSRC', 'STAR', 'S']:
cat.append(PointSource(pos, flux))
elif tt in ['EXP', 'E']:
shape = EllipseE(*t.shapeexp)
cat.append(ExpGalaxy(pos, flux, shape))
elif tt in ['DEV', 'D']:
shape = EllipseE(*t.shapedev)
cat.append(DevGalaxy(pos, flux, shape))
elif tt in ['COMP', 'C']:
eshape = EllipseE(*t.shapeexp)
dshape = EllipseE(*t.shapedev)
cat.append(FixedCompositeGalaxy(pos, flux, t.fracdev,
eshape, dshape))
else:
print('Did not understand row', i, 'of input catalog:')
t.about()
assert(False)
W = unwise_forcedphot(cat, tiles, roiradecbox=roiradecbox,
bands=opt.bands, unwise_dir=opt.unwise_dir,
use_ceres=opt.ceres, ceres_block=opt.ceresblock,
save_fits=opt.save_fits, ps=ps)
W.writeto(outfn)
def main():
import optparse
parser = optparse.OptionParser(usage='%prog [options] <NGC-number>')
parser.add_option('--threads', dest='threads', type=int, help='use multiprocessing')
parser.add_option('--itune1',dest='itune1',type=int,help='Individual tuning, first stage',default=5)
parser.add_option('--itune2',dest='itune2',type=int,help='Individual tuning, second stage',default=5)
parser.add_option('--ntune',dest='ntune',type=int,help='All objects tuning',default=0)
parser.add_option('--nocache',dest='nocache',action='store_true',default=False,help='Disable caching for memory reasons')
opt,args = parser.parse_args()
if len(args) != 1:
parser.print_help()
sys.exit(-1)
if opt.threads:
mp = multiproc(nthreads=opt.threads)
else:
mp = multiproc()
ngc = int(args[0])
j = getNGC(ngc)
print j
ra = float(j['RA'][0])
dec = float(j['DEC'][0])
itune1 = opt.itune1
itune2 = opt.itune2
ntune = opt.ntune
IRLS_scale = 25.
radius = (10.**j['LOG_D25'][0])/10.
dr8 = True
noarcsinh = False
print 'Radius', radius
print 'RA,Dec', ra, dec
sras, sdecs, smags = tychoMatch(ra,dec,(radius*1.5)/60.)
for sra,sdec,smag in zip(sras,sdecs,smags):
print sra,sdec,smag
rcfs = radec_to_sdss_rcf(ra,dec,radius=math.hypot(radius,13./2.),tablefn="dr8fields.fits")
print rcfs
#fieldPlot(ra,dec,radius,ngc)
imkw = dict(psf='dg')
if dr8:
getim = st.get_tractor_image_dr8
getsrc = st.get_tractor_sources_dr8
imkw.update(zrange=[-3,100])
else:
getim = st.get_tractor_image
getsrc = st.get_tractor_sources_dr8
imkw.update(useMags=True)
bands=['u','g','r','i','z']
#bands=['r']
bandname = 'r'
flipBands = ['r']
imsrcs = mp.map(get_ims_and_srcs, [(rcf + (bands, ra, dec, radius*60./0.396, imkw, getim, getsrc))
for rcf in rcfs])
timgs = []
sources = []
allsources = []
for ims,s in imsrcs:
if ims is None:
continue
if s is None:
continue
timgs.extend(ims)
allsources.extend(s)
sources.append(s)
#rds = [rcf[3:5] for rcf in rcfs]
plotarea(ra, dec, radius, ngc, timgs) #, rds)
lvl = logging.DEBUG
logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout)
tractor = st.Tractor(timgs, allsources, mp=mp)
sa = dict(debug=True, plotAll=False,plotBands=False)
if noarcsinh:
sa.update(nlscale=0)
elif dr8:
sa.update(chilo=-8.,chihi=8.)
if opt.nocache:
tractor.cache = NullCache()
sg.disable_galaxy_cache()
zr = timgs[0].zr
print "zr is: ",zr
print bands
print "Number of images: ", len(timgs)
# for timg,band in zip(timgs,bands):
# data = timg.getImage()/np.sqrt(timg.getInvvar())
# plt.hist(data,bins=100)
# plt.savefig('hist-%s.png' % (band))
prefix = 'ngc%d' % (ngc)
saveAll('initial-'+prefix, tractor,**sa)
#plotInvvar('initial-'+prefix,tractor)
for sra,sdec,smag in zip(sras,sdecs,smags):
print sra,sdec,smag
for img in tractor.getImages():
wcs = img.getWcs()
starx,stary = wcs.positionToPixel(RaDecPos(sra,sdec))
starr=25*(2**(max(11-smag,0.)))
if starx+starr<0. or starx-starr>img.getWidth() or stary+starr <0. or stary-starr>img.getHeight():
continue
X,Y = np.meshgrid(np.arange(img.getWidth()), np.arange(img.getHeight()))
R2 = (X - starx)**2 + (Y - stary)**2
img.getStarMask()[R2 < starr**2] = 0
#star = [(x,y) for x in range(img.getWidth()) for y in range(img.getHeight()) if (x-starx)**2+(y-stary)**2 <= starr**2]
#for (x,y) in star:
# img.getStarMask()[y][x] = 0
for timgs,sources in imsrcs:
timg = timgs[0]
wcs = timg.getWcs()
xtr,ytr = wcs.positionToPixel(RaDecPos(ra,dec))
print xtr,ytr
xt = xtr
yt = ytr
r = ((radius*60.))/.396 #radius in pixels
for src in sources:
xs,ys = wcs.positionToPixel(src.getPosition(),src)
if (xs-xt)**2+(ys-yt)**2 <= r**2:
print "Removed:", src
print xs,ys
tractor.removeSource(src)
#saveAll('removed-'+prefix, tractor,**sa)
newShape = sg.GalaxyShape(30.,1.,0.)
newBright = ba.Mags(r=15.0,g=15.0,u=15.0,z=15.0,i=15.0,order=['u','g','r','i','z'])
EG = st.ExpGalaxy(RaDecPos(ra,dec),newBright,newShape)
print EG
tractor.addSource(EG)
saveAll('added-'+prefix,tractor,**sa)
print 'Tractor has', tractor.getParamNames()
for im in tractor.images:
im.freezeAllParams()
im.thawParam('sky')
tractor.catalog.freezeAllBut(EG)
print 'Tractor has', tractor.getParamNames()
print 'values', tractor.getParams()
for i in range(itune1):
tractor.optimize()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.changeInvvar(IRLS_scale)
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
saveAll('itune1-%d-' % (i+1)+prefix,tractor,**sa)
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.clearCache()
sg.get_galaxy_cache().clear()
gc.collect()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
CGPos = EG.getPosition()
CGShape1 = EG.getShape().copy()
CGShape2 = EG.getShape().copy()
EGBright = EG.getBrightness()
CGu = EGBright[0] + 0.75
CGg = EGBright[1] + 0.75
CGr = EGBright[2] + 0.75
CGi = EGBright[3] + 0.75
CGz = EGBright[4] + 0.75
CGBright1 = ba.Mags(r=CGr,g=CGg,u=CGu,z=CGz,i=CGi,order=['u','g','r','i','z'])
CGBright2 = ba.Mags(r=CGr,g=CGg,u=CGu,z=CGz,i=CGi,order=['u','g','r','i','z'])
print EGBright
print CGBright1
CG = st.CompositeGalaxy(CGPos,CGBright1,CGShape1,CGBright2,CGShape2)
tractor.removeSource(EG)
tractor.addSource(CG)
tractor.catalog.freezeAllBut(CG)
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
for i in range(itune2):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('itune2-%d-' % (i+1)+prefix,tractor,**sa)
print resource.getpagesize()
print "RUsage is: ",resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.clearCache()
sg.get_galaxy_cache().clear()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.catalog.thawAllParams()
for i in range(ntune):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('ntune-%d-' % (i+1)+prefix,tractor,**sa)
#plotInvvar('final-'+prefix,tractor)
sa.update(plotBands=True)
saveAll('allBands-' + prefix,tractor,**sa)
print CG
print CG.getPosition()
print CGBright1
print CGBright2
print CGShape1
print CGShape2
print CGBright1+CGBright2
print CG.getBrightness()
pfn = 'ngc-%d.pickle' % ngc
pickle_to_file(CG,pfn)
makeflipbook(prefix,len(tractor.getImages()),itune1,itune2,ntune)
def main():
import optparse
parser = optparse.OptionParser(usage="%prog [options] <NGC-number>")
parser.add_option("--threads", dest="threads", type=int, help="use multiprocessing")
parser.add_option("--itune1", dest="itune1", type=int, help="Individual tuning, first stage", default=5)
parser.add_option("--itune2", dest="itune2", type=int, help="Individual tuning, second stage", default=5)
parser.add_option("--ntune", dest="ntune", type=int, help="All objects tuning", default=0)
parser.add_option(
"--nocache", dest="nocache", action="store_true", default=False, help="Disable caching for memory reasons"
)
opt, args = parser.parse_args()
if len(args) != 1:
parser.print_help()
sys.exit(-1)
if opt.threads:
mp = multiproc(nthreads=opt.threads)
else:
mp = multiproc()
ngc = int(args[0])
j = getNGC(ngc)
print j
ra = float(j["RA"][0])
dec = float(j["DEC"][0])
itune1 = opt.itune1
itune2 = opt.itune2
ntune = opt.ntune
IRLS_scale = 25.0
radius = (10.0 ** j["LOG_D25"][0]) / 10.0
dr8 = True
noarcsinh = False
print "Radius", radius
print "RA,Dec", ra, dec
sras, sdecs, smags = tychoMatch(ra, dec, (radius * 1.5) / 60.0)
for sra, sdec, smag in zip(sras, sdecs, smags):
print sra, sdec, smag
rcfs = radec_to_sdss_rcf(ra, dec, radius=math.hypot(radius, 13.0 / 2.0), tablefn="dr8fields.fits")
print rcfs
# fieldPlot(ra,dec,radius,ngc)
imkw = dict(psf="dg")
if dr8:
getim = st.get_tractor_image_dr8
getsrc = st.get_tractor_sources_dr8
imkw.update(zrange=[-3, 100])
else:
getim = st.get_tractor_image
getsrc = st.get_tractor_sources_dr8
imkw.update(useMags=True)
bands = ["u", "g", "r", "i", "z"]
# bands=['r']
bandname = "r"
flipBands = ["r"]
imsrcs = mp.map(
get_ims_and_srcs, [(rcf + (bands, ra, dec, radius * 60.0 / 0.396, imkw, getim, getsrc)) for rcf in rcfs]
)
timgs = []
sources = []
allsources = []
for ims, s in imsrcs:
if ims is None:
continue
if s is None:
continue
timgs.extend(ims)
allsources.extend(s)
sources.append(s)
# rds = [rcf[3:5] for rcf in rcfs]
plotarea(ra, dec, radius, ngc, timgs) # , rds)
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format="%(message)s", stream=sys.stdout)
tractor = st.Tractor(timgs, allsources, mp=mp)
sa = dict(debug=True, plotAll=False, plotBands=False)
if noarcsinh:
sa.update(nlscale=0)
elif dr8:
sa.update(chilo=-8.0, chihi=8.0)
if opt.nocache:
tractor.cache = NullCache()
sg.disable_galaxy_cache()
zr = timgs[0].zr
print "zr is: ", zr
print bands
print "Number of images: ", len(timgs)
# for timg,band in zip(timgs,bands):
# data = timg.getImage()/np.sqrt(timg.getInvvar())
# plt.hist(data,bins=100)
# plt.savefig('hist-%s.png' % (band))
prefix = "ngc%d" % (ngc)
saveAll("initial-" + prefix, tractor, **sa)
# plotInvvar('initial-'+prefix,tractor)
for sra, sdec, smag in zip(sras, sdecs, smags):
print sra, sdec, smag
for img in tractor.getImages():
wcs = img.getWcs()
starx, stary = wcs.positionToPixel(RaDecPos(sra, sdec))
starr = 25 * (2 ** (max(11 - smag, 0.0)))
if (
starx + starr < 0.0
or starx - starr > img.getWidth()
or stary + starr < 0.0
or stary - starr > img.getHeight()
):
continue
X, Y = np.meshgrid(np.arange(img.getWidth()), np.arange(img.getHeight()))
R2 = (X - starx) ** 2 + (Y - stary) ** 2
img.getStarMask()[R2 < starr ** 2] = 0
# star = [(x,y) for x in range(img.getWidth()) for y in range(img.getHeight()) if (x-starx)**2+(y-stary)**2 <= starr**2]
# for (x,y) in star:
# img.getStarMask()[y][x] = 0
for timgs, sources in imsrcs:
timg = timgs[0]
wcs = timg.getWcs()
xtr, ytr = wcs.positionToPixel(RaDecPos(ra, dec))
print xtr, ytr
xt = xtr
yt = ytr
r = ((radius * 60.0)) / 0.396 # radius in pixels
for src in sources:
xs, ys = wcs.positionToPixel(src.getPosition(), src)
if (xs - xt) ** 2 + (ys - yt) ** 2 <= r ** 2:
print "Removed:", src
print xs, ys
tractor.removeSource(src)
# saveAll('removed-'+prefix, tractor,**sa)
newShape = sg.GalaxyShape(30.0, 1.0, 0.0)
newBright = ba.Mags(r=15.0, g=15.0, u=15.0, z=15.0, i=15.0, order=["u", "g", "r", "i", "z"])
EG = st.ExpGalaxy(RaDecPos(ra, dec), newBright, newShape)
print EG
tractor.addSource(EG)
saveAll("added-" + prefix, tractor, **sa)
print "Tractor has", tractor.getParamNames()
for im in tractor.images:
im.freezeAllParams()
im.thawParam("sky")
tractor.catalog.freezeAllBut(EG)
print "Tractor has", tractor.getParamNames()
print "values", tractor.getParams()
for i in range(itune1):
tractor.optimize()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.changeInvvar(IRLS_scale)
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
saveAll("itune1-%d-" % (i + 1) + prefix, tractor, **sa)
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.clearCache()
sg.get_galaxy_cache().clear()
gc.collect()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
CGPos = EG.getPosition()
CGShape1 = EG.getShape().copy()
CGShape2 = EG.getShape().copy()
EGBright = EG.getBrightness()
CGu = EGBright[0] + 0.75
CGg = EGBright[1] + 0.75
CGr = EGBright[2] + 0.75
CGi = EGBright[3] + 0.75
CGz = EGBright[4] + 0.75
CGBright1 = ba.Mags(r=CGr, g=CGg, u=CGu, z=CGz, i=CGi, order=["u", "g", "r", "i", "z"])
CGBright2 = ba.Mags(r=CGr, g=CGg, u=CGu, z=CGz, i=CGi, order=["u", "g", "r", "i", "z"])
print EGBright
print CGBright1
CG = st.CompositeGalaxy(CGPos, CGBright1, CGShape1, CGBright2, CGShape2)
tractor.removeSource(EG)
tractor.addSource(CG)
tractor.catalog.freezeAllBut(CG)
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
for i in range(itune2):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll("itune2-%d-" % (i + 1) + prefix, tractor, **sa)
print resource.getpagesize()
print "RUsage is: ", resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.clearCache()
sg.get_galaxy_cache().clear()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.catalog.thawAllParams()
for i in range(ntune):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll("ntune-%d-" % (i + 1) + prefix, tractor, **sa)
# plotInvvar('final-'+prefix,tractor)
sa.update(plotBands=True)
saveAll("allBands-" + prefix, tractor, **sa)
print CG
print CG.getPosition()
print CGBright1
print CGBright2
print CGShape1
print CGShape2
print CGBright1 + CGBright2
print CG.getBrightness()
pfn = "ngc-%d.pickle" % ngc
pickle_to_file(CG, pfn)
makeflipbook(prefix, len(tractor.getImages()), itune1, itune2, ntune)
def twogalaxies(name1, ra1, dec1, name2, ra2, dec2):
name = "%s %s" % (name1, name2)
ra = float(ra1)
dec = float(dec1)
ra2 = float(ra2)
dec2 = float(dec2)
remradius = 6.
fieldradius = 6.
threads = None
itune1 = 5
itune2 = 5
ntune = 0
nocache = True
#Radius should be in arcminutes
if threads:
mp = multiproc(nthreads=threads)
else:
mp = multiproc()
IRLS_scale = 25.
dr9 = True
dr8 = False
noarcsinh = False
print(name)
prefix = '%s' % (name.replace(' ', '_'))
prefix1 = '%s' % (name1.replace(' ', '_'))
prefix2 = '%s' % (name2.replace(' ', '_'))
print('Removal Radius', remradius)
print('Field Radius', fieldradius)
print('RA,Dec', ra, dec)
print(os.getcwd())
rcfs = radec_to_sdss_rcf(ra,
dec,
radius=math.hypot(fieldradius, 13. / 2.),
tablefn="dr9fields.fits")
print(rcfs)
assert (len(rcfs) > 0)
assert (len(rcfs) < 15)
sras, sdecs, smags = tychoMatch(ra, dec, (fieldradius * 1.5) / 60.)
for sra, sdec, smag in zip(sras, sdecs, smags):
print(sra, sdec, smag)
imkw = dict(psf='dg')
if dr9:
getim = st.get_tractor_image_dr9
getsrc = st.get_tractor_sources_dr9
imkw.update(zrange=[-3, 100])
elif dr8:
getim = st.get_tractor_image_dr8
getsrc = st.get_tractor_sources_dr8
imkw.update(zrange=[-3, 100])
else:
getim = st.get_tractor_image
getsrc = st.get_tractor_sources_dr8
imkw.update(useMags=True)
bands = ['u', 'g', 'r', 'i', 'z']
bandname = 'r'
flipBands = ['r']
print(rcfs)
imsrcs = mp.map(
get_ims_and_srcs,
[(rcf +
(bands, ra, dec, fieldradius * 60. / 0.396, imkw, getim, getsrc))
for rcf in rcfs])
timgs = []
sources = []
allsources = []
for ims, s in imsrcs:
if ims is None:
continue
if s is None:
continue
timgs.extend(ims)
allsources.extend(s)
sources.append(s)
#rds = [rcf[3:5] for rcf in rcfs]
plotarea(ra, dec, fieldradius, name, prefix, timgs) #, rds)
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
tractor = st.Tractor(timgs, allsources, mp=mp)
sa = dict(debug=True, plotAll=False, plotBands=False)
if noarcsinh:
sa.update(nlscale=0)
elif dr8 or dr9:
sa.update(chilo=-8., chihi=8.)
if nocache:
tractor.cache = NullCache()
sg.disable_galaxy_cache()
zr = timgs[0].zr
print("zr is: ", zr)
print(bands)
print("Number of images: ", len(timgs))
# for timg,band in zip(timgs,bands):
# data = timg.getImage()/np.sqrt(timg.getInvvar())
# plt.hist(data,bins=100)
# plt.savefig('hist-%s.png' % (band))
saveAll('initial-' + prefix, tractor, **sa)
#plotInvvar('initial-'+prefix,tractor)
for sra, sdec, smag in zip(sras, sdecs, smags):
print(sra, sdec, smag)
for img in tractor.getImages():
wcs = img.getWcs()
starx, stary = wcs.positionToPixel(RaDecPos(sra, sdec))
starr = 25 * (2**(max(11 - smag, 0.)))
if starx + starr < 0. or starx - starr > img.getWidth(
) or stary + starr < 0. or stary - starr > img.getHeight():
continue
X, Y = np.meshgrid(np.arange(img.getWidth()),
np.arange(img.getHeight()))
R2 = (X - starx)**2 + (Y - stary)**2
img.getStarMask()[R2 < starr**2] = 0
for timgs, sources in imsrcs:
timg = timgs[0]
wcs = timg.getWcs()
xtr, ytr = wcs.positionToPixel(RaDecPos(ra, dec))
print(xtr, ytr)
xt = xtr
yt = ytr
r = ((remradius * 60.)) / .396 #radius in pixels
for src in sources:
xs, ys = wcs.positionToPixel(src.getPosition(), src)
if (xs - xt)**2 + (ys - yt)**2 <= r**2:
#print "Removed:", src
#print xs,ys
tractor.removeSource(src)
#saveAll('removed-'+prefix, tractor,**sa)
newShape = sg.GalaxyShape((remradius * 60.) / 10., 1., 0.)
newBright = ba.Mags(r=15.0,
g=15.0,
u=15.0,
z=15.0,
i=15.0,
order=['u', 'g', 'r', 'i', 'z'])
EG = st.ExpGalaxy(RaDecPos(ra, dec), newBright, newShape)
newShape2 = sg.GalaxyShape((remradius * 60.) / 10., 1., 0.)
newBright2 = ba.Mags(r=15.0,
g=15.0,
u=15.0,
z=15.0,
i=15.0,
order=['u', 'g', 'r', 'i', 'z'])
EG2 = st.ExpGalaxy(RaDecPos(ra2, dec2), newBright2, newShape2)
print(EG)
print(EG2)
tractor.addSource(EG)
tractor.addSource(EG2)
saveAll('added-' + prefix, tractor, **sa)
#print 'Tractor has', tractor.getParamNames()
for im in tractor.images:
im.freezeAllParams()
im.thawParam('sky')
tractor.catalog.freezeAllBut(EG)
tractor.catalog.thawParams(EG2)
#print 'Tractor has', tractor.getParamNames()
#print 'values', tractor.getParams()
for i in range(itune1):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('itune1-%d-' % (i + 1) + prefix, tractor, **sa)
tractor.clearCache()
sg.get_galaxy_cache().clear()
gc.collect()
print(resource.getpagesize())
print(resource.getrusage(resource.RUSAGE_SELF)[2])
CGPos = EG.getPosition()
CGShape1 = EG.getShape().copy()
CGShape2 = EG.getShape().copy()
EGBright = EG.getBrightness()
CGu = EGBright[0] + 0.75
CGg = EGBright[1] + 0.75
CGr = EGBright[2] + 0.75
CGi = EGBright[3] + 0.75
CGz = EGBright[4] + 0.75
CGBright1 = ba.Mags(r=CGr,
g=CGg,
u=CGu,
z=CGz,
i=CGi,
order=['u', 'g', 'r', 'i', 'z'])
CGBright2 = ba.Mags(r=CGr,
g=CGg,
u=CGu,
z=CGz,
i=CGi,
order=['u', 'g', 'r', 'i', 'z'])
print(EGBright)
print(CGBright1)
CG = st.CompositeGalaxy(CGPos, CGBright1, CGShape1, CGBright2, CGShape2)
CG2Pos = EG2.getPosition()
CG2Shape1 = EG2.getShape().copy()
CG2Shape2 = EG2.getShape().copy()
EG2Bright = EG2.getBrightness()
CG2u = EG2Bright[0] + 0.75
CG2g = EG2Bright[1] + 0.75
CG2r = EG2Bright[2] + 0.75
CG2i = EG2Bright[3] + 0.75
CG2z = EG2Bright[4] + 0.75
CG2Bright1 = ba.Mags(r=CG2r,
g=CG2g,
u=CG2u,
z=CG2z,
i=CG2i,
order=['u', 'g', 'r', 'i', 'z'])
CG2Bright2 = ba.Mags(r=CG2r,
g=CG2g,
u=CG2u,
z=CG2z,
i=CG2i,
order=['u', 'g', 'r', 'i', 'z'])
CG2 = st.CompositeGalaxy(CG2Pos, CG2Bright1, CG2Shape1, CG2Bright2,
CG2Shape2)
tractor.removeSource(EG)
tractor.removeSource(EG2)
tractor.addSource(CG)
tractor.addSource(CG2)
tractor.catalog.freezeAllBut(CG)
tractor.catalog.thawParams(CG2)
print(resource.getpagesize())
print(resource.getrusage(resource.RUSAGE_SELF)[2])
for i in range(itune2):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('itune2-%d-' % (i + 1) + prefix, tractor, **sa)
tractor.clearCache()
sg.get_galaxy_cache().clear()
print(resource.getpagesize())
print(resource.getrusage(resource.RUSAGE_SELF)[2])
tractor.catalog.thawAllParams()
for i in range(ntune):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('ntune-%d-' % (i + 1) + prefix, tractor, **sa)
#plotInvvar('final-'+prefix,tractor)
sa.update(plotBands=True)
saveAll('allBands-' + prefix, tractor, **sa)
print(CG)
print(CG.getPosition())
print(CGBright1)
print(CGBright2)
print(CGShape1)
print(CGShape2)
print(CGBright1 + CGBright2)
print(CG.getBrightness())
pfn = '%s.pickle' % prefix
pickle_to_file([CG, CG2], pfn)
makeflipbook(prefix, len(tractor.getImages()), itune1, itune2, ntune)
pickle_to_file(CG, "%s.pickle" % prefix1)
pickle_to_file(CG2, "%s.pickle" % prefix2)
os.system('cp %s.pickle RC3_Output' % prefix1)
os.system('cp %s.pickle RC3_Output' % prefix2)
def twogalaxies(name1,ra1,dec1,name2,ra2,dec2):
name = "%s %s" % (name1,name2)
ra = float(ra1)
dec = float(dec1)
ra2 = float(ra2)
dec2 = float(dec2)
remradius = 6.
fieldradius = 6.
threads = None
itune1=5
itune2=5
ntune=0
nocache=True
#Radius should be in arcminutes
if threads:
mp = multiproc(nthreads=threads)
else:
mp = multiproc()
IRLS_scale = 25.
dr9 = True
dr8 = False
noarcsinh = False
print name
prefix = '%s' % (name.replace(' ', '_'))
prefix1 = '%s' % (name1.replace(' ', '_'))
prefix2 = '%s' % (name2.replace(' ', '_'))
print 'Removal Radius', remradius
print 'Field Radius', fieldradius
print 'RA,Dec', ra, dec
print os.getcwd()
rcfs = radec_to_sdss_rcf(ra,dec,radius=math.hypot(fieldradius,13./2.),tablefn="dr9fields.fits")
print rcfs
assert(len(rcfs)>0)
assert(len(rcfs)<15)
sras, sdecs, smags = tychoMatch(ra,dec,(fieldradius*1.5)/60.)
for sra,sdec,smag in zip(sras,sdecs,smags):
print sra,sdec,smag
imkw = dict(psf='dg')
if dr9:
getim = st.get_tractor_image_dr9
getsrc = st.get_tractor_sources_dr9
imkw.update(zrange=[-3,100])
elif dr8:
getim = st.get_tractor_image_dr8
getsrc = st.get_tractor_sources_dr8
imkw.update(zrange=[-3,100])
else:
getim = st.get_tractor_image
getsrc = st.get_tractor_sources_dr8
imkw.update(useMags=True)
bands=['u','g','r','i','z']
bandname = 'r'
flipBands = ['r']
print rcfs
imsrcs = mp.map(get_ims_and_srcs, [(rcf + (bands, ra, dec, fieldradius*60./0.396, imkw, getim, getsrc))
for rcf in rcfs])
timgs = []
sources = []
allsources = []
for ims,s in imsrcs:
if ims is None:
continue
if s is None:
continue
timgs.extend(ims)
allsources.extend(s)
sources.append(s)
#rds = [rcf[3:5] for rcf in rcfs]
plotarea(ra, dec, fieldradius, name, prefix, timgs) #, rds)
lvl = logging.DEBUG
logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout)
tractor = st.Tractor(timgs, allsources, mp=mp)
sa = dict(debug=True, plotAll=False,plotBands=False)
if noarcsinh:
sa.update(nlscale=0)
elif dr8 or dr9:
sa.update(chilo=-8.,chihi=8.)
if nocache:
tractor.cache = NullCache()
sg.disable_galaxy_cache()
zr = timgs[0].zr
print "zr is: ",zr
print bands
print "Number of images: ", len(timgs)
# for timg,band in zip(timgs,bands):
# data = timg.getImage()/np.sqrt(timg.getInvvar())
# plt.hist(data,bins=100)
# plt.savefig('hist-%s.png' % (band))
saveAll('initial-'+prefix, tractor,**sa)
#plotInvvar('initial-'+prefix,tractor)
for sra,sdec,smag in zip(sras,sdecs,smags):
print sra,sdec,smag
for img in tractor.getImages():
wcs = img.getWcs()
starx,stary = wcs.positionToPixel(RaDecPos(sra,sdec))
starr=25*(2**(max(11-smag,0.)))
if starx+starr<0. or starx-starr>img.getWidth() or stary+starr <0. or stary-starr>img.getHeight():
continue
X,Y = np.meshgrid(np.arange(img.getWidth()), np.arange(img.getHeight()))
R2 = (X - starx)**2 + (Y - stary)**2
img.getStarMask()[R2 < starr**2] = 0
for timgs,sources in imsrcs:
timg = timgs[0]
wcs = timg.getWcs()
xtr,ytr = wcs.positionToPixel(RaDecPos(ra,dec))
print xtr,ytr
xt = xtr
yt = ytr
r = ((remradius*60.))/.396 #radius in pixels
for src in sources:
xs,ys = wcs.positionToPixel(src.getPosition(),src)
if (xs-xt)**2+(ys-yt)**2 <= r**2:
#print "Removed:", src
#print xs,ys
tractor.removeSource(src)
#saveAll('removed-'+prefix, tractor,**sa)
newShape = sg.GalaxyShape((remradius*60.)/10.,1.,0.)
newBright = ba.Mags(r=15.0,g=15.0,u=15.0,z=15.0,i=15.0,order=['u','g','r','i','z'])
EG = st.ExpGalaxy(RaDecPos(ra,dec),newBright,newShape)
newShape2 = sg.GalaxyShape((remradius*60.)/10.,1.,0.)
newBright2 = ba.Mags(r=15.0,g=15.0,u=15.0,z=15.0,i=15.0,order=['u','g','r','i','z'])
EG2 = st.ExpGalaxy(RaDecPos(ra2,dec2),newBright2,newShape2)
print EG
print EG2
tractor.addSource(EG)
tractor.addSource(EG2)
saveAll('added-'+prefix,tractor,**sa)
#print 'Tractor has', tractor.getParamNames()
for im in tractor.images:
im.freezeAllParams()
im.thawParam('sky')
tractor.catalog.freezeAllBut(EG)
tractor.catalog.thawParams(EG2)
#print 'Tractor has', tractor.getParamNames()
#print 'values', tractor.getParams()
for i in range(itune1):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('itune1-%d-' % (i+1)+prefix,tractor,**sa)
tractor.clearCache()
sg.get_galaxy_cache().clear()
gc.collect()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
CGPos = EG.getPosition()
CGShape1 = EG.getShape().copy()
CGShape2 = EG.getShape().copy()
EGBright = EG.getBrightness()
CGu = EGBright[0] + 0.75
CGg = EGBright[1] + 0.75
CGr = EGBright[2] + 0.75
CGi = EGBright[3] + 0.75
CGz = EGBright[4] + 0.75
CGBright1 = ba.Mags(r=CGr,g=CGg,u=CGu,z=CGz,i=CGi,order=['u','g','r','i','z'])
CGBright2 = ba.Mags(r=CGr,g=CGg,u=CGu,z=CGz,i=CGi,order=['u','g','r','i','z'])
print EGBright
print CGBright1
CG = st.CompositeGalaxy(CGPos,CGBright1,CGShape1,CGBright2,CGShape2)
CG2Pos = EG2.getPosition()
CG2Shape1 = EG2.getShape().copy()
CG2Shape2 = EG2.getShape().copy()
EG2Bright = EG2.getBrightness()
CG2u = EG2Bright[0] + 0.75
CG2g = EG2Bright[1] + 0.75
CG2r = EG2Bright[2] + 0.75
CG2i = EG2Bright[3] + 0.75
CG2z = EG2Bright[4] + 0.75
CG2Bright1 = ba.Mags(r=CG2r,g=CG2g,u=CG2u,z=CG2z,i=CG2i,order=['u','g','r','i','z'])
CG2Bright2 = ba.Mags(r=CG2r,g=CG2g,u=CG2u,z=CG2z,i=CG2i,order=['u','g','r','i','z'])
CG2 = st.CompositeGalaxy(CG2Pos,CG2Bright1,CG2Shape1,CG2Bright2,CG2Shape2)
tractor.removeSource(EG)
tractor.removeSource(EG2)
tractor.addSource(CG)
tractor.addSource(CG2)
tractor.catalog.freezeAllBut(CG)
tractor.catalog.thawParams(CG2)
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
for i in range(itune2):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('itune2-%d-' % (i+1)+prefix,tractor,**sa)
tractor.clearCache()
sg.get_galaxy_cache().clear()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.catalog.thawAllParams()
for i in range(ntune):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('ntune-%d-' % (i+1)+prefix,tractor,**sa)
#plotInvvar('final-'+prefix,tractor)
sa.update(plotBands=True)
saveAll('allBands-' + prefix,tractor,**sa)
print CG
print CG.getPosition()
print CGBright1
print CGBright2
print CGShape1
print CGShape2
print CGBright1+CGBright2
print CG.getBrightness()
pfn = '%s.pickle' % prefix
pickle_to_file([CG,CG2],pfn)
makeflipbook(prefix,len(tractor.getImages()),itune1,itune2,ntune)
pickle_to_file(CG,"%s.pickle" % prefix1)
pickle_to_file(CG2,"%s.pickle" % prefix2)
os.system('cp %s.pickle RC3_Output' % prefix1)
os.system('cp %s.pickle RC3_Output' % prefix2)
def test_psfex(self):
if ps is not None:
from astrometry.util.plotutils import dimshow
import pylab as plt
H, W = 100, 100
cx, cy = W / 2., H / 2.
pixpsf = self.psf.constantPsfAt(cx, cy)
ph, pw = pixpsf.shape
xx, yy = np.meshgrid(np.arange(pw), np.arange(ph))
im = pixpsf.img.copy()
im /= np.sum(im)
cenx, ceny = np.sum(im * xx), np.sum(im * yy)
print('Pixpsf centroid:', cenx, ceny)
print('shape:', ph, pw)
dx, dy = cenx - pw // 2, ceny - ph // 2
print('dx,dy', dx, dy)
# gpsf = GaussianMixturePSF.fromStamp(im, N=1)
# print('Fit gpsf:', gpsf)
# self.assertTrue(np.abs(gpsf.mog.mean[0,0] - dx) < 0.1)
# self.assertTrue(np.abs(gpsf.mog.mean[0,1] - dy) < 0.1)
# self.assertTrue(np.abs(gpsf.mog.var[0,0,0] - 15.5) < 1.)
# self.assertTrue(np.abs(gpsf.mog.var[0,1,1] - 13.5) < 1.)
# self.assertTrue(np.abs(gpsf.mog.var[0,1,0] - -1) < 1.)
gpsf = GaussianMixturePSF.fromStamp(im, N=2)
print('Fit gpsf:', gpsf)
print('Params:', ', '.join(['%.1f' % p for p in gpsf.getParams()]))
pp = np.array(
[0.8, 0.2, 0.1, -0.0, 1.2, 0.2, 7.6, 6.0, -1.0, 51.6, 49.1, -1.3])
self.assertTrue(np.all(np.abs(np.array(gpsf.getParams()) - pp) < 0.1))
tim = Image(data=np.zeros((H, W)),
invvar=np.ones((H, W)),
psf=self.psf)
xx, yy = np.meshgrid(np.arange(W), np.arange(H))
star = PointSource(PixPos(cx, cy), Flux(100.))
gal = ExpGalaxy(PixPos(cx, cy), Flux(100.), EllipseE(1., 0., 0.))
tr1 = Tractor([tim], [star])
tr2 = Tractor([tim], [gal])
disable_galaxy_cache()
tim.psf = self.psf
mod = tr1.getModelImage(0)
mod1 = mod
im = mod.copy()
im /= im.sum()
cenx, ceny = np.sum(im * xx), np.sum(im * yy)
print('Star model + PsfEx centroid', cenx, ceny)
self.assertTrue(np.abs(cenx - (cx + dx)) < 0.1)
self.assertTrue(np.abs(ceny - (cy + dy)) < 0.1)
if ps is not None:
plt.clf()
dimshow(mod)
plt.title('Star model, PsfEx')
ps.savefig()
tim.psf = pixpsf
mod = tr1.getModelImage(0)
if ps is not None:
plt.clf()
dimshow(mod)
plt.title('Star model, pixpsf')
ps.savefig()
tim.psf = gpsf
mod = tr1.getModelImage(0)
mod2 = mod
if ps is not None:
plt.clf()
dimshow(mod)
plt.title('Star model, gpsf')
plt.colorbar()
ps.savefig()
plt.clf()
dimshow(mod1 - mod2)
plt.title('Star model, PsfEx - gpsf')
plt.colorbar()
ps.savefig()
# range ~ -0.15 to +0.25
self.assertTrue(np.all(np.abs(mod1 - mod2) < 0.25))
tim.psf = self.psf
mod = tr2.getModelImage(0)
mod1 = mod
im = mod.copy()
im /= im.sum()
cenx, ceny = np.sum(im * xx), np.sum(im * yy)
print('Gal model + PsfEx centroid', cenx, ceny)
self.assertTrue(np.abs(cenx - (cx + dx)) < 0.1)
self.assertTrue(np.abs(ceny - (cy + dy)) < 0.1)
if ps is not None:
plt.clf()
dimshow(mod)
plt.title('Gal model, PsfEx')
ps.savefig()
# tim.psf = pixpsf
# mod = tr2.getModelImage(0)
# plt.clf()
# dimshow(mod)
# plt.title('Gal model, pixpsf')
# ps.savefig()
tim.psf = gpsf
mod = tr2.getModelImage(0)
mod2 = mod
# range ~ -0.1 to +0.2
self.assertTrue(np.all(np.abs(mod1 - mod2) < 0.2))
if ps is not None:
plt.clf()
dimshow(mod)
plt.title('Gal model, gpsf')
ps.savefig()
plt.clf()
dimshow(mod1 - mod2)
plt.title('Gal model, PsfEx - gpsf')
plt.colorbar()
ps.savefig()
def main(survey=None, opt=None):
'''Driver function for forced photometry of individual DECam images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0,x1,y0,y1) = opt.zoom
zoomslice = (slice(y0,y1), slice(x0,x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.filename)
opt.filename = None
except:
# make this 'None' for survey.find_ccds()
expnum = None
# Try parsing HDU number
try:
opt.hdu = int(opt.hdu)
ccdname = None
except:
ccdname = opt.hdu
opt.hdu = -1
if survey is None:
survey = LegacySurveyData()
if opt.filename is not None and opt.hdu >= 0:
# Read metadata from file
T = exposure_metadata([opt.filename], hdus=[opt.hdu])
print('Metadata:')
T.about()
else:
# Read metadata from survey-ccds.fits table
T = survey.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if opt.hdu >= 0:
T.cut(T.image_hdu == opt.hdu)
print(len(T), 'with HDU', opt.hdu)
if opt.filename is not None:
T.cut(np.array([f.strip() == opt.filename for f in T.image_filename]))
print(len(T), 'with filename', opt.filename)
assert(len(T) == 1)
ccd = T[0]
im = survey.get_image_object(ccd)
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True,
constant_invvar=opt.constant_invvar)
print('Got tim:', tim)
print('Read image:', Time()-t0)
if opt.catfn in ['DR1', 'DR2', 'DR3']:
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, survey=survey)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = survey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W,H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) *
(yy >= -margin) * (yy <= (H+margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print('Cut to', len(T), 'on out_of_bounds and left_blob')
if len(T):
TT.append(T)
if len(TT) == 0:
print('No sources to photometer.')
return 0
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
surveydir = survey.get_survey_dir()
del survey
cat = read_fits_catalog(T)
# print('Got cat:', cat)
print('Read catalog:', Time()-t0)
print('Forced photom...')
opti = None
forced_kwargs = {}
if opt.ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
opti = CeresOptimizer(BW=B, BH=B)
#forced_kwargs.update(verbose=True)
for src in cat:
# Limit sizes of huge models
from tractor.galaxy import ProfileGalaxy
if isinstance(src, ProfileGalaxy):
px,py = tim.wcs.positionToPixel(src.getPosition())
h = src._getUnitFluxPatchSize(tim, px, py, tim.modelMinval)
MAXHALF = 128
if h > MAXHALF:
print('halfsize', h,'for',src,'-> setting to',MAXHALF)
src.halfsize = MAXHALF
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
for src in cat:
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
disable_galaxy_cache()
F = fits_table()
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32)
ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x-1).astype(np.float32)
F.y = (y-1).astype(np.float32)
if opt.forced:
if opt.plots is None:
forced_kwargs.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True, fitstats=True,
shared_params=False, priors=False, **forced_kwargs)
if opt.plots:
(data,mod,ie,chi,roi) = R.ims1[0]
ima = tim.ima
imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5)
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
F.flux = np.array([src.getBrightness().getFlux(tim.band)
for src in cat]).astype(np.float32)
F.flux_ivar = R.IV.astype(np.float32)
F.fracflux = R.fitstats.profracflux.astype(np.float32)
F.rchi2 = R.fitstats.prochi2 .astype(np.float32)
print('Forced photom:', Time()-t0)
if opt.apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
print('Apertures:', apertures, 'pixels')
for rad in apertures:
aper = photutils.CircularAperture(apxy, rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = ap.astype(np.float32)
ap = 1./(np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux_ivar = ap.astype(np.float32)
print('Aperture photom:', Time()-t0)
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, surveydir)
filename = getattr(ccd, 'image_filename')
if filename is None:
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
filename = os.path.join(d2, d1, fname)
print('Trimmed filename to', filename)
version_hdr.add_record(dict(name='CPFILE', value=filename, comment='CP file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext'))
version_hdr.add_record(dict(name='CAMERA', value=ccd.camera, comment='Camera'))
version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='Exposure num'))
version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='CCD name'))
version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(dict(name='EXPOSURE',
value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname),
comment='Name of this image'))
keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV',
'INSTRUME']
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {'exptime':'sec', 'flux':'nanomaggy', 'flux_ivar':'1/nanomaggy^2'}
columns = F.get_columns()
for i,col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
if opt.save_model or opt.save_data:
hdr = fitsio.FITSHDR()
tim.getWcs().wcs.add_to_header(hdr)
if opt.save_model:
print('Getting model image...')
mod = tr.getModelImage(tim)
fitsio.write(opt.save_model, mod, header=hdr, clobber=True)
print('Wrote', opt.save_model)
if opt.save_data:
fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True)
print('Wrote', opt.save_data)
print('Finished forced phot:', Time()-t0)
return 0
def general(name,ra,dec,remradius,fieldradius,threads=None,itune1=5,itune2=5,ntune=0,nocache=False,scale=1,ab=1.,angle=0.):
#Radius should be in arcminutes
if threads:
mp = multiproc(nthreads=threads)
else:
mp = multiproc()
IRLS_scale = 25.
dr9 = True
dr8 = False
noarcsinh = False
print name
prefix = 'swapCG_%s' % (name.replace(' ', '_'))
print 'Removal Radius', remradius
print 'Field Radius', fieldradius
print 'RA,Dec', ra, dec
print os.getcwd()
print ra,dec,math.hypot(fieldradius,13./2.)
rcfs = radec_to_sdss_rcf(ra,dec,radius=math.hypot(fieldradius,13./2.),tablefn="dr9fields.fits")
print 'RCFS:', rcfs
print len(rcfs)
assert(len(rcfs)>0)
if 10 <= len(rcfs) < 20:
scale = 2
elif 20 <= len(rcfs) < 40:
scale = 4
elif 40 <= len(rcfs) < 80:
scale = 8
assert(len(rcfs)<80)
sras, sdecs, smags = tychoMatch(ra,dec,(fieldradius*1.5)/60.)
imkw = dict(psf='kl-gm')
if dr9:
getim = st.get_tractor_image_dr9
getsrc = st.get_tractor_sources_dr9
imkw.update(zrange=[-3,100])
elif dr8:
getim = st.get_tractor_image_dr8
getsrc = st.get_tractor_sources_dr8
imkw.update(zrange=[-3,100])
else:
getim = st.get_tractor_image
getsrc = st.get_tractor_sources_dr8
imkw.update(useMags=True)
bands=['u','g','r','i','z']
#bands=['r']
bandname = 'r'
flipBands = ['r']
print rcfs
imsrcs = mp.map(get_ims_and_srcs, [(rcf + (bands, ra, dec, fieldradius*60./0.396, imkw, getim, getsrc))
for rcf in rcfs])
timgs = []
sources = []
allsources = []
for ims,s in imsrcs:
if ims is None:
continue
if s is None:
continue
if scale > 1:
for im in ims:
timgs.append(st.scale_sdss_image(im,scale))
else:
timgs.extend(ims)
allsources.extend(s)
sources.append(s)
#rds = [rcf[3:5] for rcf in rcfs]
#plotarea(ra, dec, fieldradius, name, prefix, timgs) #, rds)
#lvl = logging.DEBUG
#logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout)
tractor = st.Tractor(timgs, allsources, mp=mp)
sa = dict(debug=True, plotAll=False,plotBands=False)
if noarcsinh:
sa.update(nlscale=0)
elif dr8 or dr9:
sa.update(chilo=-8.,chihi=8.)
if nocache:
tractor.cache = NullCache()
sg.disable_galaxy_cache()
zr = timgs[0].zr
print "zr is: ",zr
print bands
print "Number of images: ", len(timgs)
#for timg,band in zip(timgs,bands):
# data = timg.getImage()/np.sqrt(timg.getInvvar())
# plt.hist(data,bins=100)
# plt.savefig('hist-%s.png' % (band))
saveAll('initial-'+prefix, tractor,**sa)
#plotInvvar('initial-'+prefix,tractor)
for sra,sdec,smag in zip(sras,sdecs,smags):
for img in tractor.getImages():
wcs = img.getWcs()
starx,stary = wcs.positionToPixel(RaDecPos(sra,sdec))
starr=25*(2**(max(11-smag,0.)))
if starx+starr<0. or starx-starr>img.getWidth() or stary+starr <0. or stary-starr>img.getHeight():
continue
X,Y = np.meshgrid(np.arange(img.getWidth()), np.arange(img.getHeight()))
R2 = (X - starx)**2 + (Y - stary)**2
img.getStarMask()[R2 < starr**2] = 0
for timgs,sources in imsrcs:
timg = timgs[0]
wcs = timg.getWcs()
xtr,ytr = wcs.positionToPixel(RaDecPos(ra,dec))
xt = xtr
yt = ytr
r = ((remradius*60.))/.396 #radius in pixels
for src in sources:
xs,ys = wcs.positionToPixel(src.getPosition(),src)
if (xs-xt)**2+(ys-yt)**2 <= r**2:
#print "Removed:", src
#print xs,ys
tractor.removeSource(src)
#saveAll('removed-'+prefix, tractor,**sa)
newShape = sg.GalaxyShape((remradius*60.)/10.,ab,angle)
newBright = ba.Mags(r=15.0,g=15.0,u=15.0,z=15.0,i=15.0,order=['u','g','r','i','z'])
EG = st.ExpGalaxy(RaDecPos(ra,dec),newBright,newShape)
print EG
tractor.addSource(EG)
saveAll('added-'+prefix,tractor,**sa)
#print 'Tractor has', tractor.getParamNames()
for im in tractor.images:
im.freezeAllParams()
im.thawParam('sky')
tractor.catalog.freezeAllBut(EG)
#print 'Tractor has', tractor.getParamNames()
#print 'values', tractor.getParams()
for i in range(itune1):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('itune1-%d-' % (i+1)+prefix,tractor,**sa)
tractor.clearCache()
sg.get_galaxy_cache().clear()
gc.collect()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
CGPos = EG.getPosition()
CGShape1 = EG.getShape().copy()
CGShape2 = EG.getShape().copy()
EGBright = EG.getBrightness()
CGu = EGBright[0] + 0.75
CGg = EGBright[1] + 0.75
CGr = EGBright[2] + 0.75
CGi = EGBright[3] + 0.75
CGz = EGBright[4] + 0.75
CGBright1 = ba.Mags(r=CGr,g=CGg,u=CGu,z=CGz,i=CGi,order=['u','g','r','i','z'])
CGBright2 = ba.Mags(r=CGr,g=CGg,u=CGu,z=CGz,i=CGi,order=['u','g','r','i','z'])
print EGBright
print CGBright1
CG = st.CompositeGalaxy(CGPos,CGBright1,CGShape1,CGBright2,CGShape2)
tractor.removeSource(EG)
tractor.addSource(CG)
tractor.catalog.freezeAllBut(CG)
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
for i in range(itune2):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('itune2-%d-' % (i+1)+prefix,tractor,**sa)
tractor.clearCache()
sg.get_galaxy_cache().clear()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.catalog.thawAllParams()
for i in range(ntune):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('ntune-%d-' % (i+1)+prefix,tractor,**sa)
#plotInvvar('final-'+prefix,tractor)
sa.update(plotBands=True)
saveAll('allBands-' + prefix,tractor,**sa)
print "end of first round of optimization:", tractor.getLogLikelihood()
print CG
print CG.getPosition()
print CGBright1
print CGBright2
print CGShape1
print CGShape2
print CGBright1+CGBright2
print CG.getBrightness()
pfn = '%s.pickle' % prefix
pickle_to_file(CG,pfn)
makeflipbook(prefix,len(tractor.getImages()),itune1,itune2,ntune)
# now SWAP exp and dev and DO IT AGAIN
newCG = st.CompositeGalaxy(CG.getPosition, CG.brightnessDev.copy(),
CG.shapeDev.copy(), CG.brightnessExp.copy(),
CG.shapeExp.copy())
tractor.removeSource(CG)
tractor.addSource(newCG)
tractor.catalog.thawAllParams()
for i in range(ntune):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('ntune-swap-%d-' % (i+1)+prefix,tractor,**sa)
#plotInvvar('final-'+prefix,tractor)
sa.update(plotBands=True)
saveAll('allBands-swap-' + prefix,tractor,**sa)
print "end of second (swapped) round of optimization:", tractor.getLogLikelihood()
print newCG
print newCG.getPosition()
print newCG.getBrightness()
pfn = '%s-swap.pickle' % prefix
pickle_to_file(newCG,pfn)
makeflipbook(prefix+"-swap",len(tractor.getImages()),itune1,itune2,ntune)
def general(name,
ra,
dec,
remradius,
fieldradius,
threads=None,
itune1=5,
itune2=5,
ntune=0,
nocache=False,
scale=1,
ab=1.,
angle=0.):
#Radius should be in arcminutes
if threads:
mp = multiproc(nthreads=threads)
else:
mp = multiproc()
IRLS_scale = 25.
dr9 = True
dr8 = False
noarcsinh = False
print name
prefix = 'swapCG_%s' % (name.replace(' ', '_'))
print 'Removal Radius', remradius
print 'Field Radius', fieldradius
print 'RA,Dec', ra, dec
print os.getcwd()
print ra, dec, math.hypot(fieldradius, 13. / 2.)
rcfs = radec_to_sdss_rcf(ra,
dec,
radius=math.hypot(fieldradius, 13. / 2.),
tablefn="dr9fields.fits")
print 'RCFS:', rcfs
print len(rcfs)
assert (len(rcfs) > 0)
if 10 <= len(rcfs) < 20:
scale = 2
elif 20 <= len(rcfs) < 40:
scale = 4
elif 40 <= len(rcfs) < 80:
scale = 8
assert (len(rcfs) < 80)
sras, sdecs, smags = tychoMatch(ra, dec, (fieldradius * 1.5) / 60.)
imkw = dict(psf='kl-gm')
if dr9:
getim = st.get_tractor_image_dr9
getsrc = st.get_tractor_sources_dr9
imkw.update(zrange=[-3, 100])
elif dr8:
getim = st.get_tractor_image_dr8
getsrc = st.get_tractor_sources_dr8
imkw.update(zrange=[-3, 100])
else:
getim = st.get_tractor_image
getsrc = st.get_tractor_sources_dr8
imkw.update(useMags=True)
bands = ['u', 'g', 'r', 'i', 'z']
#bands=['r']
bandname = 'r'
flipBands = ['r']
print rcfs
imsrcs = mp.map(
get_ims_and_srcs,
[(rcf +
(bands, ra, dec, fieldradius * 60. / 0.396, imkw, getim, getsrc))
for rcf in rcfs])
timgs = []
sources = []
allsources = []
for ims, s in imsrcs:
if ims is None:
continue
if s is None:
continue
if scale > 1:
for im in ims:
timgs.append(st.scale_sdss_image(im, scale))
else:
timgs.extend(ims)
allsources.extend(s)
sources.append(s)
#rds = [rcf[3:5] for rcf in rcfs]
#plotarea(ra, dec, fieldradius, name, prefix, timgs) #, rds)
#lvl = logging.DEBUG
#logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout)
tractor = st.Tractor(timgs, allsources, mp=mp)
sa = dict(debug=True, plotAll=False, plotBands=False)
if noarcsinh:
sa.update(nlscale=0)
elif dr8 or dr9:
sa.update(chilo=-8., chihi=8.)
if nocache:
tractor.cache = NullCache()
sg.disable_galaxy_cache()
zr = timgs[0].zr
print "zr is: ", zr
print bands
print "Number of images: ", len(timgs)
#for timg,band in zip(timgs,bands):
# data = timg.getImage()/np.sqrt(timg.getInvvar())
# plt.hist(data,bins=100)
# plt.savefig('hist-%s.png' % (band))
saveAll('initial-' + prefix, tractor, **sa)
#plotInvvar('initial-'+prefix,tractor)
for sra, sdec, smag in zip(sras, sdecs, smags):
for img in tractor.getImages():
wcs = img.getWcs()
starx, stary = wcs.positionToPixel(RaDecPos(sra, sdec))
starr = 25 * (2**(max(11 - smag, 0.)))
if starx + starr < 0. or starx - starr > img.getWidth(
) or stary + starr < 0. or stary - starr > img.getHeight():
continue
X, Y = np.meshgrid(np.arange(img.getWidth()),
np.arange(img.getHeight()))
R2 = (X - starx)**2 + (Y - stary)**2
img.getStarMask()[R2 < starr**2] = 0
for timgs, sources in imsrcs:
timg = timgs[0]
wcs = timg.getWcs()
xtr, ytr = wcs.positionToPixel(RaDecPos(ra, dec))
xt = xtr
yt = ytr
r = ((remradius * 60.)) / .396 #radius in pixels
for src in sources:
xs, ys = wcs.positionToPixel(src.getPosition(), src)
if (xs - xt)**2 + (ys - yt)**2 <= r**2:
#print "Removed:", src
#print xs,ys
tractor.removeSource(src)
#saveAll('removed-'+prefix, tractor,**sa)
newShape = sg.GalaxyShape((remradius * 60.) / 10., ab, angle)
newBright = ba.Mags(r=15.0,
g=15.0,
u=15.0,
z=15.0,
i=15.0,
order=['u', 'g', 'r', 'i', 'z'])
EG = st.ExpGalaxy(RaDecPos(ra, dec), newBright, newShape)
print EG
tractor.addSource(EG)
saveAll('added-' + prefix, tractor, **sa)
#print 'Tractor has', tractor.getParamNames()
for im in tractor.images:
im.freezeAllParams()
im.thawParam('sky')
tractor.catalog.freezeAllBut(EG)
#print 'Tractor has', tractor.getParamNames()
#print 'values', tractor.getParams()
for i in range(itune1):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('itune1-%d-' % (i + 1) + prefix, tractor, **sa)
tractor.clearCache()
sg.get_galaxy_cache().clear()
gc.collect()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
CGPos = EG.getPosition()
CGShape1 = EG.getShape().copy()
CGShape2 = EG.getShape().copy()
EGBright = EG.getBrightness()
CGu = EGBright[0] + 0.75
CGg = EGBright[1] + 0.75
CGr = EGBright[2] + 0.75
CGi = EGBright[3] + 0.75
CGz = EGBright[4] + 0.75
CGBright1 = ba.Mags(r=CGr,
g=CGg,
u=CGu,
z=CGz,
i=CGi,
order=['u', 'g', 'r', 'i', 'z'])
CGBright2 = ba.Mags(r=CGr,
g=CGg,
u=CGu,
z=CGz,
i=CGi,
order=['u', 'g', 'r', 'i', 'z'])
print EGBright
print CGBright1
CG = st.CompositeGalaxy(CGPos, CGBright1, CGShape1, CGBright2, CGShape2)
tractor.removeSource(EG)
tractor.addSource(CG)
tractor.catalog.freezeAllBut(CG)
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
for i in range(itune2):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('itune2-%d-' % (i + 1) + prefix, tractor, **sa)
tractor.clearCache()
sg.get_galaxy_cache().clear()
print resource.getpagesize()
print resource.getrusage(resource.RUSAGE_SELF)[2]
tractor.catalog.thawAllParams()
for i in range(ntune):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('ntune-%d-' % (i + 1) + prefix, tractor, **sa)
#plotInvvar('final-'+prefix,tractor)
sa.update(plotBands=True)
saveAll('allBands-' + prefix, tractor, **sa)
print "end of first round of optimization:", tractor.getLogLikelihood()
print CG
print CG.getPosition()
print CGBright1
print CGBright2
print CGShape1
print CGShape2
print CGBright1 + CGBright2
print CG.getBrightness()
pfn = '%s.pickle' % prefix
pickle_to_file(CG, pfn)
makeflipbook(prefix, len(tractor.getImages()), itune1, itune2, ntune)
# now SWAP exp and dev and DO IT AGAIN
newCG = st.CompositeGalaxy(CG.getPosition, CG.brightnessDev.copy(),
CG.shapeDev.copy(), CG.brightnessExp.copy(),
CG.shapeExp.copy())
tractor.removeSource(CG)
tractor.addSource(newCG)
tractor.catalog.thawAllParams()
for i in range(ntune):
tractor.optimize()
tractor.changeInvvar(IRLS_scale)
saveAll('ntune-swap-%d-' % (i + 1) + prefix, tractor, **sa)
#plotInvvar('final-'+prefix,tractor)
sa.update(plotBands=True)
saveAll('allBands-swap-' + prefix, tractor, **sa)
print "end of second (swapped) round of optimization:", tractor.getLogLikelihood(
)
print newCG
print newCG.getPosition()
print newCG.getBrightness()
pfn = '%s-swap.pickle' % prefix
pickle_to_file(newCG, pfn)
makeflipbook(prefix + "-swap", len(tractor.getImages()), itune1, itune2,
ntune)
