def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--force', action='store_true',
help='Run calib processes even if files already exist?')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--expnum', type=int, help='Cut to a single exposure')
parser.add_argument('--extname', '--ccdname', help='Cut to a single extension/CCD name')
parser.add_argument('--no-astrom', dest='astrom', action='store_false',
help='Do not compute astrometric calibs')
parser.add_argument('--no-psf', dest='psfex', action='store_false',
help='Do not compute PsfEx calibs')
parser.add_argument('--no-sky', dest='sky', action='store_false',
help='Do not compute sky models')
parser.add_argument('--run-se', action='store_true', help='Run SourceExtractor')
parser.add_argument('--splinesky', action='store_true', help='Spline sky, not constant')
parser.add_argument('args',nargs=argparse.REMAINDER)
opt = parser.parse_args()
D = Decals()
if opt.ccds is not None:
T = fits_table(opt.ccds)
print('Read', len(T), 'from', opt.ccds)
else:
T = D.get_ccds()
#print len(T), 'CCDs'
if len(opt.args) == 0:
if opt.expnum is not None:
T.cut(T.expnum == opt.expnum)
print('Cut to', len(T), 'with expnum =', opt.expnum)
if opt.extname is not None:
T.cut(np.array([(t.strip() == opt.extname) for t in T.ccdname]))
print('Cut to', len(T), 'with extname =', opt.extname)
opt.args = range(len(T))
for a in opt.args:
i = int(a)
t = T[i]
im = D.get_image_object(t)
print('Running', im.calname)
kwargs = dict(pvastrom=opt.astrom, psfex=opt.psfex, sky=opt.sky)
if opt.force:
kwargs.update(force=True)
if opt.run_se:
kwargs.update(se=True)
if opt.splinesky:
kwargs.update(splinesky=True)
run_calibs((im, kwargs))
return 0
def main():
brickname = '2428p117'
objtype = 'STAR'
lobjtype = objtype.lower()
decals_sim_dir = '/Users/ioannis/decals_sim_dir/star/2428p117'
zoom = [1800,2400,1800,2400]
#zoom = [1800,2000,1800,2000]
#zoom = None
decals = Decals()
brickinfo = decals.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale()
if zoom is not None:
# See also runbrick.stage_tims()
#(x0,x1,y0,y1) = args.zoom
(x0,x1,y0,y1) = zoom
W = x1-x0
H = y1-y0
targetwcs = brickwcs.get_subimage(x0, y0, W, H)
bounds = brickwcs.radec_bounds()
ra_range = bounds[1]-bounds[0]
dec_range = bounds[3]-bounds[2]
radec_center = brickwcs.radec_center()
print(radec_center, ra_range, dec_range)
corners = np.array([brickwcs.pixelxy2radec(x,y) for x,y in
[(1,1),(W,1),(W,H),(1,H),(1,1)]])
# Identify the CCDs in the region of interest.
ccdinfo = decals.ccds_touching_wcs(brickwcs)
ccdinfo.about()
log.info('Got {} CCDs'.format(len(ccdinfo)))
# Generate the catalog of simulated sources here!
simcat = fits_table('simcat-{}-{}-00.fits'.format(brickname,lobjtype))
#simcat = fits.getdata('simcat-{}-{}-00.fits'.format(brickname,lobjtype))
simcat.about()
blobxy = zip(simcat.x,simcat.y)
simdecals = SimDecals(sim_sources=simcat)
# For testing!
#sim = SimImage(simdecals,ccdinfo[0])
#tim = sim.get_tractor_image(const2psf=True)
run_brick(brickname, decals=simdecals, outdir=decals_sim_dir,
threads=8, zoom=zoom, wise=False, sdssInit=False,
forceAll=True, writePickles=False, blobxy=blobxy,
pixPsf=False, stages=['tims','srcs','fitblobs',
'fitblobs_finish','coadds',
'writecat'])
def __init__(self,expnum=None,ccdname=None,ccdwcs=None):
"""Initialize the class with either the exposure number *and* CCD name, or
directly with the WCS of the CCD of interest.
"""
self.ps1dir = os.getenv('PS1CAT_DIR')
self.nside = 32
if ccdwcs is None:
from legacypipe.common import Decals, DecamImage
decals = Decals()
ccd = decals.find_ccds(expnum=expnum,ccdname=ccdname)[0]
im = DecamImage(decals,ccd)
self.ccdwcs = im.get_wcs()
else:
self.ccdwcs = ccdwcs
def __init__(self,expnum=None,ccdname=None,ccdwcs=None):
"""Initialize the class with either the exposure number *and* CCD name, or
directly with the WCS of the CCD of interest.
"""
self.ps1dir = os.getenv('PS1CAT_DIR')
if self.ps1dir is None:
raise ValueError('You must have the PS1CAT_DIR environment variable set to point to Pan-STARRS1 catalogs')
self.nside = 32
if ccdwcs is None:
from legacypipe.common import Decals
decals = Decals()
ccd = decals.find_ccds(expnum=expnum,ccdname=ccdname)[0]
im = decals.get_image_object(ccd)
self.ccdwcs = im.get_wcs()
else:
self.ccdwcs = ccdwcs
def __init__(self, expnum=None, ccdname=None, ccdwcs=None):
"""Initialize the class with either the exposure number *and* CCD name, or
directly with the WCS of the CCD of interest.
"""
self.ps1dir = os.getenv('PS1CAT_DIR')
if self.ps1dir is None:
raise ValueError(
'You must have the PS1CAT_DIR environment variable set to point to Pan-STARRS1 catalogs'
)
self.nside = 32
if ccdwcs is None:
from legacypipe.common import Decals
decals = Decals()
ccd = decals.find_ccds(expnum=expnum, ccdname=ccdname)[0]
im = decals.get_image_object(ccd)
self.ccdwcs = im.get_wcs()
else:
self.ccdwcs = ccdwcs
def psf_residuals(expnum,
ccdname,
stampsize=35,
nstar=30,
magrange=(13, 17),
verbose=0,
splinesky=False):
# Set the debugging level.
if verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
pngprefix = 'qapsf-{}-{}'.format(expnum, ccdname)
# Gather all the info we need about this CCD.
decals = Decals()
ccd = decals.find_ccds(expnum=expnum, ccdname=ccdname)[0]
band = ccd.filter
ps1band = dict(g=0, r=1, i=2, z=3, Y=4)
print('Band {}'.format(band))
#scales = dict(g=0.0066, r=0.01, z=0.025)
#vmin, vmax = np.arcsinh(-1), np.arcsinh(100)
#print(scales[band])
im = decals.get_image_object(ccd)
iminfo = im.get_image_info()
H, W = iminfo['dims']
wcs = im.get_wcs()
# Choose a uniformly selected subset of PS1 stars on this CCD.
ps1 = ps1cat(ccdwcs=wcs)
cat = ps1.get_stars(band=band, magrange=magrange)
rand = np.random.RandomState(seed=expnum * ccd.ccdnum)
these = rand.choice(len(cat) - 1, nstar, replace=False)
#these = rand.random_integers(0,len(cat)-1,nstar)
cat = cat[these]
cat = cat[np.argsort(cat.median[:, ps1band[band]])] # sort by magnitude
#print(cat.nmag_ok)
get_tim_kwargs = dict(const2psf=True, splinesky=splinesky)
# Make a QAplot of the positions of all the stars.
tim = im.get_tractor_image(**get_tim_kwargs)
img = tim.getImage()
#img = tim.getImage()/scales[band]
fig = plt.figure(figsize=(5, 10))
ax = fig.gca()
ax.get_xaxis().get_major_formatter().set_useOffset(False)
#ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest',
# origin='lower',vmin=vmax,vmax=vmax)
ax.imshow(img, **tim.ima)
ax.axis('off')
ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.format(
band, expnum, ccdname, ccd.airmass, ccd.seeing))
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ax.text(xpos,
ypos,
'{:2d}'.format(istar + 1),
color='red',
horizontalalignment='left')
circ = plt.Circle((xpos, ypos), radius=30, color='g', fill=False, lw=1)
ax.add_patch(circ)
#radec = wcs.radec_bounds()
#ax.scatter(cat.ra,cat.dec)
#ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998])
#ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015])
#ax.set_xlabel('$RA\ (deg)$',fontsize=18)
#ax.set_ylabel('$Dec\ (deg)$',fontsize=18)
fig.savefig(pngprefix + '-ccd.png', bbox_inches='tight')
# Initialize the many-stamp QAplot
ncols = 3
nrows = np.ceil(nstar / ncols).astype('int')
inchperstamp = 2.0
fig = plt.figure(figsize=(inchperstamp * 3 * ncols, inchperstamp * nrows))
irow = 0
icol = 0
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
mag = ps1star.median[ps1band[band]] # r-band
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ix, iy = int(xpos), int(ypos)
# create a little tractor Image object around the star
slc = (slice(max(iy - stampsize, 0), min(iy + stampsize + 1, H)),
slice(max(ix - stampsize, 0), min(ix + stampsize + 1, W)))
# The PSF model 'const2Psf' is the one used in DR1: a 2-component
# Gaussian fit to PsfEx instantiated in the image center.
tim = im.get_tractor_image(slc=slc, **get_tim_kwargs)
stamp = tim.getImage()
ivarstamp = tim.getInvvar()
# Initialize a tractor PointSource from PS1 measurements
flux = NanoMaggies.magToNanomaggies(mag)
star = PointSource(RaDecPos(ra, dec), NanoMaggies(**{band: flux}))
# Fit just the source RA,Dec,flux.
tractor = Tractor([tim], [star])
tractor.freezeParam('images')
print('2-component MOG:', tim.psf)
tractor.printThawedParams()
for step in range(50):
dlnp, X, alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_mog = tractor.getModelImage(0)
chi2_mog = -2.0 * tractor.getLogLikelihood()
mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
# Now change the PSF model to a pixelized PSF model from PsfEx instantiated
# at this place in the image.
psf = PixelizedPsfEx(im.psffn)
tim.psf = psf.constantPsfAt(xpos, ypos)
#print('PSF model:', tim.psf)
#tractor.printThawedParams()
for step in range(50):
dlnp, X, alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_psfex = tractor.getModelImage(0)
chi2_psfex = -2.0 * tractor.getLogLikelihood()
mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
#mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95])
sig = np.std((stamp - model_psfex)[ivarstamp > 0])
mn, mx = [-2.0 * sig, 5 * sig]
# Generate a QAplot.
if (istar > 0) and (istar % (ncols) == 0):
irow = irow + 1
icol = 3 * istar - 3 * ncols * irow
#print(istar, irow, icol, icol+1, icol+2)
ax1 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol),
aspect='equal')
ax1.axis('off')
#ax1.imshow(stamp, **tim.ima)
ax1.imshow(stamp,
cmap='gray',
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
ax1.text(0.1,
0.9,
'{:2d}'.format(istar + 1),
color='white',
horizontalalignment='left',
verticalalignment='top',
transform=ax1.transAxes)
ax2 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 1),
aspect='equal')
ax2.axis('off')
#ax2.imshow(stamp-model_mog, **tim.ima)
ax2.imshow(stamp - model_mog,
cmap='gray',
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
ax2.text(0.1,
0.9,
'MoG',
color='white',
horizontalalignment='left',
verticalalignment='top',
transform=ax2.transAxes)
ax2.text(0.08,
0.08,
'{:.3f}'.format(mag_mog),
color='white',
horizontalalignment='left',
verticalalignment='bottom',
transform=ax2.transAxes)
#ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14)
#ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex))
ax3 = plt.subplot2grid((nrows, 3 * ncols), (irow, icol + 2),
aspect='equal')
ax3.axis('off')
#ax3.imshow(stamp-model_psfex, **tim.ima)
ax3.imshow(stamp - model_psfex,
cmap='gray',
interpolation='nearest',
origin='lower',
vmin=mn,
vmax=mx)
ax3.text(0.1,
0.9,
'PSFEx',
color='white',
horizontalalignment='left',
verticalalignment='top',
transform=ax3.transAxes)
ax3.text(0.08,
0.08,
'{:.3f}'.format(mag_psfex),
color='white',
horizontalalignment='left',
verticalalignment='bottom',
transform=ax3.transAxes)
if istar == (nstar - 1):
break
fig.savefig(pngprefix + '-stargrid.png', bbox_inches='tight')
from glob import glob
from legacypipe.common import Decals, wcs_for_brick
from map.utils import tiles_touching_wcs
from map.views import map_decals_dr2
class duck(object):
pass
req = duck()
req.META = dict(HTTP_IF_MODIFIED_SINCE=None)
if __name__ == '__main__':
decals = Decals()
fns = glob('dr2p/coadd/*/*/*-image.jpg')
fns.sort()
bricknames = []
for fn in fns:
print 'File', fn
(h,t) = os.path.split(fn)
(h,t) = os.path.split(h)
print 'Brick:', t
brickname = t
bricknames.append(brickname)
for brickname in bricknames:
scaledfns = glob('data/scaled/decals-dr2/*/%s/image-%s-*.fits' %
(brickname[:3], brickname))
def main():
decals = Decals()
ccds = decals.get_ccds()
print(len(ccds), 'CCDs')
expnums = np.unique(ccds.expnum)
print(len(expnums), 'unique exposures')
for expnum in expnums:
expnumstr = '%08i' % expnum
skyoutfn = os.path.join('splinesky', expnumstr[:5],
'decam-%s.fits' % expnumstr)
psfoutfn = os.path.join('psfex', expnumstr[:5],
'decam-%s.fits' % expnumstr)
if os.path.exists(skyoutfn) and os.path.exists(psfoutfn):
print('Exposure', expnum, 'is done already')
continue
C = ccds[ccds.expnum == expnum]
print(len(C), 'CCDs in expnum', expnum)
psfex = []
psfhdrvals = []
splinesky = []
skyhdrvals = []
for ccd in C:
im = decals.get_image_object(ccd)
fn = im.splineskyfn
if os.path.exists(fn):
T = fits_table(fn)
splinesky.append(T)
# print(fn)
# T.about()
hdr = fitsio.read_header(fn)
skyhdrvals.append(
[hdr[k] for k in ['SKY', 'LEGPIPEV', 'PLVER']] +
[expnum, ccd.ccdname])
else:
print('File not found:', fn)
fn = im.psffn
if os.path.exists(fn):
T = fits_table(fn)
hdr = fitsio.read_header(fn, ext=1)
keys = [
'LOADED',
'ACCEPTED',
'CHI2',
'POLNAXIS',
'POLNGRP',
'PSF_FWHM',
'PSF_SAMP',
'PSFNAXIS',
'PSFAXIS1',
'PSFAXIS2',
'PSFAXIS3',
]
if hdr['POLNAXIS'] == 0:
# No polynomials. Fake it.
T.polgrp1 = np.array([0])
T.polgrp2 = np.array([0])
T.polname1 = np.array(['fake'])
T.polname2 = np.array(['fake'])
T.polzero1 = np.array([0])
T.polzero2 = np.array([0])
T.polscal1 = np.array([1])
T.polscal2 = np.array([1])
T.poldeg1 = np.array([0])
T.poldeg2 = np.array([0])
else:
keys.extend([
'POLGRP1', 'POLNAME1', 'POLZERO1', 'POLSCAL1',
'POLGRP2', 'POLNAME2', 'POLZERO2', 'POLSCAL2',
'POLDEG1'
])
for k in keys:
T.set(k.lower(), np.array([hdr[k]]))
psfex.append(T)
#print(fn)
#T.about()
hdr = fitsio.read_header(fn)
psfhdrvals.append(
[hdr.get(k, '')
for k in ['LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname])
else:
print('File not found:', fn)
if len(psfex):
padded = pad_arrays([p.psf_mask[0] for p in psfex])
cols = psfex[0].columns()
cols.remove('psf_mask')
T = merge_tables(psfex, columns=cols)
T.psf_mask = np.concatenate([[p] for p in padded])
T.legpipev = np.array([h[0] for h in psfhdrvals])
T.plver = np.array([h[1] for h in psfhdrvals])
T.expnum = np.array([h[2] for h in psfhdrvals])
T.ccdname = np.array([h[3] for h in psfhdrvals])
fn = psfoutfn
trymakedirs(fn, dir=True)
T.writeto(fn)
print('Wrote', fn)
if len(splinesky):
T = fits_table()
T.gridw = np.array([t.gridvals[0].shape[1] for t in splinesky])
T.gridh = np.array([t.gridvals[0].shape[0] for t in splinesky])
padded = pad_arrays([t.gridvals[0] for t in splinesky])
T.gridvals = np.concatenate([[p] for p in padded])
padded = pad_arrays([t.xgrid[0] for t in splinesky])
T.xgrid = np.concatenate([[p] for p in padded])
padded = pad_arrays([t.xgrid[0] for t in splinesky])
T.ygrid = np.concatenate([[p] for p in padded])
cols = splinesky[0].columns()
print('Columns:', cols)
for c in ['gridvals', 'xgrid', 'ygrid']:
cols.remove(c)
T.add_columns_from(merge_tables(splinesky, columns=cols))
T.skyclass = np.array([h[0] for h in skyhdrvals])
T.legpipev = np.array([h[1] for h in skyhdrvals])
T.plver = np.array([h[2] for h in skyhdrvals])
T.expnum = np.array([h[3] for h in skyhdrvals])
T.ccdname = np.array([h[4] for h in skyhdrvals])
fn = skyoutfn
trymakedirs(fn, dir=True)
T.writeto(fn)
print('Wrote', fn)
def psf_residuals(expnum,ccdname,stampsize=35,nstar=30,
magrange=(13,17),verbose=0):
# Set the debugging level.
if verbose==0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl,format='%(message)s',stream=sys.stdout)
pngprefix = 'qapsf-{}-{}'.format(expnum,ccdname)
# Gather all the info we need about this CCD.
decals = Decals()
ccd = decals.find_ccds(expnum=expnum,ccdname=ccdname)[0]
band = ccd.filter
ps1band = dict(g=0,r=1,i=2,z=3,Y=4)
print('Band {}'.format(band))
#scales = dict(g=0.0066, r=0.01, z=0.025)
#vmin, vmax = np.arcsinh(-1), np.arcsinh(100)
#print(scales[band])
im = DecamImage(decals,ccd)
iminfo = im.get_image_info()
H,W = iminfo['dims']
wcs = im.get_wcs()
# Choose a uniformly selected subset of PS1 stars on this CCD.
ps1 = ps1cat(ccdwcs=wcs)
cat = ps1.get_stars(band=band,magrange=magrange)
rand = np.random.RandomState(seed=expnum*ccd.ccdnum)
these = rand.choice(len(cat)-1,nstar,replace=False)
#these = rand.random_integers(0,len(cat)-1,nstar)
cat = cat[these]
cat = cat[np.argsort(cat.median[:,ps1band[band]])] # sort by magnitude
#print(cat.nmag_ok)
# Make a QAplot of the positions of all the stars.
tim = im.get_tractor_image(const2psf=True)
img = tim.getImage()
#img = tim.getImage()/scales[band]
fig = plt.figure(figsize=(5,10))
ax = fig.gca()
ax.get_xaxis().get_major_formatter().set_useOffset(False)
#ax.imshow(np.arcsinh(img),cmap='gray',interpolation='nearest',
# origin='lower',vmin=vmax,vmax=vmax)
ax.imshow(img, **tim.ima)
ax.axis('off')
ax.set_title('{}: {}/{} AM={:.2f} Seeing={:.3f}"'.
format(band,expnum,ccdname,ccd.airmass,ccd.seeing))
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ax.text(xpos,ypos,'{:2d}'.format(istar+1),color='red',
horizontalalignment='left')
circ = plt.Circle((xpos,ypos),radius=30,color='g',fill=False,lw=1)
ax.add_patch(circ)
#radec = wcs.radec_bounds()
#ax.scatter(cat.ra,cat.dec)
#ax.set_xlim([radec[1],radec[0]])#*[1.0002,0.9998])
#ax.set_ylim([radec[2],radec[3]])#*[0.985,1.015])
#ax.set_xlabel('$RA\ (deg)$',fontsize=18)
#ax.set_ylabel('$Dec\ (deg)$',fontsize=18)
fig.savefig(pngprefix+'-ccd.png',bbox_inches='tight')
# Initialize the many-stamp QAplot
ncols = 3
nrows = np.ceil(nstar/ncols).astype('int')
inchperstamp = 2.0
fig = plt.figure(figsize=(inchperstamp*3*ncols,inchperstamp*nrows))
irow = 0
icol = 0
for istar, ps1star in enumerate(cat):
ra, dec = (ps1star.ra, ps1star.dec)
mag = ps1star.median[ps1band[band]] # r-band
ok, xpos, ypos = wcs.radec2pixelxy(ra, dec)
ix,iy = int(xpos), int(ypos)
# create a little tractor Image object around the star
slc = (slice(max(iy-stampsize, 0), min(iy+stampsize+1, H)),
slice(max(ix-stampsize, 0), min(ix+stampsize+1, W)))
# The PSF model 'const2Psf' is the one used in DR1: a 2-component
# Gaussian fit to PsfEx instantiated in the image center.
tim = im.get_tractor_image(slc=slc, const2psf=True)
stamp = tim.getImage()
ivarstamp = tim.getInvvar()
# Initialize a tractor PointSource from PS1 measurements
flux = NanoMaggies.magToNanomaggies(mag)
star = PointSource(RaDecPos(ra,dec), NanoMaggies(**{band: flux}))
# Fit just the source RA,Dec,flux.
tractor = Tractor([tim], [star])
tractor.freezeParam('images')
print('2-component MOG:', tim.psf)
tractor.printThawedParams()
for step in range(50):
dlnp,X,alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_mog = tractor.getModelImage(0)
chi2_mog = -2.0*tractor.getLogLikelihood()
mag_mog = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
# Now change the PSF model to a pixelized PSF model from PsfEx instantiated
# at this place in the image.
psf = PixelizedPsfEx(im.psffn)
tim.psf = psf.constantPsfAt(xpos, ypos)
#print('PSF model:', tim.psf)
#tractor.printThawedParams()
for step in range(50):
dlnp,X,alpha = tractor.optimize()
if dlnp < 0.1:
break
print('Fit:', star)
model_psfex = tractor.getModelImage(0)
chi2_psfex = -2.0*tractor.getLogLikelihood()
mag_psfex = NanoMaggies.nanomaggiesToMag(star.brightness)[0]
#mn, mx = np.percentile((stamp-model_psfex)[ivarstamp>0],[1,95])
sig = np.std((stamp-model_psfex)[ivarstamp>0])
mn, mx = [-2.0*sig,5*sig]
# Generate a QAplot.
if (istar>0) and (istar%(ncols)==0):
irow = irow+1
icol = 3*istar - 3*ncols*irow
#print(istar, irow, icol, icol+1, icol+2)
ax1 = plt.subplot2grid((nrows,3*ncols), (irow,icol), aspect='equal')
ax1.axis('off')
#ax1.imshow(stamp, **tim.ima)
ax1.imshow(stamp,cmap='gray',interpolation='nearest',
origin='lower',vmin=mn,vmax=mx)
ax1.text(0.1,0.9,'{:2d}'.format(istar+1),color='white',
horizontalalignment='left',verticalalignment='top',
transform=ax1.transAxes)
ax2 = plt.subplot2grid((nrows,3*ncols), (irow,icol+1), aspect='equal')
ax2.axis('off')
#ax2.imshow(stamp-model_mog, **tim.ima)
ax2.imshow(stamp-model_mog,cmap='gray',interpolation='nearest',
origin='lower',vmin=mn,vmax=mx)
ax2.text(0.1,0.9,'MoG',color='white',
horizontalalignment='left',verticalalignment='top',
transform=ax2.transAxes)
ax2.text(0.08,0.08,'{:.3f}'.format(mag_mog),color='white',
horizontalalignment='left',verticalalignment='bottom',
transform=ax2.transAxes)
#ax2.set_title('{:.3f}, {:.2f}'.format(mag_psfex,chi2_psfex),fontsize=14)
#ax2.set_title('{:.3f}, $\chi^{2}$={:.2f}'.format(mag_psfex,chi2_psfex))
ax3 = plt.subplot2grid((nrows,3*ncols), (irow,icol+2), aspect='equal')
ax3.axis('off')
#ax3.imshow(stamp-model_psfex, **tim.ima)
ax3.imshow(stamp-model_psfex,cmap='gray',interpolation='nearest',
origin='lower',vmin=mn,vmax=mx)
ax3.text(0.1,0.9,'PSFEx',color='white',
horizontalalignment='left',verticalalignment='top',
transform=ax3.transAxes)
ax3.text(0.08,0.08,'{:.3f}'.format(mag_psfex),color='white',
horizontalalignment='left',verticalalignment='bottom',
transform=ax3.transAxes)
if istar==(nstar-1):
break
fig.savefig(pngprefix+'-stargrid.png',bbox_inches='tight')
def main(outfn='ccds-annotated.fits', ccds=None):
decals = Decals()
if ccds is None:
ccds = decals.get_ccds()
# File from the "observing" svn repo:
# https://desi.lbl.gov/svn/decam/code/observing/trunk
tiles = fits_table('decam-tiles_obstatus.fits')
#ccds.cut(np.arange(100))
#print("HACK!")
#ccds.cut(np.array([name in ['N15', 'N16', 'N21', 'N9']
# for name in ccds.ccdname]) *
# ccds.expnum == 229683)
I = decals.photometric_ccds(ccds)
ccds.photometric = np.zeros(len(ccds), bool)
ccds.photometric[I] = True
I = decals.apply_blacklist(ccds)
ccds.blacklist_ok = np.zeros(len(ccds), bool)
ccds.blacklist_ok[I] = True
ccds.good_region = np.empty((len(ccds), 4), np.int16)
ccds.good_region[:,:] = -1
ccds.ra0 = np.zeros(len(ccds), np.float64)
ccds.dec0 = np.zeros(len(ccds), np.float64)
ccds.ra1 = np.zeros(len(ccds), np.float64)
ccds.dec1 = np.zeros(len(ccds), np.float64)
ccds.ra2 = np.zeros(len(ccds), np.float64)
ccds.dec2 = np.zeros(len(ccds), np.float64)
ccds.ra3 = np.zeros(len(ccds), np.float64)
ccds.dec3 = np.zeros(len(ccds), np.float64)
ccds.dra = np.zeros(len(ccds), np.float32)
ccds.ddec = np.zeros(len(ccds), np.float32)
ccds.ra_center = np.zeros(len(ccds), np.float64)
ccds.dec_center = np.zeros(len(ccds), np.float64)
ccds.sig1 = np.zeros(len(ccds), np.float32)
ccds.meansky = np.zeros(len(ccds), np.float32)
ccds.stdsky = np.zeros(len(ccds), np.float32)
ccds.maxsky = np.zeros(len(ccds), np.float32)
ccds.minsky = np.zeros(len(ccds), np.float32)
ccds.pixscale_mean = np.zeros(len(ccds), np.float32)
ccds.pixscale_std = np.zeros(len(ccds), np.float32)
ccds.pixscale_max = np.zeros(len(ccds), np.float32)
ccds.pixscale_min = np.zeros(len(ccds), np.float32)
ccds.psfnorm_mean = np.zeros(len(ccds), np.float32)
ccds.psfnorm_std = np.zeros(len(ccds), np.float32)
ccds.galnorm_mean = np.zeros(len(ccds), np.float32)
ccds.galnorm_std = np.zeros(len(ccds), np.float32)
gaussgalnorm = np.zeros(len(ccds), np.float32)
# 2nd moments
ccds.psf_mx2 = np.zeros(len(ccds), np.float32)
ccds.psf_my2 = np.zeros(len(ccds), np.float32)
ccds.psf_mxy = np.zeros(len(ccds), np.float32)
#
ccds.psf_a = np.zeros(len(ccds), np.float32)
ccds.psf_b = np.zeros(len(ccds), np.float32)
ccds.psf_theta = np.zeros(len(ccds), np.float32)
ccds.psf_ell = np.zeros(len(ccds), np.float32)
ccds.humidity = np.zeros(len(ccds), np.float32)
ccds.outtemp = np.zeros(len(ccds), np.float32)
ccds.tileid = np.zeros(len(ccds), np.int32)
ccds.tilepass = np.zeros(len(ccds), np.uint8)
ccds.tileebv = np.zeros(len(ccds), np.float32)
plvers = []
for iccd,ccd in enumerate(ccds):
im = decals.get_image_object(ccd)
print('Reading CCD %i of %i:' % (iccd+1, len(ccds)), im)
X = im.get_good_image_subregion()
for i,x in enumerate(X):
if x is not None:
ccds.good_region[iccd,i] = x
W,H = ccd.width, ccd.height
psf = None
wcs = None
sky = None
try:
tim = im.get_tractor_image(pixPsf=True, splinesky=True,
subsky=False, pixels=False)
except:
import traceback
traceback.print_exc()
plvers.append('')
continue
if tim is None:
plvers.append('')
continue
psf = tim.psf
wcs = tim.wcs.wcs
sky = tim.sky
hdr = tim.primhdr
# print('Got PSF', psf)
# print('Got sky', type(sky))
# print('Got WCS', wcs)
ccds.humidity[iccd] = hdr.get('HUMIDITY')
ccds.outtemp[iccd] = hdr.get('OUTTEMP')
ccds.sig1[iccd] = tim.sig1
plvers.append(tim.plver)
obj = hdr.get('OBJECT')
# parse 'DECaLS_15150_r'
words = obj.split('_')
tile = None
if len(words) == 3 and words[0] == 'DECaLS':
try:
tileid = int(words[1])
tile = tiles[tileid - 1]
if tile.tileid != tileid:
I = np.flatnonzero(tile.tileid == tileid)
tile = tiles[I[0]]
except:
pass
if tile is not None:
ccds.tileid [iccd] = tile.tileid
ccds.tilepass[iccd] = tile.get('pass')
ccds.tileebv [iccd] = tile.ebv_med
# Instantiate PSF on a grid
S = 32
xx = np.linspace(1+S, W-S, 5)
yy = np.linspace(1+S, H-S, 5)
xx,yy = np.meshgrid(xx, yy)
psfnorms = []
galnorms = []
for x,y in zip(xx.ravel(), yy.ravel()):
p = im.psf_norm(tim, x=x, y=y)
g = im.galaxy_norm(tim, x=x, y=y)
psfnorms.append(p)
galnorms.append(g)
ccds.psfnorm_mean[iccd] = np.mean(psfnorms)
ccds.psfnorm_std [iccd] = np.std (psfnorms)
ccds.galnorm_mean[iccd] = np.mean(galnorms)
ccds.galnorm_std [iccd] = np.std (galnorms)
# PSF in center of field
cx,cy = (W+1)/2., (H+1)/2.
p = psf.getPointSourcePatch(cx, cy).patch
ph,pw = p.shape
px,py = np.meshgrid(np.arange(pw), np.arange(ph))
psum = np.sum(p)
# print('psum', psum)
p /= psum
# centroids
cenx = np.sum(p * px)
ceny = np.sum(p * py)
# print('cenx,ceny', cenx,ceny)
# second moments
x2 = np.sum(p * (px - cenx)**2)
y2 = np.sum(p * (py - ceny)**2)
xy = np.sum(p * (px - cenx)*(py - ceny))
# semi-major/minor axes and position angle
theta = np.rad2deg(np.arctan2(2 * xy, x2 - y2) / 2.)
theta = np.abs(theta) * np.sign(xy)
s = np.sqrt(((x2 - y2)/2.)**2 + xy**2)
a = np.sqrt((x2 + y2) / 2. + s)
b = np.sqrt((x2 + y2) / 2. - s)
ell = 1. - b/a
# print('PSF second moments', x2, y2, xy)
# print('PSF position angle', theta)
# print('PSF semi-axes', a, b)
# print('PSF ellipticity', ell)
ccds.psf_mx2[iccd] = x2
ccds.psf_my2[iccd] = y2
ccds.psf_mxy[iccd] = xy
ccds.psf_a[iccd] = a
ccds.psf_b[iccd] = b
ccds.psf_theta[iccd] = theta
ccds.psf_ell [iccd] = ell
# Galaxy norm using Gaussian approximation of PSF.
realpsf = tim.psf
tim.psf = im.read_psf_model(0, 0, gaussPsf=True,
psf_sigma=tim.psf_sigma)
gaussgalnorm[iccd] = im.galaxy_norm(tim, x=cx, y=cy)
tim.psf = realpsf
# Sky
mod = np.zeros((ccd.height, ccd.width), np.float32)
sky.addTo(mod)
ccds.meansky[iccd] = np.mean(mod)
ccds.stdsky[iccd] = np.std(mod)
ccds.maxsky[iccd] = mod.max()
ccds.minsky[iccd] = mod.min()
# WCS
ccds.ra0[iccd],ccds.dec0[iccd] = wcs.pixelxy2radec(1, 1)
ccds.ra1[iccd],ccds.dec1[iccd] = wcs.pixelxy2radec(1, H)
ccds.ra2[iccd],ccds.dec2[iccd] = wcs.pixelxy2radec(W, H)
ccds.ra3[iccd],ccds.dec3[iccd] = wcs.pixelxy2radec(W, 1)
midx, midy = (W+1)/2., (H+1)/2.
rc,dc = wcs.pixelxy2radec(midx, midy)
ra,dec = wcs.pixelxy2radec([1,W,midx,midx], [midy,midy,1,H])
ccds.dra [iccd] = max(degrees_between(ra, dc+np.zeros_like(ra),
rc, dc))
ccds.ddec[iccd] = max(degrees_between(rc+np.zeros_like(dec), dec,
rc, dc))
ccds.ra_center [iccd] = rc
ccds.dec_center[iccd] = dc
# Compute scale change across the chip
# how many pixels to step
step = 10
xx = np.linspace(1+step, W-step, 5)
yy = np.linspace(1+step, H-step, 5)
xx,yy = np.meshgrid(xx, yy)
pixscale = []
for x,y in zip(xx.ravel(), yy.ravel()):
sx = [x-step, x-step, x+step, x+step, x-step]
sy = [y-step, y+step, y+step, y-step, y-step]
sr,sd = wcs.pixelxy2radec(sx, sy)
rc,dc = wcs.pixelxy2radec(x, y)
# project around a tiny little TAN WCS at (x,y), with 1" pixels
locwcs = Tan(rc, dc, 0., 0., 1./3600, 0., 0., 1./3600, 1., 1.)
ok,lx,ly = locwcs.radec2pixelxy(sr, sd)
#print('local x,y:', lx, ly)
A = polygon_area((lx, ly))
pixscale.append(np.sqrt(A / (2*step)**2))
# print('Pixel scales:', pixscale)
ccds.pixscale_mean[iccd] = np.mean(pixscale)
ccds.pixscale_min[iccd] = min(pixscale)
ccds.pixscale_max[iccd] = max(pixscale)
ccds.pixscale_std[iccd] = np.std(pixscale)
ccds.plver = np.array(plvers)
sfd = tractor.sfd.SFDMap()
allbands = 'ugrizY'
filts = ['%s %s' % ('DES', f) for f in allbands]
wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4']
ebv,ext = sfd.extinction(filts + wisebands, ccds.ra_center,
ccds.dec_center, get_ebv=True)
ext = ext.astype(np.float32)
ccds.ebv = ebv.astype(np.float32)
ccds.decam_extinction = ext[:,:len(allbands)]
ccds.wise_extinction = ext[:,len(allbands):]
# Depth
detsig1 = ccds.sig1 / ccds.psfnorm_mean
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.psfdepth = -2.5 * (np.log10(depth) - 9)
detsig1 = ccds.sig1 / ccds.galnorm_mean
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.galdepth = -2.5 * (np.log10(depth) - 9)
# Depth using Gaussian FWHM.
psf_sigma = ccds.fwhm / 2.35
gnorm = 1./(2. * np.sqrt(np.pi) * psf_sigma)
detsig1 = ccds.sig1 / gnorm
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.gausspsfdepth = -2.5 * (np.log10(depth) - 9)
# Gaussian galaxy depth
detsig1 = ccds.sig1 / gaussgalnorm
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.gaussgaldepth = -2.5 * (np.log10(depth) - 9)
ccds.writeto(outfn)
main(outfn=outfn, ccds=ccds)
except:
import traceback
traceback.print_exc()
if __name__ == '__main__':
TT = [fits_table('ccds-annotated/ccds-annotated-%03i.fits' % i) for i in range(515)]
T = merge_tables(TT)
T.writeto('ccds-annotated.fits')
import sys
sys.exit()
#sys.exit(main())
decals = Decals()
ccds = decals.get_ccds()
from astrometry.util.multiproc import *
#mp = multiproc(8)
mp = multiproc(4)
N = 1000
args = []
i = 0
while len(ccds):
c = ccds[:N]
ccds = ccds[N:]
args.append((i, c))
i += 1
print('Split CCDs file into', len(args), 'pieces')
print('sizes:', [len(c) for i,c in args])
mp.map(_bounce_main, args)
def main(decals=None, opt=None):
'''Driver function for forced photometry of individual DECam images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0, x1, y0, y1) = opt.zoom
zoomslice = (slice(y0, y1), slice(x0, x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.filename)
opt.filename = None
except:
# make this 'None' for decals.find_ccds()
expnum = None
# Try parsing HDU number
try:
opt.hdu = int(opt.hdu)
ccdname = None
except:
ccdname = opt.hdu
opt.hdu = -1
if decals is None:
decals = Decals()
if opt.filename is not None and opt.hdu >= 0:
# Read metadata from file
T = exposure_metadata([opt.filename], hdus=[opt.hdu])
print('Metadata:')
T.about()
else:
# Read metadata from decals-ccds.fits table
T = decals.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if opt.hdu >= 0:
T.cut(T.image_hdu == opt.hdu)
print(len(T), 'with HDU', opt.hdu)
if opt.filename is not None:
T.cut(
np.array([f.strip() == opt.filename
for f in T.image_filename]))
print(len(T), 'with filename', opt.filename)
assert (len(T) == 1)
im = decals.get_image_object(T[0])
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True)
print('Got tim:', tim)
if opt.catfn in ['DR1', 'DR2']:
if opt.catalog_path is None:
opt.catalog_path = opt.catfn.lower()
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, decals=decals)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3],
'tractor-%s.fits' % b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) *
(yy >= -margin) * (yy <= (H + margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print('Cut to', len(T), 'on out_of_bounds and left_blob')
TT.append(T)
T = merge_tables(TT)
T._header = TT[0]._header
del TT
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(
np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T
T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T
cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE)
# print('Got cat:', cat)
print('Forced photom...')
opti = None
if opt.ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
opti = CeresOptimizer(BW=B, BH=B)
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
for src in cat:
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
F = fits_table()
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T))
ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x - 1).astype(np.float32)
F.y = (y - 1).astype(np.float32)
if opt.apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack(
[tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
print('Apertures:', apertures, 'pixels')
for rad in apertures:
aper = photutils.CircularAperture(apxy, rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = ap
ap = 1. / (np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux_ivar = ap
if opt.forced:
kwa = {}
if opt.plots is None:
kwa.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True,
fitstats=True,
shared_params=False,
**kwa)
if opt.plots:
(data, mod, ie, chi, roi) = R.ims1[0]
ima = tim.ima
imchi = dict(interpolation='nearest',
origin='lower',
vmin=-5,
vmax=5)
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
F.flux = np.array([
src.getBrightness().getFlux(tim.band) for src in cat
]).astype(np.float32)
F.flux_ivar = R.IV.astype(np.float32)
F.fracflux = R.fitstats.profracflux.astype(np.float32)
F.rchi2 = R.fitstats.prochi2.astype(np.float32)
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, decals.decals_dir)
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
fname = os.path.join(d2, d1, fname)
print('Trimmed filename to', fname)
#version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file'))
version_hdr.add_record(
dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file'))
version_hdr.add_record(
dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext'))
version_hdr.add_record(
dict(name='CAMERA', value='DECam', comment='Dark Energy Camera'))
version_hdr.add_record(
dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num'))
version_hdr.add_record(
dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name'))
version_hdr.add_record(
dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(
dict(name='EXPOSURE',
value='decam-%s-%s' % (im.expnum, im.ccdname),
comment='Name of this image'))
keys = [
'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME'
]
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {
'mjd': 'sec',
'exptime': 'sec',
'flux': 'nanomaggy',
'flux_ivar': '1/nanomaggy^2'
}
columns = F.get_columns()
for i, col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
print('Finished forced phot:', Time() - t0)
return 0
def main():
import optparse
parser = optparse.OptionParser()
parser.add_option('--zoom', '-z', type=int, action='append', default=[],
help='Add zoom level; default 13')
parser.add_option('--threads', type=int, default=1, help='Number of threads')
parser.add_option('--y0', type=int, default=0, help='Start row')
parser.add_option('--y1', type=int, default=None, help='End row (non-inclusive)')
parser.add_option('--x0', type=int, default=None)
parser.add_option('--x1', type=int, default=None)
parser.add_option('-x', type=int)
parser.add_option('-y', type=int)
parser.add_option('--mindec', type=float, default=None, help='Minimum Dec to run')
parser.add_option('--maxdec', type=float, default=None, help='Maximum Dec to run')
parser.add_option('--minra', type=float, default=0., help='Minimum RA to run')
parser.add_option('--maxra', type=float, default=360., help='Maximum RA to run')
parser.add_option('--near', action='store_true', help='Only run tiles near bricks')
parser.add_option('--near-ccds', action='store_true', help='Only run tiles near CCDs')
parser.add_option('--queue', action='store_true', default=False,
help='Print qdo commands')
parser.add_option('--all', action='store_true', help='Render all tiles')
parser.add_option('--ignore', action='store_true', help='Ignore cached tile files',
default=False)
parser.add_option('--top', action='store_true', help='Top levels of the pyramid')
parser.add_option('--kind', default='image')
parser.add_option('--scale', action='store_true', help='Scale images?')
parser.add_option('--coadd', action='store_true', help='Create SDSS coadd images?')
parser.add_option('--grass', action='store_true', help='progress plot')
opt,args = parser.parse_args()
if len(opt.zoom) == 0:
opt.zoom = [13]
mp = multiproc(opt.threads)
if opt.kind == 'sdss':
if opt.maxdec is None:
opt.maxdec = 90
if opt.mindec is None:
opt.mindec = -25
elif opt.kind in ['halpha', 'unwise-neo1']:
if opt.maxdec is None:
opt.maxdec = 90
if opt.mindec is None:
opt.mindec = -90
else:
if opt.maxdec is None:
opt.maxdec = 40
if opt.mindec is None:
opt.mindec = -20
if opt.top:
top_levels(mp, opt)
sys.exit(0)
from legacypipe.common import Decals
decals = Decals()
if opt.near:
if opt.kind == 'sdss':
B = fits_table(os.path.join(settings.DATA_DIR, 'bricks-sdssco.fits'))
else:
B = decals.get_bricks()
print len(B), 'bricks'
if opt.scale:
opt.near_ccds = True
if opt.near_ccds:
if opt.kind == 'sdss':
C = fits_table(os.path.join(settings.DATA_DIR, 'sdss', 'window_flist.fits'),
columns=['rerun','ra','dec', 'run', 'camcol', 'field', 'score'])
C.cut(C.rerun == '301')
C.cut(C.score >= 0.6)
#C.delete_column('rerun')
# SDSS field size
radius = 1.01 * np.hypot(10., 14.)/2. / 60.
ccdsize = radius
print len(C), 'SDSS fields'
else:
C = decals.get_ccds()
print len(C), 'CCDs'
ccdsize = 0.2
if opt.x is not None:
opt.x0 = opt.x
opt.x1 = opt.x + 1
if opt.y is not None:
opt.y0 = opt.y
opt.y1 = opt.y + 1
if opt.coadd and opt.kind == 'sdss':
from legacypipe.common import wcs_for_brick
from map.views import trymakedirs
B = decals.get_bricks()
print len(B), 'bricks'
B.cut((B.dec >= opt.mindec) * (B.dec < opt.maxdec))
print len(B), 'in Dec range'
B.cut((B.ra >= opt.minra) * (B.ra < opt.maxra))
print len(B), 'in RA range'
if opt.queue:
# ~ square-degree tiles
# RA slices
rr = np.arange(opt.minra , opt.maxra +1)
dd = np.arange(opt.mindec, opt.maxdec+1)
for rlo,rhi in zip(rr, rr[1:]):
for dlo,dhi in zip(dd, dd[1:]):
print 'time python render-tiles.py --kind sdss --coadd --minra %f --maxra %f --mindec %f --maxdec %f' % (rlo, rhi, dlo, dhi)
sys.exit(0)
if opt.grass:
basedir = settings.DATA_DIR
codir = os.path.join(basedir, 'coadd', 'sdssco')
rr,dd = [],[]
exist = []
for i,b in enumerate(B):
print 'Brick', b.brickname,
fn = os.path.join(codir, b.brickname[:3], 'sdssco-%s-%s.fits' % (b.brickname, 'r'))
print '-->', fn,
if not os.path.exists(fn):
print
continue
print 'found'
rr.append(b.ra)
dd.append(b.dec)
exist.append(i)
exist = np.array(exist)
B.cut(exist)
B.writeto('bricks-sdssco-exist.fits')
import pylab as plt
plt.clf()
plt.plot(rr, dd, 'k.')
plt.title('SDSS coadd tiles')
plt.savefig('sdss.png')
sys.exit(0)
basedir = settings.DATA_DIR
codir = os.path.join(basedir, 'coadd', 'sdssco')
for b in B:
print 'Brick', b.brickname
wcs = wcs_for_brick(b, W=2400, H=2400, pixscale=0.396)
bands = 'gri'
dirnm = os.path.join(codir, b.brickname[:3])
fns = [os.path.join(dirnm, 'sdssco-%s-%s.fits' % (b.brickname, band))
for band in bands]
hdr = fitsio.FITSHDR()
hdr['SURVEY'] = 'SDSS'
wcs.add_to_header(hdr)
if all([os.path.exists(fn) for fn in fns]):
print 'Already exist'
continue
ims = map_sdss(req, 1, 0, 0, 0, get_images=True, wcs=wcs, ignoreCached=True,
forcescale=0)
if ims is None:
print 'No overlap'
continue
trymakedirs(os.path.join(dirnm, 'xxx'))
for fn,band,im in zip(fns,bands, ims):
fitsio.write(fn, im, header=hdr, clobber=True)
print 'Wrote', fn
# Also write scaled versions
dirnm = os.path.join(basedir, 'scaled', 'sdssco')
scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(brickname).3s', 'sdssco-%(brickname)s-%(band)s.fits')
for im,band in zip(ims,bands):
scalekwargs = dict(band=band, brick=b.brickid, brickname=b.brickname)
imwcs = wcs
for scale in range(1, 7):
print 'Writing scale level', scale
im,imwcs,sfn = get_scaled(scalepat, scalekwargs, scale, None,
wcs=imwcs, img=im, return_data=True)
sys.exit(0)
if opt.scale:
if opt.kind == 'sdss':
C.cut((C.dec >= opt.mindec) * (C.dec < opt.maxdec))
print len(C), 'in Dec range'
C.cut((C.ra >= opt.minra) * (C.ra < opt.maxra))
print len(C), 'in RA range'
from astrometry.sdss import AsTransWrapper, DR9
sdss = DR9(basedir=settings.SDSS_DIR)
sdss.saveUnzippedFiles(settings.SDSS_DIR)
sdss.setFitsioReadBZ2()
if settings.SDSS_PHOTOOBJS:
sdss.useLocalTree(photoObjs=settings.SDSS_PHOTOOBJS,
resolve=settings.SDSS_RESOLVE)
basedir = settings.DATA_DIR
scaledir = 'sdss'
dirnm = os.path.join(basedir, 'scaled', scaledir)
for im in C:
from map.views import _read_sip_wcs
#if im.rerun != '301':
# continue
for band in 'gri':
scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(rerun)s', '%(run)i', '%(camcol)i', 'sdss-%(run)i-%(camcol)i-%(field)i-%(band)s.fits')
tmpsuff = '.tmp%08i' % np.random.randint(100000000)
basefn = sdss.retrieve('frame', im.run, im.camcol, field=im.field,
band=band, rerun=im.rerun, tempsuffix=tmpsuff)
fnargs = dict(band=band, rerun=im.rerun, run=im.run,
camcol=im.camcol, field=im.field)
scaled = 1
fn = get_scaled(scalepat, fnargs, scaled, basefn,
read_base_wcs=read_astrans, read_wcs=_read_sip_wcs)
print 'get_scaled:', fn
return 0
else:
assert(False)
for zoom in opt.zoom:
N = 2**zoom
if opt.y1 is None:
y1 = N
else:
y1 = opt.y1
if opt.x0 is None:
opt.x0 = 0
x1 = opt.x1
if x1 is None:
x1 = N
# Find grid of Ra,Dec tile centers and select the ones near DECaLS bricks.
rr,dd = [],[]
yy = np.arange(opt.y0, y1)
xx = np.arange(opt.x0, x1)
if opt.grass:
import pylab as plt
tileexists = np.zeros((len(yy),len(xx)), bool)
basedir = settings.DATA_DIR
ver = tileversions[opt.kind][-1]
tiledir = os.path.join(basedir, 'tiles', opt.kind, '%i'%ver, '%i'%zoom)
for dirpath,dirnames,filenames in os.walk(tiledir):
# change walk order
dirnames.sort()
if len(filenames) == 0:
continue
print 'Dirpath', dirpath
#print 'Dirnames', dirnames
#print 'Filenames', filenames
# check for symlinks
if False:
fns = []
for fn in filenames:
fullfn = os.path.join(tiledir, dirpath, fn)
if os.path.isfile(fullfn) and not os.path.islink(fullfn):
fns.append(fn)
print len(fns), 'of', len(filenames), 'are files (not symlinks)'
filenames = fns
x = os.path.basename(dirpath)
x = int(x)
#print 'x', x
yy = [int(fn.replace('.jpg','')) for fn in filenames]
#print 'yy', yy
print len(yy), 'tiles'
for y in yy:
tileexists[y - opt.y0, x - opt.x0] = True
plt.clf()
plt.imshow(tileexists, interpolation='nearest', origin='upper',
vmin=0, vmax=1, cmap='gray')
fn = 'exist-%s-z%02i' % (opt.kind, zoom)
plt.savefig(fn+'.png')
fitsio.write(fn+'.fits', tileexists, clobber=True)
print 'Wrote', fn+'.png and', fn+'.fits'
continue
if not opt.all:
for y in yy:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, y)
r,d = wcs.get_center()
dd.append(d)
for x in xx:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, x, 0)
r,d = wcs.get_center()
rr.append(r)
dd = np.array(dd)
rr = np.array(rr)
if len(dd) > 1:
tilesize = max(np.abs(np.diff(dd)))
print 'Tile size:', tilesize
else:
if opt.near_ccds or opt.near:
try:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0+1)
r2,d2 = wcs.get_center()
except:
wcs,W,H,zoomscale,zoom,x,y = get_tile_wcs(zoom, 0, opt.y0-1)
r2,d2 = wcs.get_center()
tilesize = np.abs(dd[0] - d2)
print 'Tile size:', tilesize
else:
tilesize = 180.
I = np.flatnonzero((dd >= opt.mindec) * (dd <= opt.maxdec))
print 'Keeping', len(I), 'Dec points between', opt.mindec, 'and', opt.maxdec
dd = dd[I]
yy = yy[I]
I = np.flatnonzero((rr >= opt.minra) * (rr <= opt.maxra))
print 'Keeping', len(I), 'RA points between', opt.minra, 'and', opt.maxra
rr = rr[I]
xx = xx[I]
print len(rr), 'RA points x', len(dd), 'Dec points'
print 'x tile range:', xx.min(), xx.max(), 'y tile range:', yy.min(), yy.max()
for iy,y in enumerate(yy):
print
print 'Y row', y
if opt.near:
d = dd[iy]
I,J,dist = match_radec(rr, d+np.zeros_like(rr), B.ra, B.dec, 0.25 + tilesize, nearest=True)
if len(I) == 0:
print 'No matches to bricks'
continue
keep = np.zeros(len(rr), bool)
keep[I] = True
print 'Keeping', sum(keep), 'tiles in row', y, 'Dec', d
x = xx[keep]
elif opt.near_ccds:
d = dd[iy]
print 'RA range of tiles:', rr.min(), rr.max()
print 'Dec of tile row:', d
I,J,dist = match_radec(rr, d+np.zeros_like(rr), C.ra, C.dec, ccdsize + tilesize, nearest=True)
if len(I) == 0:
print 'No matches to CCDs'
continue
keep = np.zeros(len(rr), bool)
keep[I] = True
print 'Keeping', sum(keep), 'tiles in row', y, 'Dec', d
x = xx[keep]
else:
x = xx
if opt.queue:
cmd = 'python -u render-tiles.py --zoom %i --y0 %i --y1 %i --kind %s' % (zoom, y, y+1, opt.kind)
if opt.near_ccds:
cmd += ' --near-ccds'
if opt.all:
cmd += ' --all'
if opt.ignore:
cmd += ' --ignore'
print cmd
continue
# if opt.grass:
# for xi in x:
# basedir = settings.DATA_DIR
# ver = tileversions[opt.kind][-1]
# tilefn = os.path.join(basedir, 'tiles', opt.kind,
# '%i/%i/%i/%i.jpg' % (ver, zoom, xi, y))
# print 'Checking for', tilefn
# if os.path.exists(tilefn):
# print 'EXISTS'
# tileexists[yi-opt.y0, xi-opt.x0]
# continue
args = []
for xi in x:
args.append((opt.kind,zoom,xi,y, opt.ignore))
print 'Rendering', len(args), 'tiles in row y =', y
mp.map(_bounce_one_tile, args, chunksize=min(100, max(1, len(args)/opt.threads)))
print 'Rendered', len(args), 'tiles'
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).")
parser.add_argument('--calibs', action='store_true',
help='Output CCDs that need to be calibrated.')
parser.add_argument('--nper', type=int, default=None,
help='Batch N calibs per line')
parser.add_argument('--forced', action='store_true',
help='Output forced-photometry commands')
parser.add_argument('--lsb', action='store_true',
help='Output Low-Surface-Brightness commands')
parser.add_argument('--touching', action='store_true',
help='Cut to only CCDs touching selected bricks')
parser.add_argument('--check', action='store_true',
help='Check which calibrations actually need to run.')
parser.add_argument('--check-coadd', action='store_true',
help='Check which caoadds actually need to run.')
parser.add_argument('--out', help='Output filename for calibs, default %(default)s',
default='jobs')
parser.add_argument('--command', action='store_true',
help='Write out full command-line to run calib')
parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
parser.add_argument('--mindec', type=float, help='Minimum Dec to run')
parser.add_argument('--region', help='Region to select')
parser.add_argument('--bricks', help='Set bricks.fits file to load')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--delete-sky', action='store_true',
help='Delete any existing sky calibration files')
parser.add_argument('--delete-pvastrom', action='store_true',
help='Delete any existing PV WCS calibration files')
parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')
opt = parser.parse_args()
decals = Decals()
if opt.bricks is not None:
B = fits_table(opt.bricks)
log('Read', len(B), 'from', opt.bricks)
else:
B = decals.get_bricks()
if opt.ccds is not None:
T = fits_table(opt.ccds)
log('Read', len(T), 'from', opt.ccds)
else:
T = decals.get_ccds()
log(len(T), 'CCDs')
T.index = np.arange(len(T))
# I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
# plt.clf()
# plt.hist(counts, counts.max()+1)
# plt.savefig('bricks.png')
# B.cut(I[counts >= 9])
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# #plt.scatter(B.ra[I], B.dec[I], c=counts)
# plt.savefig('bricks2.png')
# DES Stripe82
#rlo,rhi = 350.,360.
# rlo,rhi = 300., 10.
# dlo,dhi = -6., 4.
# TINY bit
#rlo,rhi = 350.,351.1
#dlo,dhi = 0., 1.1
# EDR+
# 860 bricks
# ~10,000 CCDs
#rlo,rhi = 239,246
#dlo,dhi = 5, 13
# DR1
#rlo,rhi = 0, 360
# part 1
#dlo,dhi = 25, 40
# part 2
#dlo,dhi = 20,25
# part 3
#dlo,dhi = 15,20
# part 4
#dlo,dhi = 10,15
# part 5
#dlo,dhi = 5,10
# the rest
#dlo,dhi = -11, 5
#dlo,dhi = 15,25.5
dlo,dhi = -15, 40
rlo,rhi = 0, 360
# Arjun says 3x3 coverage area is roughly
# RA=240-252 DEC=6-12 (but not completely rectangular)
# COSMOS
#rlo,rhi = 148.9, 151.2
#dlo,dhi = 0.9, 3.5
# A nice well-behaved region (EDR2/3)
# rlo,rhi = 243.6, 244.6
# dlo,dhi = 8.1, 8.6
# 56 bricks, ~725 CCDs
#B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
# 240 bricks, ~3000 CCDs
#B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
# 535 bricks, ~7000 CCDs
#B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))
if opt.region in ['test1', 'test2', 'test3', 'test4']:
nm = dict(test1='2446p115', # weird stuff around bright star
test2='1183p292', # faint sources around bright galaxy
test3='3503p005', # DES
test4='1163p277', # Pollux
)[opt.region]
B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
log('Cut to', len(B), 'bricks')
log(B.ra, B.dec)
dlo,dhi = -90,90
rlo,rhi = 0, 360
elif opt.region == 'badsky':
# A handful of exposures in DR2 with inconsistent sky estimates.
#C = decals.get_ccds()
#log(len(C), 'CCDs')
#C.cut((C.expnum >= 257400) * (C.expnum < 257500))
T.cut(np.array([e in [257460, 257461, 257462, 257463, 257464,
257465, 257466, 257467, 257469, 257472,
257483, 257496]
for e in T.expnum]))
log(len(T), 'CCDs with bad sky')
# CCD radius
radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600.
# Brick radius
radius += np.hypot(0.25, 0.25)/2.
I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05)
keep = np.zeros(len(B), bool)
keep[I] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs with bad sky')
elif opt.region == 'badsky2':
# UGH, missed this one in original 'badsky' definition.
T.cut(T.expnum == 257466)
log(len(T), 'CCDs with bad sky')
# CCD radius
radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600.
# Brick radius
radius += np.hypot(0.25, 0.25)/2.
I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05)
keep = np.zeros(len(B), bool)
keep[I] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs with bad sky')
elif opt.region == 'edr':
# EDR:
# 535 bricks, ~7000 CCDs
rlo,rhi = 240,245
dlo,dhi = 5, 12
elif opt.region == 'edr-south':
rlo,rhi = 240,245
dlo,dhi = 5, 10
elif opt.region == 'cosmos1':
# 16 bricks in the core of the COSMOS field.
rlo,rhi = 149.75, 150.75
dlo,dhi = 1.6, 2.6
elif opt.region == 'pristine':
# Stream?
rlo,rhi = 240,250
dlo,dhi = 10,15
elif opt.region == 'des':
dlo, dhi = -6., 4.
rlo, rhi = 317., 7.
T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
log('Cut to', len(T), 'CCDs with "CPDES82" in filename')
elif opt.region == 'subdes':
rlo,rhi = 320., 360.
dlo,dhi = -1.25, 1.25
elif opt.region == 'northwest':
rlo,rhi = 240,360
dlo,dhi = 20,40
elif opt.region == 'north':
rlo,rhi = 120,240
dlo,dhi = 20,40
elif opt.region == 'northeast':
rlo,rhi = 0,120
dlo,dhi = 20,40
elif opt.region == 'southwest':
rlo,rhi = 240,360
dlo,dhi = -20,0
elif opt.region == 'south':
rlo,rhi = 120,240
dlo,dhi = -20,0
elif opt.region == 'southeast':
rlo,rhi = 0,120
dlo,dhi = -20,0
elif opt.region == 'midwest':
rlo,rhi = 240,360
dlo,dhi = 0,20
elif opt.region == 'middle':
rlo,rhi = 120,240
dlo,dhi = 0,20
elif opt.region == 'mideast':
rlo,rhi = 0,120
dlo,dhi = 0,20
elif opt.region == 'grz':
# Bricks with grz coverage.
# Be sure to use --bricks decals-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
log('Cut to', len(B), 'bricks with grz coverage')
elif opt.region == 'nogrz':
# Bricks without grz coverage.
# Be sure to use --bricks decals-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
log('Cut to', len(B), 'bricks withOUT grz coverage')
elif opt.region == 'deep2':
rlo,rhi = 250,260
dlo,dhi = 30,35
elif opt.region == 'virgo':
rlo,rhi = 185,190
dlo,dhi = 10, 15
elif opt.region == 'virgo2':
rlo,rhi = 182,192
dlo,dhi = 8, 18
elif opt.region == 'lsb':
rlo,rhi = 147.2, 147.8
dlo,dhi = -0.4, 0.4
if opt.mindec is not None:
dlo = opt.mindec
if opt.maxdec is not None:
dhi = opt.maxdec
if rlo < rhi:
B.cut((B.ra >= rlo) * (B.ra <= rhi) *
(B.dec >= dlo) * (B.dec <= dhi))
else: # RA wrap
B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) *
(B.dec >= dlo) * (B.dec <= dhi))
log(len(B), 'bricks in range')
I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, 0.25)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs')
if opt.touching:
keep = np.zeros(len(T), bool)
for j in J:
keep[j] = True
T.cut(keep)
log('Cut to', len(T), 'CCDs near bricks')
# Aside -- how many near DR1=1 CCDs?
if False:
T2 = D.get_ccds()
log(len(T2), 'CCDs')
T2.cut(T2.dr1 == 1)
log(len(T2), 'CCDs marked DR1=1')
log(len(B), 'bricks in range')
I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, 0.25)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B2 = B[keep]
log('Total of', len(B2), 'bricks near CCDs with DR1=1')
for band in 'grz':
Tb = T2[T2.filter == band]
log(len(Tb), 'in filter', band)
I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, 0.25)
good = np.zeros(len(B2), np.uint8)
for i in I:
good[i] = 1
B2.set('has_' + band, good)
B2.writeto('decals-bricks-in-dr1.fits')
sys.exit(0)
# sort by dec decreasing
B.cut(np.argsort(-B.dec))
for b in B:
if opt.check:
fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
if opt.check_coadd:
fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
print(b.brickname)
if not (opt.calibs or opt.forced or opt.lsb):
sys.exit(0)
bands = 'grz'
log('Filters:', np.unique(T.filter))
T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
log('Cut to', len(T), 'CCDs in filters', bands)
if opt.touching:
allI = set()
for b in B:
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
if len(I) == 0:
continue
allI.update(I)
allI = list(allI)
allI.sort()
else:
allI = np.arange(len(T))
if opt.write_ccds:
T[allI].writeto(opt.write_ccds)
log('Wrote', opt.write_ccds)
## Be careful here -- T has been cut; we want to write out T.index.
## 'allI' contains indices into T.
if opt.forced:
log('Writing forced-photometry commands to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
for j,i in enumerate(allI):
expstr = '%08i' % T.expnum[i]
#outdir = os.path.join('forced', expstr[:5], expstr)
#trymakedirs(outdir)
outfn = os.path.join('forced', expstr[:5], expstr,
'decam-%s-%s-forced.fits' %
(expstr, T.ccdname[i]))
imgfn = os.path.join(decals.decals_dir, 'images',
T.image_filename[i].strip())
if (not os.path.exists(imgfn) and
imgfn.endswith('.fz') and
os.path.exists(imgfn[:-3])):
imgfn = imgfn[:-3]
f.write('python legacypipe/forced-photom-decam.py %s %i DR1 %s\n' %
(imgfn, T.cpimage_hdu[i], outfn))
f.close()
log('Wrote', opt.out)
sys.exit(0)
if opt.lsb:
log('Writing LSB commands to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
for j,i in enumerate(allI):
exp = T.expnum[i]
ext = T.ccdname[i].strip()
outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext))
f.close()
log('Wrote', opt.out)
sys.exit(0)
log('Writing calibs to', opt.out)
f = open(opt.out,'w')
log('Total of', len(allI), 'CCDs')
batch = []
def write_batch(f, batch, cmd):
if opt.command:
s = '; '.join(batch)
else:
s = ' '.join(batch)
f.write(s + '\n')
for j,i in enumerate(allI):
if opt.delete_sky or opt.delete_pvastrom:
log(j+1, 'of', len(allI))
im = decals.get_image_object(T[i])
if opt.delete_sky and os.path.exists(im.skyfn):
log(' deleting:', im.skyfn)
os.unlink(im.skyfn)
if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
log(' deleting:', im.pvwcsfn)
os.unlink(im.pvwcsfn)
if opt.check:
log(j+1, 'of', len(allI))
im = decals.get_image_object(T[i])
if not im.run_calibs(im, just_check=True):
log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done')
continue
if opt.command:
s = ('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
(T.expnum[i], T.ccdname[i]))
else:
s = '%i' % T.index[i]
if not opt.nper:
f.write(s + '\n')
else:
batch.append(s)
if len(batch) >= opt.nper:
write_batch(f, batch, opt.command)
batch = []
if opt.check:
f.flush()
if len(batch):
write_batch(f, batch, opt.command)
f.close()
log('Wrote', opt.out)
return 0
def main():
"""Main routine which parses the optional inputs."""
parser = argparse.ArgumentParser(formatter_class=argparse.
ArgumentDefaultsHelpFormatter,
description='DECaLS simulations.')
parser.add_argument('-n', '--nobj', type=long, default=None, metavar='',
help='number of objects to simulate (required input)')
parser.add_argument('-c', '--chunksize', type=long, default=500, metavar='',
help='divide NOBJ into CHUNKSIZE chunks')
parser.add_argument('-b', '--brick', type=str, default='2428p117', metavar='',
help='simulate objects in this brick')
parser.add_argument('-o', '--objtype', type=str, default='STAR', metavar='',
help='object type (STAR, ELG, LRG, QSO, LSB)')
parser.add_argument('-t', '--threads', type=int, default=8, metavar='',
help='number of threads to use when calling The Tractor')
parser.add_argument('-s', '--seed', type=long, default=None, metavar='',
help='random number seed')
parser.add_argument('-z', '--zoom', nargs=4, type=int, metavar='',
help='see runbrick.py; (default is 0 3600 0 3600)')
parser.add_argument('--rmag-range', nargs=2, type=float, default=(18,25), metavar='',
help='r-band magnitude range')
parser.add_argument('--test', action='store_true',
help='run the test code and write out test images')
parser.add_argument('-v', '--verbose', action='store_true',
help='toggle on verbose output')
args = parser.parse_args()
if args.nobj is None:
parser.print_help()
sys.exit(1)
# Set the debugging level
if args.verbose:
lvl = logging.DEBUG
else:
lvl = logging.INFO
logging.basicConfig(format='%(message)s',level=lvl,stream=sys.stdout)
log = logging.getLogger('__name__')
brickname = args.brick
objtype = args.objtype.upper()
lobjtype = objtype.lower()
if objtype=='LRG':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
elif objtype=='LSB':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
elif objtype=='QSO':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
# Determine how many chunks we need
nobj = args.nobj
chunksize = args.chunksize
nchunk = long(np.ceil(nobj/chunksize))
log.info('Object type = {}'.format(objtype))
log.info('Number of objects = {}'.format(nobj))
log.info('Chunksize = {}'.format(chunksize))
log.info('Number of chunks = {}'.format(nchunk))
# Optionally zoom into a portion of the brick
decals = Decals()
brickinfo = decals.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale()
print(W, H, pixscale)
log.info('Brick = {}'.format(brickname))
if args.zoom is not None: # See also runbrick.stage_tims()
(x0,x1,y0,y1) = args.zoom
W = x1-x0
H = y1-y0
brickwcs = brickwcs.get_subimage(x0, y0, W, H)
log.info('Zoom (pixel boundaries) = {}'.format(args.zoom))
targetrd = np.array([brickwcs.pixelxy2radec(x,y) for x,y in
[(1,1),(W,1),(W,H),(1,H),(1,1)]])
radec_center = brickwcs.radec_center()
log.info('RA, Dec center = {}'.format(radec_center))
log.info('Brick = {}'.format(brickname))
if args.seed is not None:
log.info('Random seed = {}'.format(args.seed))
# Pack the input parameters into a meta-data table.
metacat = Table()
metacat['brickname'] = Column([brickname],dtype='S10')
metacat['objtype'] = Column([objtype],dtype='S10')
if args.seed is not None:
metacat['seed'] = Column([args.seed],dtype='i4')
metacat['nobj'] = Column([args.nobj],dtype='i4')
metacat['chunksize'] = Column([args.chunksize],dtype='i2')
metacat['nchunk'] = Column([nchunk],dtype='i2')
#metacat['RA'] = Column([ra_range],dtype='f8')
#metacat['DEC'] = Column([dec_range],dtype='f8')
if args.zoom is None:
metacat['zoom'] = Column([[0,3600,0,3600]],dtype='i4')
else:
metacat['zoom'] = Column([args.zoom],dtype='i4')
metacat['rmag_range'] = Column([args.rmag_range],dtype='f4')
print('Hack!')
decals_sim_dir = './'
metafile = os.path.join(decals_sim_dir,'metacat-'+brickname+'-'+lobjtype+'.fits')
log.info('Writing {}'.format(metafile))
if os.path.isfile(metafile):
os.remove(metafile)
metacat.write(metafile)
# Work in chunks.
for ichunk in range(nchunk):
log.info('Working on chunk {:02d}/{:02d}'.format(ichunk+1,nchunk))
chunksuffix = '{:02d}'.format(ichunk)
# There's probably a smarter way to do this
nobjchunk = np.min((nobj,chunksize))
if (nchunk>1) and (ichunk==(nchunk-1)):
nobjchunk = np.max((nobjchunk,nobj%((nchunk-1)*chunksize)))
# Build and write out the simulated object catalog.
simcat = build_simcat(nobjchunk,brickname,brickwcs,metacat)
simcatfile = os.path.join(decals_sim_dir,'simcat-'+brickname+'-'+
lobjtype+'-'+chunksuffix+'.fits')
log.info('Writing {}'.format(simcatfile))
if os.path.isfile(simcatfile):
os.remove(simcatfile)
simcat.write(simcatfile)
# # Use Tractor to just process the blobs containing the simulated sources.
if args.test:
# Test code
simdecals = SimDecals(metacat=metacat, simcat=simcat, test=True)
ccdinfo = decals.ccds_touching_wcs(brickwcs)
sim = SimImage(simdecals, ccdinfo[19])
tim = sim.get_tractor_image(const2psf=True, radecpoly=targetrd)
#for ii, ccd in enumerate(ccdinfo):
# log.info('Working on CCD {}'.format(ii))
# sim = SimImage(simdecals,ccd)
# tim = sim.get_tractor_image(const2psf=True, radecpoly=targetrd, test=args.test)
else:
simdecals = SimDecals(metacat=metacat,simcat=simcat)
blobxy = zip(simcat['x'],simcat['y'])
run_brick(brickname, decals=simdecals, outdir=decals_sim_dir,
threads=8, zoom=args.zoom, wise=False, sdssInit=False,
forceAll=True, writePickles=False, do_calibs=False,
write_metrics=False, pixPsf=False, blobxy=blobxy,
early_coadds=False, stages=['writecat'])
def main():
"""Main program.
"""
import argparse
parser = argparse.ArgumentParser(
description=
"This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)."
)
parser.add_argument('--calibs',
action='store_true',
help='Output CCDs that need to be calibrated.')
parser.add_argument('--nper',
type=int,
default=None,
help='Batch N calibs per line')
parser.add_argument('--forced',
action='store_true',
help='Output forced-photometry commands')
parser.add_argument('--lsb',
action='store_true',
help='Output Low-Surface-Brightness commands')
parser.add_argument('--touching',
action='store_true',
help='Cut to only CCDs touching selected bricks')
parser.add_argument('--check',
action='store_true',
help='Check which calibrations actually need to run.')
parser.add_argument('--check-coadd',
action='store_true',
help='Check which caoadds actually need to run.')
parser.add_argument('--out',
help='Output filename for calibs, default %(default)s',
default='jobs')
parser.add_argument('--command',
action='store_true',
help='Write out full command-line to run calib')
parser.add_argument('--maxdec', type=float, help='Maximum Dec to run')
parser.add_argument('--mindec', type=float, help='Minimum Dec to run')
parser.add_argument('--region', help='Region to select')
parser.add_argument('--bricks', help='Set bricks.fits file to load')
parser.add_argument('--ccds', help='Set ccds.fits file to load')
parser.add_argument('--delete-sky',
action='store_true',
help='Delete any existing sky calibration files')
parser.add_argument('--delete-pvastrom',
action='store_true',
help='Delete any existing PV WCS calibration files')
parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?')
opt = parser.parse_args()
decals = Decals()
if opt.bricks is not None:
B = fits_table(opt.bricks)
log('Read', len(B), 'from', opt.bricks)
else:
B = decals.get_bricks()
if opt.ccds is not None:
T = fits_table(opt.ccds)
log('Read', len(T), 'from', opt.ccds)
else:
T = decals.get_ccds()
log(len(T), 'CCDs')
T.index = np.arange(len(T))
# I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True)
# plt.clf()
# plt.hist(counts, counts.max()+1)
# plt.savefig('bricks.png')
# B.cut(I[counts >= 9])
# plt.clf()
# plt.plot(B.ra, B.dec, 'b.')
# #plt.scatter(B.ra[I], B.dec[I], c=counts)
# plt.savefig('bricks2.png')
# DES Stripe82
#rlo,rhi = 350.,360.
# rlo,rhi = 300., 10.
# dlo,dhi = -6., 4.
# TINY bit
#rlo,rhi = 350.,351.1
#dlo,dhi = 0., 1.1
# EDR+
# 860 bricks
# ~10,000 CCDs
#rlo,rhi = 239,246
#dlo,dhi = 5, 13
# DR1
#rlo,rhi = 0, 360
# part 1
#dlo,dhi = 25, 40
# part 2
#dlo,dhi = 20,25
# part 3
#dlo,dhi = 15,20
# part 4
#dlo,dhi = 10,15
# part 5
#dlo,dhi = 5,10
# the rest
#dlo,dhi = -11, 5
#dlo,dhi = 15,25.5
dlo, dhi = -15, 40
rlo, rhi = 0, 360
# Arjun says 3x3 coverage area is roughly
# RA=240-252 DEC=6-12 (but not completely rectangular)
# COSMOS
#rlo,rhi = 148.9, 151.2
#dlo,dhi = 0.9, 3.5
# A nice well-behaved region (EDR2/3)
# rlo,rhi = 243.6, 244.6
# dlo,dhi = 8.1, 8.6
# 56 bricks, ~725 CCDs
#B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7))
# 240 bricks, ~3000 CCDs
#B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9))
# 535 bricks, ~7000 CCDs
#B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12))
if opt.region in ['test1', 'test2', 'test3', 'test4']:
nm = dict(
test1='2446p115', # weird stuff around bright star
test2='1183p292', # faint sources around bright galaxy
test3='3503p005', # DES
test4='1163p277', # Pollux
)[opt.region]
B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname])))
log('Cut to', len(B), 'bricks')
log(B.ra, B.dec)
dlo, dhi = -90, 90
rlo, rhi = 0, 360
elif opt.region == 'badsky':
# A handful of exposures in DR2 with inconsistent sky estimates.
#C = decals.get_ccds()
#log(len(C), 'CCDs')
#C.cut((C.expnum >= 257400) * (C.expnum < 257500))
T.cut(
np.array([
e in [
257460, 257461, 257462, 257463, 257464, 257465, 257466,
257467, 257469, 257472, 257483, 257496
] for e in T.expnum
]))
log(len(T), 'CCDs with bad sky')
# CCD radius
radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600.
# Brick radius
radius += np.hypot(0.25, 0.25) / 2.
I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05)
keep = np.zeros(len(B), bool)
keep[I] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs with bad sky')
elif opt.region == 'badsky2':
# UGH, missed this one in original 'badsky' definition.
T.cut(T.expnum == 257466)
log(len(T), 'CCDs with bad sky')
# CCD radius
radius = np.hypot(2048, 4096) / 2. * 0.262 / 3600.
# Brick radius
radius += np.hypot(0.25, 0.25) / 2.
I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, radius * 1.05)
keep = np.zeros(len(B), bool)
keep[I] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs with bad sky')
elif opt.region == 'edr':
# EDR:
# 535 bricks, ~7000 CCDs
rlo, rhi = 240, 245
dlo, dhi = 5, 12
elif opt.region == 'edr-south':
rlo, rhi = 240, 245
dlo, dhi = 5, 10
elif opt.region == 'cosmos1':
# 16 bricks in the core of the COSMOS field.
rlo, rhi = 149.75, 150.75
dlo, dhi = 1.6, 2.6
elif opt.region == 'pristine':
# Stream?
rlo, rhi = 240, 250
dlo, dhi = 10, 15
elif opt.region == 'des':
dlo, dhi = -6., 4.
rlo, rhi = 317., 7.
T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage])))
log('Cut to', len(T), 'CCDs with "CPDES82" in filename')
elif opt.region == 'subdes':
rlo, rhi = 320., 360.
dlo, dhi = -1.25, 1.25
elif opt.region == 'northwest':
rlo, rhi = 240, 360
dlo, dhi = 20, 40
elif opt.region == 'north':
rlo, rhi = 120, 240
dlo, dhi = 20, 40
elif opt.region == 'northeast':
rlo, rhi = 0, 120
dlo, dhi = 20, 40
elif opt.region == 'southwest':
rlo, rhi = 240, 360
dlo, dhi = -20, 0
elif opt.region == 'south':
rlo, rhi = 120, 240
dlo, dhi = -20, 0
elif opt.region == 'southeast':
rlo, rhi = 0, 120
dlo, dhi = -20, 0
elif opt.region == 'midwest':
rlo, rhi = 240, 360
dlo, dhi = 0, 20
elif opt.region == 'middle':
rlo, rhi = 120, 240
dlo, dhi = 0, 20
elif opt.region == 'mideast':
rlo, rhi = 0, 120
dlo, dhi = 0, 20
elif opt.region == 'grz':
# Bricks with grz coverage.
# Be sure to use --bricks decals-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))
log('Cut to', len(B), 'bricks with grz coverage')
elif opt.region == 'nogrz':
# Bricks without grz coverage.
# Be sure to use --bricks decals-bricks-in-dr1.fits
# which has_[grz] columns.
B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)))
log('Cut to', len(B), 'bricks withOUT grz coverage')
elif opt.region == 'deep2':
rlo, rhi = 250, 260
dlo, dhi = 30, 35
elif opt.region == 'virgo':
rlo, rhi = 185, 190
dlo, dhi = 10, 15
elif opt.region == 'virgo2':
rlo, rhi = 182, 192
dlo, dhi = 8, 18
elif opt.region == 'lsb':
rlo, rhi = 147.2, 147.8
dlo, dhi = -0.4, 0.4
if opt.mindec is not None:
dlo = opt.mindec
if opt.maxdec is not None:
dhi = opt.maxdec
if rlo < rhi:
B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi))
else: # RA wrap
B.cut(
np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) *
(B.dec <= dhi))
log(len(B), 'bricks in range')
I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, 0.25)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B.cut(keep)
log('Cut to', len(B), 'bricks near CCDs')
if opt.touching:
keep = np.zeros(len(T), bool)
for j in J:
keep[j] = True
T.cut(keep)
log('Cut to', len(T), 'CCDs near bricks')
# Aside -- how many near DR1=1 CCDs?
if False:
T2 = D.get_ccds()
log(len(T2), 'CCDs')
T2.cut(T2.dr1 == 1)
log(len(T2), 'CCDs marked DR1=1')
log(len(B), 'bricks in range')
I, J, d = match_radec(B.ra, B.dec, T2.ra, T2.dec, 0.25)
keep = np.zeros(len(B), bool)
for i in I:
keep[i] = True
B2 = B[keep]
log('Total of', len(B2), 'bricks near CCDs with DR1=1')
for band in 'grz':
Tb = T2[T2.filter == band]
log(len(Tb), 'in filter', band)
I, J, d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, 0.25)
good = np.zeros(len(B2), np.uint8)
for i in I:
good[i] = 1
B2.set('has_' + band, good)
B2.writeto('decals-bricks-in-dr1.fits')
sys.exit(0)
# sort by dec decreasing
B.cut(np.argsort(-B.dec))
for b in B:
if opt.check:
fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3],
b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
if opt.check_coadd:
fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (
b.brickname[:3], b.brickname, b.brickname)
if os.path.exists(fn):
print('Exists:', fn, file=sys.stderr)
continue
print(b.brickname)
if not (opt.calibs or opt.forced or opt.lsb):
sys.exit(0)
bands = 'grz'
log('Filters:', np.unique(T.filter))
T.cut(np.flatnonzero(np.array([f in bands for f in T.filter])))
log('Cut to', len(T), 'CCDs in filters', bands)
if opt.touching:
allI = set()
for b in B:
wcs = wcs_for_brick(b)
I = ccds_touching_wcs(wcs, T)
log(len(I), 'CCDs for brick', b.brickid,
'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec))
if len(I) == 0:
continue
allI.update(I)
allI = list(allI)
allI.sort()
else:
allI = np.arange(len(T))
if opt.write_ccds:
T[allI].writeto(opt.write_ccds)
log('Wrote', opt.write_ccds)
## Be careful here -- T has been cut; we want to write out T.index.
## 'allI' contains indices into T.
if opt.forced:
log('Writing forced-photometry commands to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
for j, i in enumerate(allI):
expstr = '%08i' % T.expnum[i]
#outdir = os.path.join('forced', expstr[:5], expstr)
#trymakedirs(outdir)
outfn = os.path.join(
'forced', expstr[:5], expstr,
'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i]))
imgfn = os.path.join(decals.decals_dir, 'images',
T.image_filename[i].strip())
if (not os.path.exists(imgfn) and imgfn.endswith('.fz')
and os.path.exists(imgfn[:-3])):
imgfn = imgfn[:-3]
f.write('python legacypipe/forced-photom-decam.py %s %i DR1 %s\n' %
(imgfn, T.cpimage_hdu[i], outfn))
f.close()
log('Wrote', opt.out)
sys.exit(0)
if opt.lsb:
log('Writing LSB commands to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
for j, i in enumerate(allI):
exp = T.expnum[i]
ext = T.ccdname[i].strip()
outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext)
f.write(
'python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n'
% (exp, ext, outfn, exp, ext))
f.close()
log('Wrote', opt.out)
sys.exit(0)
log('Writing calibs to', opt.out)
f = open(opt.out, 'w')
log('Total of', len(allI), 'CCDs')
batch = []
def write_batch(f, batch, cmd):
if opt.command:
s = '; '.join(batch)
else:
s = ' '.join(batch)
f.write(s + '\n')
for j, i in enumerate(allI):
if opt.delete_sky or opt.delete_pvastrom:
log(j + 1, 'of', len(allI))
im = decals.get_image_object(T[i])
if opt.delete_sky and os.path.exists(im.skyfn):
log(' deleting:', im.skyfn)
os.unlink(im.skyfn)
if opt.delete_pvastrom and os.path.exists(im.pvwcsfn):
log(' deleting:', im.pvwcsfn)
os.unlink(im.pvwcsfn)
if opt.check:
log(j + 1, 'of', len(allI))
im = decals.get_image_object(T[i])
if not im.run_calibs(im, just_check=True):
log('Calibs for', im.expnum, im.ccdname, im.calname,
'already done')
continue
if opt.command:
s = ('python legacypipe/run-calib.py --expnum %i --ccdname %s' %
(T.expnum[i], T.ccdname[i]))
else:
s = '%i' % T.index[i]
if not opt.nper:
f.write(s + '\n')
else:
batch.append(s)
if len(batch) >= opt.nper:
write_batch(f, batch, opt.command)
batch = []
if opt.check:
f.flush()
if len(batch):
write_batch(f, batch, opt.command)
f.close()
log('Wrote', opt.out)
return 0
def main():
"""Main routine which parses the optional inputs."""
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description='DECaLS simulations.')
parser.add_argument('-n',
'--nobj',
type=long,
default=None,
metavar='',
help='number of objects to simulate (required input)')
parser.add_argument('-c',
'--chunksize',
type=long,
default=500,
metavar='',
help='divide NOBJ into CHUNKSIZE chunks')
parser.add_argument('-b',
'--brick',
type=str,
default='2428p117',
metavar='',
help='simulate objects in this brick')
parser.add_argument('-o',
'--objtype',
type=str,
default='STAR',
metavar='',
help='object type (STAR, ELG, LRG, QSO, LSB)')
parser.add_argument(
'-t',
'--threads',
type=int,
default=8,
metavar='',
help='number of threads to use when calling The Tractor')
parser.add_argument('-s',
'--seed',
type=long,
default=None,
metavar='',
help='random number seed')
parser.add_argument('-z',
'--zoom',
nargs=4,
type=int,
metavar='',
help='see runbrick.py; (default is 0 3600 0 3600)')
parser.add_argument('--rmag-range',
nargs=2,
type=float,
default=(18, 26),
metavar='',
help='r-band magnitude range')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='toggle on verbose output')
args = parser.parse_args()
if args.nobj is None:
parser.print_help()
sys.exit(1)
# Set the debugging level
if args.verbose:
lvl = logging.DEBUG
else:
lvl = logging.INFO
logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout)
log = logging.getLogger('__name__')
brickname = args.brick
objtype = args.objtype.upper()
lobjtype = objtype.lower()
if objtype == 'LRG':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
elif objtype == 'LSB':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
elif objtype == 'QSO':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
# Determine how many chunks we need
nobj = args.nobj
chunksize = args.chunksize
nchunk = long(np.ceil(nobj / chunksize))
log.info('Object type = {}'.format(objtype))
log.info('Number of objects = {}'.format(nobj))
log.info('Chunksize = {}'.format(chunksize))
log.info('Number of chunks = {}'.format(nchunk))
# Optionally zoom into a portion of the brick
decals = Decals()
brickinfo = decals.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(
), brickwcs.pixel_scale()
print(W, H, pixscale)
log.info('Brick = {}'.format(brickname))
if args.zoom is not None: # See also runbrick.stage_tims()
(x0, x1, y0, y1) = args.zoom
W = x1 - x0
H = y1 - y0
brickwcs = brickwcs.get_subimage(x0, y0, W, H)
log.info('Zoom (pixel boundaries) = {}'.format(args.zoom))
targetrd = np.array([
brickwcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
radec_center = brickwcs.radec_center()
log.info('RA, Dec center = {}'.format(radec_center))
log.info('Brick = {}'.format(brickname))
# Pack the input parameters into a meta-data table.
metacat = Table()
metacat['brickname'] = Column([brickname], dtype='S10')
metacat['objtype'] = Column([objtype], dtype='S10')
metacat['nobj'] = Column([args.nobj], dtype='i4')
metacat['chunksize'] = Column([args.chunksize], dtype='i2')
metacat['nchunk'] = Column([nchunk], dtype='i2')
#metacat['RA'] = Column([ra_range],dtype='f8')
#metacat['DEC'] = Column([dec_range],dtype='f8')
if args.zoom is None:
metacat['zoom'] = Column([[0, 3600, 0, 3600]], dtype='i4')
else:
metacat['zoom'] = Column([args.zoom], dtype='i4')
metacat['rmag_range'] = Column([args.rmag_range], dtype='f4')
# Deal with the random seed
if args.seed is not None:
log.info('Random seed = {}'.format(args.seed))
metacat['seed'] = Column([args.seed], dtype='i4')
rand = np.random.RandomState(args.seed)
seeds = rand.random_integers(0, 2**18, nchunk)
if 'DECALS_SIM_DIR' in os.environ:
decals_sim_dir = os.getenv('DECALS_SIM_DIR')
else:
decals_sim_dir = '.'
if not os.path.exists(os.path.join(decals_sim_dir, brickname)):
os.makedirs(os.path.join(decals_sim_dir, brickname))
metafile = os.path.join(decals_sim_dir, brickname,
'metacat-' + brickname + '-' + lobjtype + '.fits')
log.info('Writing {}'.format(metafile))
if os.path.isfile(metafile):
os.remove(metafile)
metacat.write(metafile)
# Work in chunks.
for ichunk in range(nchunk):
log.info('Working on chunk {:02d}/{:02d}'.format(ichunk + 1, nchunk))
chunksuffix = '{:02d}'.format(ichunk)
# There's probably a smarter way to do this
nobjchunk = np.min((nobj, chunksize))
if (nchunk > 1) and (ichunk == (nchunk - 1)):
nobjchunk = np.max((nobjchunk, nobj % ((nchunk - 1) * chunksize)))
# Build and write out the simulated object catalog.
simcat = build_simcat(nobjchunk, brickname, brickwcs, metacat,
seeds[ichunk])
simcatfile = os.path.join(
decals_sim_dir, brickname, 'simcat-' + brickname + '-' + lobjtype +
'-' + chunksuffix + '.fits')
log.info('Writing {}'.format(simcatfile))
if os.path.isfile(simcatfile):
os.remove(simcatfile)
simcat.write(simcatfile)
# # Use Tractor to just process the blobs containing the simulated sources.
simdecals = SimDecals(metacat=metacat, simcat=simcat)
blobxy = zip(simcat['x'], simcat['y'])
run_brick(brickname,
decals=simdecals,
outdir=os.path.join(decals_sim_dir, brickname),
threads=args.threads,
zoom=args.zoom,
wise=False,
sdssInit=False,
forceAll=True,
writePickles=False,
do_calibs=True,
write_metrics=False,
pixPsf=True,
blobxy=blobxy,
early_coadds=False,
stages=['writecat'],
splinesky=True)
log.info('Cleaning up...')
shutil.move(
os.path.join(decals_sim_dir, brickname, 'tractor', brickname[:3],
'tractor-' + brickname + '.fits'),
os.path.join(
decals_sim_dir, brickname, 'tractor-' + brickname + '-' +
lobjtype + '-' + chunksuffix + '.fits'))
shutil.move(
os.path.join(decals_sim_dir, brickname, 'coadd', brickname[:3],
brickname, 'decals-' + brickname + '-image.jpg'),
os.path.join(
decals_sim_dir, brickname, 'qa-' + brickname + '-' + lobjtype +
'-image-' + chunksuffix + '.jpg'))
shutil.move(
os.path.join(decals_sim_dir, brickname, 'coadd', brickname[:3],
brickname, 'decals-' + brickname + '-resid.jpg'),
os.path.join(
decals_sim_dir, brickname, 'qa-' + brickname + '-' + lobjtype +
'-resid-' + chunksuffix + '.jpg'))
shutil.rmtree(os.path.join(decals_sim_dir, brickname, 'coadd'))
shutil.rmtree(os.path.join(decals_sim_dir, brickname, 'tractor'))
#shutil.rmtree(os.path.join(decals_sim_dir,brickname,'images'))
#shutil.rmtree(os.path.join(decals_sim_dir,brickname,'metrics'))
# Write a log file
log.info('All done!')
def main(outfn='ccds-annotated.fits', ccds=None):
decals = Decals()
if ccds is None:
ccds = decals.get_ccds()
# File from the "observing" svn repo:
# https://desi.lbl.gov/svn/decam/code/observing/trunk
tiles = fits_table('decam-tiles_obstatus.fits')
#ccds.cut(np.arange(100))
#print("HACK!")
#ccds.cut(np.array([name in ['N15', 'N16', 'N21', 'N9']
# for name in ccds.ccdname]) *
# ccds.expnum == 229683)
I = decals.photometric_ccds(ccds)
ccds.photometric = np.zeros(len(ccds), bool)
ccds.photometric[I] = True
I = decals.apply_blacklist(ccds)
ccds.blacklist_ok = np.zeros(len(ccds), bool)
ccds.blacklist_ok[I] = True
ccds.good_region = np.empty((len(ccds), 4), np.int16)
ccds.good_region[:, :] = -1
ccds.ra0 = np.zeros(len(ccds), np.float64)
ccds.dec0 = np.zeros(len(ccds), np.float64)
ccds.ra1 = np.zeros(len(ccds), np.float64)
ccds.dec1 = np.zeros(len(ccds), np.float64)
ccds.ra2 = np.zeros(len(ccds), np.float64)
ccds.dec2 = np.zeros(len(ccds), np.float64)
ccds.ra3 = np.zeros(len(ccds), np.float64)
ccds.dec3 = np.zeros(len(ccds), np.float64)
ccds.dra = np.zeros(len(ccds), np.float32)
ccds.ddec = np.zeros(len(ccds), np.float32)
ccds.ra_center = np.zeros(len(ccds), np.float64)
ccds.dec_center = np.zeros(len(ccds), np.float64)
ccds.sig1 = np.zeros(len(ccds), np.float32)
ccds.meansky = np.zeros(len(ccds), np.float32)
ccds.stdsky = np.zeros(len(ccds), np.float32)
ccds.maxsky = np.zeros(len(ccds), np.float32)
ccds.minsky = np.zeros(len(ccds), np.float32)
ccds.pixscale_mean = np.zeros(len(ccds), np.float32)
ccds.pixscale_std = np.zeros(len(ccds), np.float32)
ccds.pixscale_max = np.zeros(len(ccds), np.float32)
ccds.pixscale_min = np.zeros(len(ccds), np.float32)
ccds.psfnorm_mean = np.zeros(len(ccds), np.float32)
ccds.psfnorm_std = np.zeros(len(ccds), np.float32)
ccds.galnorm_mean = np.zeros(len(ccds), np.float32)
ccds.galnorm_std = np.zeros(len(ccds), np.float32)
gaussgalnorm = np.zeros(len(ccds), np.float32)
# 2nd moments
ccds.psf_mx2 = np.zeros(len(ccds), np.float32)
ccds.psf_my2 = np.zeros(len(ccds), np.float32)
ccds.psf_mxy = np.zeros(len(ccds), np.float32)
#
ccds.psf_a = np.zeros(len(ccds), np.float32)
ccds.psf_b = np.zeros(len(ccds), np.float32)
ccds.psf_theta = np.zeros(len(ccds), np.float32)
ccds.psf_ell = np.zeros(len(ccds), np.float32)
ccds.humidity = np.zeros(len(ccds), np.float32)
ccds.outtemp = np.zeros(len(ccds), np.float32)
ccds.tileid = np.zeros(len(ccds), np.int32)
ccds.tilepass = np.zeros(len(ccds), np.uint8)
ccds.tileebv = np.zeros(len(ccds), np.float32)
plvers = []
for iccd, ccd in enumerate(ccds):
im = decals.get_image_object(ccd)
print('Reading CCD %i of %i:' % (iccd + 1, len(ccds)), im)
X = im.get_good_image_subregion()
for i, x in enumerate(X):
if x is not None:
ccds.good_region[iccd, i] = x
W, H = ccd.width, ccd.height
psf = None
wcs = None
sky = None
try:
tim = im.get_tractor_image(pixPsf=True,
splinesky=True,
subsky=False,
pixels=False)
except:
import traceback
traceback.print_exc()
plvers.append('')
continue
if tim is None:
plvers.append('')
continue
psf = tim.psf
wcs = tim.wcs.wcs
sky = tim.sky
hdr = tim.primhdr
# print('Got PSF', psf)
# print('Got sky', type(sky))
# print('Got WCS', wcs)
ccds.humidity[iccd] = hdr.get('HUMIDITY')
ccds.outtemp[iccd] = hdr.get('OUTTEMP')
ccds.sig1[iccd] = tim.sig1
plvers.append(tim.plver)
obj = hdr.get('OBJECT')
# parse 'DECaLS_15150_r'
words = obj.split('_')
tile = None
if len(words) == 3 and words[0] == 'DECaLS':
try:
tileid = int(words[1])
tile = tiles[tileid - 1]
if tile.tileid != tileid:
I = np.flatnonzero(tile.tileid == tileid)
tile = tiles[I[0]]
except:
pass
if tile is not None:
ccds.tileid[iccd] = tile.tileid
ccds.tilepass[iccd] = tile.get('pass')
ccds.tileebv[iccd] = tile.ebv_med
# Instantiate PSF on a grid
S = 32
xx = np.linspace(1 + S, W - S, 5)
yy = np.linspace(1 + S, H - S, 5)
xx, yy = np.meshgrid(xx, yy)
psfnorms = []
galnorms = []
for x, y in zip(xx.ravel(), yy.ravel()):
p = im.psf_norm(tim, x=x, y=y)
g = im.galaxy_norm(tim, x=x, y=y)
psfnorms.append(p)
galnorms.append(g)
ccds.psfnorm_mean[iccd] = np.mean(psfnorms)
ccds.psfnorm_std[iccd] = np.std(psfnorms)
ccds.galnorm_mean[iccd] = np.mean(galnorms)
ccds.galnorm_std[iccd] = np.std(galnorms)
# PSF in center of field
cx, cy = (W + 1) / 2., (H + 1) / 2.
p = psf.getPointSourcePatch(cx, cy).patch
ph, pw = p.shape
px, py = np.meshgrid(np.arange(pw), np.arange(ph))
psum = np.sum(p)
# print('psum', psum)
p /= psum
# centroids
cenx = np.sum(p * px)
ceny = np.sum(p * py)
# print('cenx,ceny', cenx,ceny)
# second moments
x2 = np.sum(p * (px - cenx)**2)
y2 = np.sum(p * (py - ceny)**2)
xy = np.sum(p * (px - cenx) * (py - ceny))
# semi-major/minor axes and position angle
theta = np.rad2deg(np.arctan2(2 * xy, x2 - y2) / 2.)
theta = np.abs(theta) * np.sign(xy)
s = np.sqrt(((x2 - y2) / 2.)**2 + xy**2)
a = np.sqrt((x2 + y2) / 2. + s)
b = np.sqrt((x2 + y2) / 2. - s)
ell = 1. - b / a
# print('PSF second moments', x2, y2, xy)
# print('PSF position angle', theta)
# print('PSF semi-axes', a, b)
# print('PSF ellipticity', ell)
ccds.psf_mx2[iccd] = x2
ccds.psf_my2[iccd] = y2
ccds.psf_mxy[iccd] = xy
ccds.psf_a[iccd] = a
ccds.psf_b[iccd] = b
ccds.psf_theta[iccd] = theta
ccds.psf_ell[iccd] = ell
# Galaxy norm using Gaussian approximation of PSF.
realpsf = tim.psf
tim.psf = im.read_psf_model(0,
0,
gaussPsf=True,
psf_sigma=tim.psf_sigma)
gaussgalnorm[iccd] = im.galaxy_norm(tim, x=cx, y=cy)
tim.psf = realpsf
# Sky
mod = np.zeros((ccd.height, ccd.width), np.float32)
sky.addTo(mod)
ccds.meansky[iccd] = np.mean(mod)
ccds.stdsky[iccd] = np.std(mod)
ccds.maxsky[iccd] = mod.max()
ccds.minsky[iccd] = mod.min()
# WCS
ccds.ra0[iccd], ccds.dec0[iccd] = wcs.pixelxy2radec(1, 1)
ccds.ra1[iccd], ccds.dec1[iccd] = wcs.pixelxy2radec(1, H)
ccds.ra2[iccd], ccds.dec2[iccd] = wcs.pixelxy2radec(W, H)
ccds.ra3[iccd], ccds.dec3[iccd] = wcs.pixelxy2radec(W, 1)
midx, midy = (W + 1) / 2., (H + 1) / 2.
rc, dc = wcs.pixelxy2radec(midx, midy)
ra, dec = wcs.pixelxy2radec([1, W, midx, midx], [midy, midy, 1, H])
ccds.dra[iccd] = max(
degrees_between(ra, dc + np.zeros_like(ra), rc, dc))
ccds.ddec[iccd] = max(
degrees_between(rc + np.zeros_like(dec), dec, rc, dc))
ccds.ra_center[iccd] = rc
ccds.dec_center[iccd] = dc
# Compute scale change across the chip
# how many pixels to step
step = 10
xx = np.linspace(1 + step, W - step, 5)
yy = np.linspace(1 + step, H - step, 5)
xx, yy = np.meshgrid(xx, yy)
pixscale = []
for x, y in zip(xx.ravel(), yy.ravel()):
sx = [x - step, x - step, x + step, x + step, x - step]
sy = [y - step, y + step, y + step, y - step, y - step]
sr, sd = wcs.pixelxy2radec(sx, sy)
rc, dc = wcs.pixelxy2radec(x, y)
# project around a tiny little TAN WCS at (x,y), with 1" pixels
locwcs = Tan(rc, dc, 0., 0., 1. / 3600, 0., 0., 1. / 3600, 1., 1.)
ok, lx, ly = locwcs.radec2pixelxy(sr, sd)
#print('local x,y:', lx, ly)
A = polygon_area((lx, ly))
pixscale.append(np.sqrt(A / (2 * step)**2))
# print('Pixel scales:', pixscale)
ccds.pixscale_mean[iccd] = np.mean(pixscale)
ccds.pixscale_min[iccd] = min(pixscale)
ccds.pixscale_max[iccd] = max(pixscale)
ccds.pixscale_std[iccd] = np.std(pixscale)
ccds.plver = np.array(plvers)
sfd = tractor.sfd.SFDMap()
allbands = 'ugrizY'
filts = ['%s %s' % ('DES', f) for f in allbands]
wisebands = ['WISE W1', 'WISE W2', 'WISE W3', 'WISE W4']
ebv, ext = sfd.extinction(filts + wisebands,
ccds.ra_center,
ccds.dec_center,
get_ebv=True)
ext = ext.astype(np.float32)
ccds.ebv = ebv.astype(np.float32)
ccds.decam_extinction = ext[:, :len(allbands)]
ccds.wise_extinction = ext[:, len(allbands):]
# Depth
detsig1 = ccds.sig1 / ccds.psfnorm_mean
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.psfdepth = -2.5 * (np.log10(depth) - 9)
detsig1 = ccds.sig1 / ccds.galnorm_mean
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.galdepth = -2.5 * (np.log10(depth) - 9)
# Depth using Gaussian FWHM.
psf_sigma = ccds.fwhm / 2.35
gnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
detsig1 = ccds.sig1 / gnorm
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.gausspsfdepth = -2.5 * (np.log10(depth) - 9)
# Gaussian galaxy depth
detsig1 = ccds.sig1 / gaussgalnorm
depth = 5. * detsig1
# that's flux in nanomaggies -- convert to mag
ccds.gaussgaldepth = -2.5 * (np.log10(depth) - 9)
ccds.writeto(outfn)
if __name__ == '__main__':
TT = [
fits_table('ccds-annotated/ccds-annotated-%03i.fits' % i)
for i in range(515)
]
T = merge_tables(TT)
T.writeto('ccds-annotated.fits')
import sys
sys.exit()
#sys.exit(main())
decals = Decals()
ccds = decals.get_ccds()
from astrometry.util.multiproc import *
#mp = multiproc(8)
mp = multiproc(4)
N = 1000
args = []
i = 0
while len(ccds):
c = ccds[:N]
ccds = ccds[N:]
args.append((i, c))
i += 1
print('Split CCDs file into', len(args), 'pieces')
print('sizes:', [len(c) for i, c in args])
mp.map(_bounce_main, args)
from glob import glob
from legacypipe.common import Decals, wcs_for_brick
from map.utils import tiles_touching_wcs
from map.views import map_decals_dr2
class duck(object):
pass
req = duck()
req.META = dict(HTTP_IF_MODIFIED_SINCE=None)
if __name__ == '__main__':
decals = Decals()
fns = glob('dr2p/coadd/*/*/*-image.jpg')
fns.sort()
bricknames = []
for fn in fns:
print 'File', fn
(h, t) = os.path.split(fn)
(h, t) = os.path.split(h)
print 'Brick:', t
brickname = t
bricknames.append(brickname)
for brickname in bricknames:
scaledfns = glob('data/scaled/decals-dr2/*/%s/image-%s-*.fits' %
(brickname[:3], brickname))
def main():
"""Main routine which parses the optional inputs."""
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description='DECaLS simulations.')
parser.add_argument('-n',
'--nobj',
type=long,
default=None,
metavar='',
help='number of objects to simulate (required input)')
parser.add_argument('-c',
'--chunksize',
type=long,
default=500,
metavar='',
help='divide NOBJ into CHUNKSIZE chunks')
parser.add_argument('-b',
'--brick',
type=str,
default='2428p117',
metavar='',
help='simulate objects in this brick')
parser.add_argument('-o',
'--objtype',
type=str,
default='STAR',
metavar='',
help='object type (STAR, ELG, LRG, QSO, LSB)')
parser.add_argument(
'-t',
'--threads',
type=int,
default=8,
metavar='',
help='number of threads to use when calling The Tractor')
parser.add_argument('-s',
'--seed',
type=long,
default=None,
metavar='',
help='random number seed')
parser.add_argument('-z',
'--zoom',
nargs=4,
type=int,
metavar='',
help='see runbrick.py; (default is 0 3600 0 3600)')
parser.add_argument('--rmag-range',
nargs=2,
type=float,
default=(18, 25),
metavar='',
help='r-band magnitude range')
parser.add_argument('--no-qaplots',
action='store_true',
help='do not generate QAplots')
parser.add_argument('-v',
'--verbose',
dest='verbose',
action='count',
default=0,
help='toggle on verbose output')
args = parser.parse_args()
if args.nobj is None:
parser.print_help()
sys.exit(1)
# Set the debugging level
if args.verbose:
lvl = logging.DEBUG
else:
lvl = logging.INFO
logging.basicConfig(format='%(message)s', level=lvl, stream=sys.stdout)
log = logging.getLogger('__name__')
brickname = args.brick
objtype = args.objtype.upper()
lobjtype = objtype.lower()
if objtype == 'LRG':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
elif objtype == 'LSB':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
elif objtype == 'QSO':
log.warning('{} objtype not yet supported!'.format(objtype))
sys.exit(1)
# Determine how many chunks we need
nobj = args.nobj
chunksize = args.chunksize
nchunk = long(np.ceil(nobj / chunksize))
log.info('Object type = {}'.format(objtype))
log.info('Number of objects = {}'.format(nobj))
log.info('Chunksize = {}'.format(chunksize))
log.info('Number of chunks = {}'.format(nchunk))
# Optionally zoom into a portion of the brick
decals = Decals()
brickinfo = decals.get_brick_by_name(brickname)
brickwcs = wcs_for_brick(brickinfo)
W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(
), brickwcs.pixel_scale()
log.info('Brick = {}'.format(brickname))
if args.zoom is not None: # See also runbrick.stage_tims()
(x0, x1, y0, y1) = args.zoom
W = x1 - x0
H = y1 - y0
brickwcs = brickwcs.get_subimage(x0, y0, W, H)
log.info('Zoom (pixel boundaries) = {}'.format(args.zoom))
radec_center = brickwcs.radec_center()
log.info('RA, Dec center = {}'.format(radec_center))
log.info('Brick = {}'.format(brickname))
if args.seed is not None:
log.info('Random seed = {}'.format(args.seed))
# Pack the input parameters into a meta-data table.
meta = Table()
meta['brickname'] = Column([brickname], dtype='S10')
meta['objtype'] = Column([objtype], dtype='S10')
if args.seed is not None:
meta['seed'] = Column([args.seed], dtype='i4')
meta['nobj'] = Column([args.nobj], dtype='i4')
meta['chunksize'] = Column([args.chunksize], dtype='i2')
meta['nchunk'] = Column([nchunk], dtype='i2')
#meta['RA'] = Column([ra_range],dtype='f8')
#meta['DEC'] = Column([dec_range],dtype='f8')
if args.zoom is None:
meta['zoom'] = Column([[0, 3600, 0, 3600]], dtype='i4')
else:
meta['zoom'] = Column([args.zoom], dtype='i4')
meta['rmag_range'] = Column([args.rmag_range], dtype='f4')
print('Hack!')
decals_sim_dir = './'
metafile = os.path.join(decals_sim_dir,
'metacat-' + brickname + '-' + lobjtype + '.fits')
log.info('Writing {}'.format(metafile))
if os.path.isfile(metafile):
os.remove(metafile)
meta.write(metafile)
# Work in chunks.
for ichunk in range(nchunk):
log.info('Working on chunk {:02d}/{:02d}'.format(ichunk + 1, nchunk))
chunksuffix = '{:02d}'.format(ichunk)
# There's probably a smarter way to do this
if ((ichunk + 1) * chunksize) > nobj:
nobjchunk = (ichunk + 1) * chunksize - nobj
else:
nobjchunk = chunksize
print(nobjchunk)
# Build and write out the simulated object catalog.
simcat = build_simcat(nobjchunk, brickname, brickwcs, meta)
simcatfile = os.path.join(
decals_sim_dir, 'simcat-' + brickname + '-' + lobjtype + '-' +
chunksuffix + '.fits')
log.info('Writing {}'.format(simcatfile))
if os.path.isfile(simcatfile):
os.remove(simcatfile)
simcat.write(simcatfile)
#fitsio.write(simcatfile,simcat)
# Use Tractor to just process the blobs containing the simulated sources.
simdecals = SimDecals(sim_sources=simcat)
# Test code
ccdinfo = decals.ccds_touching_wcs(brickwcs)
sim = SimImage(simdecals, ccdinfo[0])
tim = sim.get_tractor_image(const2psf=True)
def main(decals=None, opt=None):
'''Driver function for forced photometry of individual DECam images.
'''
if opt is None:
parser = get_parser()
opt = parser.parse_args()
Time.add_measurement(MemMeas)
t0 = Time()
if os.path.exists(opt.outfn):
print('Ouput file exists:', opt.outfn)
sys.exit(0)
if not opt.forced:
opt.apphot = True
zoomslice = None
if opt.zoom is not None:
(x0,x1,y0,y1) = opt.zoom
zoomslice = (slice(y0,y1), slice(x0,x1))
ps = None
if opt.plots is not None:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(opt.plots)
# Try parsing filename as exposure number.
try:
expnum = int(opt.filename)
opt.filename = None
except:
# make this 'None' for decals.find_ccds()
expnum = None
# Try parsing HDU number
try:
opt.hdu = int(opt.hdu)
ccdname = None
except:
ccdname = opt.hdu
opt.hdu = -1
if decals is None:
decals = Decals()
if opt.filename is not None and opt.hdu >= 0:
# Read metadata from file
T = exposure_metadata([opt.filename], hdus=[opt.hdu])
print('Metadata:')
T.about()
else:
# Read metadata from decals-ccds.fits table
T = decals.find_ccds(expnum=expnum, ccdname=ccdname)
print(len(T), 'with expnum', expnum, 'and CCDname', ccdname)
if opt.hdu >= 0:
T.cut(T.image_hdu == opt.hdu)
print(len(T), 'with HDU', opt.hdu)
if opt.filename is not None:
T.cut(np.array([f.strip() == opt.filename for f in T.image_filename]))
print(len(T), 'with filename', opt.filename)
assert(len(T) == 1)
im = decals.get_image_object(T[0])
tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True)
print('Got tim:', tim)
if opt.catfn in ['DR1', 'DR2']:
if opt.catalog_path is None:
opt.catalog_path = opt.catfn.lower()
margin = 20
TT = []
chipwcs = tim.subwcs
bricks = bricks_touching_wcs(chipwcs, decals=decals)
for b in bricks:
# there is some overlap with this brick... read the catalog.
fn = os.path.join(opt.catalog_path, 'tractor', b.brickname[:3],
'tractor-%s.fits' % b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn)
ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W,H = chipwcs.get_width(), chipwcs.get_height()
I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) *
(yy >= -margin) * (yy <= (H+margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
# print('Brick_primary:', np.unique(T.brick_primary))
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
T.cut((T.out_of_bounds == False) * (T.left_blob == False))
print('Cut to', len(T), 'on out_of_bounds and left_blob')
TT.append(T)
T = merge_tables(TT)
T._header = TT[0]._header
del TT
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapedev_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I = np.flatnonzero(np.array([((t.type == 'COMP') and
(not np.isfinite(t.shapeexp_r)))
for t in T]))
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
if opt.write_cat:
T.writeto(opt.write_cat)
print('Wrote catalog to', opt.write_cat)
else:
T = fits_table(opt.catfn)
T.shapeexp = np.vstack((T.shapeexp_r, T.shapeexp_e1, T.shapeexp_e2)).T
T.shapedev = np.vstack((T.shapedev_r, T.shapedev_e1, T.shapedev_e2)).T
cat = read_fits_catalog(T, ellipseClass=tractor.ellipses.EllipseE)
# print('Got cat:', cat)
print('Forced photom...')
opti = None
if opt.ceres:
from tractor.ceres_optimizer import CeresOptimizer
B = 8
opti = CeresOptimizer(BW=B, BH=B)
tr = Tractor([tim], cat, optimizer=opti)
tr.freezeParam('images')
for src in cat:
src.freezeAllBut('brightness')
src.getBrightness().freezeAllBut(tim.band)
F = fits_table()
F.brickid = T.brickid
F.brickname = T.brickname
F.objid = T.objid
F.filter = np.array([tim.band] * len(T))
F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T))
F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T))
ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec)
F.x = (x-1).astype(np.float32)
F.y = (y-1).astype(np.float32)
if opt.apphot:
import photutils
img = tim.getImage()
ie = tim.getInvError()
with np.errstate(divide='ignore'):
imsigma = 1. / ie
imsigma[ie == 0] = 0.
apimg = []
apimgerr = []
# Aperture photometry locations
xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T
apxy = xxyy - 1.
apertures = apertures_arcsec / tim.wcs.pixel_scale()
print('Apertures:', apertures, 'pixels')
for rad in apertures:
aper = photutils.CircularAperture(apxy, rad)
p = photutils.aperture_photometry(img, aper, error=imsigma)
apimg.append(p.field('aperture_sum'))
apimgerr.append(p.field('aperture_sum_err'))
ap = np.vstack(apimg).T
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux = ap
ap = 1./(np.vstack(apimgerr).T)**2
ap[np.logical_not(np.isfinite(ap))] = 0.
F.apflux_ivar = ap
if opt.forced:
kwa = {}
if opt.plots is None:
kwa.update(wantims=False)
R = tr.optimize_forced_photometry(variance=True, fitstats=True,
shared_params=False, **kwa)
if opt.plots:
(data,mod,ie,chi,roi) = R.ims1[0]
ima = tim.ima
imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5)
plt.clf()
plt.imshow(data, **ima)
plt.title('Data: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(mod, **ima)
plt.title('Model: %s' % tim.name)
ps.savefig()
plt.clf()
plt.imshow(chi, **imchi)
plt.title('Chi: %s' % tim.name)
ps.savefig()
F.flux = np.array([src.getBrightness().getFlux(tim.band)
for src in cat]).astype(np.float32)
F.flux_ivar = R.IV.astype(np.float32)
F.fracflux = R.fitstats.profracflux.astype(np.float32)
F.rchi2 = R.fitstats.prochi2 .astype(np.float32)
program_name = sys.argv[0]
version_hdr = get_version_header(program_name, decals.decals_dir)
# HACK -- print only two directory names + filename of CPFILE.
fname = os.path.basename(im.imgfn)
d = os.path.dirname(im.imgfn)
d1 = os.path.basename(d)
d = os.path.dirname(d)
d2 = os.path.basename(d)
fname = os.path.join(d2, d1, fname)
print('Trimmed filename to', fname)
#version_hdr.add_record(dict(name='CPFILE', value=im.imgfn, comment='DECam comm.pipeline file'))
version_hdr.add_record(dict(name='CPFILE', value=fname, comment='DECam comm.pipeline file'))
version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='DECam comm.pipeline ext'))
version_hdr.add_record(dict(name='CAMERA', value='DECam', comment='Dark Energy Camera'))
version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='DECam exposure num'))
version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='DECam CCD name'))
version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image'))
version_hdr.add_record(dict(name='EXPOSURE', value='decam-%s-%s' % (im.expnum, im.ccdname), comment='Name of this image'))
keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV',
'INSTRUME']
for key in keys:
if key in tim.primhdr:
version_hdr.add_record(dict(name=key, value=tim.primhdr[key]))
hdr = fitsio.FITSHDR()
units = {'mjd':'sec', 'exptime':'sec', 'flux':'nanomaggy',
'flux_ivar':'1/nanomaggy^2'}
columns = F.get_columns()
for i,col in enumerate(columns):
if col in units:
hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col]))
outdir = os.path.dirname(opt.outfn)
if len(outdir):
trymakedirs(outdir)
fitsio.write(opt.outfn, None, header=version_hdr, clobber=True)
F.writeto(opt.outfn, header=hdr, append=True)
print('Wrote', opt.outfn)
print('Finished forced phot:', Time()-t0)
return 0
import numpy as np
from astrometry.util.plotutils import *
from legacyanalysis.ps1cat import ps1cat
from legacypipe.common import Decals
from tractor import Image, PointSource, PixPos, NanoMaggies, Tractor
ps = PlotSequence('rewcs')
expnum, ccdname = 431109, 'N14'
cat = ps1cat(expnum=expnum, ccdname=ccdname)
stars = cat.get_stars()
print len(stars), 'stars'
decals = Decals()
ccd = decals.find_ccds(expnum=expnum,ccdname=ccdname)[0]
im = decals.get_image_object(ccd)
wcs = im.get_wcs()
tim = im.get_tractor_image(pixPsf=True, splinesky=True)
margin = 15
ok,stars.xx,stars.yy = wcs.radec2pixelxy(stars.ra, stars.dec)
stars.xx -= 1.
stars.yy -= 1.
W,H = wcs.get_width(), wcs.get_height()
stars.ix = np.round(stars.xx).astype(int)
stars.iy = np.round(stars.yy).astype(int)
stars.cut((stars.ix >= margin) * (stars.ix < (W-margin)) *
(stars.iy >= margin) * (stars.iy < (H-margin)))
def main():
decals = Decals()
ccds = decals.get_ccds()
print(len(ccds), 'CCDs')
expnums = np.unique(ccds.expnum)
print(len(expnums), 'unique exposures')
for expnum in expnums:
expnumstr = '%08i' % expnum
skyoutfn = os.path.join('splinesky', expnumstr[:5], 'decam-%s.fits' % expnumstr)
psfoutfn = os.path.join('psfex', expnumstr[:5], 'decam-%s.fits' % expnumstr)
if os.path.exists(skyoutfn) and os.path.exists(psfoutfn):
print('Exposure', expnum, 'is done already')
continue
C = ccds[ccds.expnum == expnum]
print(len(C), 'CCDs in expnum', expnum)
psfex = []
psfhdrvals = []
splinesky = []
skyhdrvals = []
for ccd in C:
im = decals.get_image_object(ccd)
fn = im.splineskyfn
if os.path.exists(fn):
T = fits_table(fn)
splinesky.append(T)
# print(fn)
# T.about()
hdr = fitsio.read_header(fn)
skyhdrvals.append([hdr[k] for k in [
'SKY', 'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname])
else:
print('File not found:', fn)
fn = im.psffn
if os.path.exists(fn):
T = fits_table(fn)
hdr = fitsio.read_header(fn, ext=1)
keys = ['LOADED', 'ACCEPTED', 'CHI2', 'POLNAXIS',
'POLNGRP', 'PSF_FWHM', 'PSF_SAMP', 'PSFNAXIS',
'PSFAXIS1', 'PSFAXIS2', 'PSFAXIS3',]
if hdr['POLNAXIS'] == 0:
# No polynomials. Fake it.
T.polgrp1 = np.array([0])
T.polgrp2 = np.array([0])
T.polname1 = np.array(['fake'])
T.polname2 = np.array(['fake'])
T.polzero1 = np.array([0])
T.polzero2 = np.array([0])
T.polscal1 = np.array([1])
T.polscal2 = np.array([1])
T.poldeg1 = np.array([0])
T.poldeg2 = np.array([0])
else:
keys.extend([
'POLGRP1', 'POLNAME1', 'POLZERO1', 'POLSCAL1',
'POLGRP2', 'POLNAME2', 'POLZERO2', 'POLSCAL2',
'POLDEG1'])
for k in keys:
T.set(k.lower(), np.array([hdr[k]]))
psfex.append(T)
#print(fn)
#T.about()
hdr = fitsio.read_header(fn)
psfhdrvals.append([hdr.get(k,'') for k in [
'LEGPIPEV', 'PLVER']] + [expnum, ccd.ccdname])
else:
print('File not found:', fn)
if len(psfex):
padded = pad_arrays([p.psf_mask[0] for p in psfex])
cols = psfex[0].columns()
cols.remove('psf_mask')
T = merge_tables(psfex, columns=cols)
T.psf_mask = np.concatenate([[p] for p in padded])
T.legpipev = np.array([h[0] for h in psfhdrvals])
T.plver = np.array([h[1] for h in psfhdrvals])
T.expnum = np.array([h[2] for h in psfhdrvals])
T.ccdname = np.array([h[3] for h in psfhdrvals])
fn = psfoutfn
trymakedirs(fn, dir=True)
T.writeto(fn)
print('Wrote', fn)
if len(splinesky):
T = fits_table()
T.gridw = np.array([t.gridvals[0].shape[1] for t in splinesky])
T.gridh = np.array([t.gridvals[0].shape[0] for t in splinesky])
padded = pad_arrays([t.gridvals[0] for t in splinesky])
T.gridvals = np.concatenate([[p] for p in padded])
padded = pad_arrays([t.xgrid[0] for t in splinesky])
T.xgrid = np.concatenate([[p] for p in padded])
padded = pad_arrays([t.xgrid[0] for t in splinesky])
T.ygrid = np.concatenate([[p] for p in padded])
cols = splinesky[0].columns()
print('Columns:', cols)
for c in ['gridvals', 'xgrid', 'ygrid']:
cols.remove(c)
T.add_columns_from(merge_tables(splinesky, columns=cols))
T.skyclass = np.array([h[0] for h in skyhdrvals])
T.legpipev = np.array([h[1] for h in skyhdrvals])
T.plver = np.array([h[2] for h in skyhdrvals])
T.expnum = np.array([h[3] for h in skyhdrvals])
T.ccdname = np.array([h[4] for h in skyhdrvals])
fn = skyoutfn
trymakedirs(fn, dir=True)
T.writeto(fn)
print('Wrote', fn)
