def add_brick_data(T, north):
B = fits_table('survey-bricks.fits.gz')
print('Looking up brick bounds')
ibrick = dict([(n, i) for i, n in enumerate(B.brickname)])
bi = np.array([ibrick[n] for n in T.brickname])
T.brickid = B.brickid[bi]
T.ra1 = B.ra1[bi]
T.ra2 = B.ra2[bi]
T.dec1 = B.dec1[bi]
T.dec2 = B.dec2[bi]
assert (np.all(T.ra2 > T.ra1))
T.area = ((T.ra2 - T.ra1) * (T.dec2 - T.dec1) *
np.cos(np.deg2rad((T.dec1 + T.dec2) / 2.)))
print('Resolving north/south split')
from astrometry.util.starutil_numpy import radectolb
ll, bb = radectolb(T.ra, T.dec)
from desitarget.io import desitarget_resolve_dec
decsplit = desitarget_resolve_dec()
if north:
T.survey_primary = (bb > 0) * (T.dec >= decsplit)
else:
T.survey_primary = np.logical_not((bb > 0) * (T.dec >= decsplit))
print('Looking up in_desi')
from desimodel.io import load_tiles
from desimodel.footprint import is_point_in_desi
desitiles = load_tiles()
T.in_desi = is_point_in_desi(desitiles, T.ra, T.dec)
def ebv(self, ra, dec):
l, b = radectolb(ra, dec)
ebv = np.zeros_like(l)
N = (b >= 0)
for wcs, image, cut in [(self.northwcs, self.north, N),
(self.southwcs, self.south, np.logical_not(N))
]:
# Our WCS routines are mis-named... the SFD WCSes convert
# X,Y <-> L,B.
if sum(cut) == 0:
continue
ok, x, y = wcs.radec2pixelxy(l[cut], b[cut])
#assert(np.all(ok == 0))
H, W = image.shape
assert (np.all(x >= 0.5))
assert (np.all(x <= (W + 0.5)))
assert (np.all(y >= 0.5))
assert (np.all(y <= (H + 0.5)))
ebv[cut] = SFDMap.bilinear_interp_nonzero(image, x - 1., y - 1.)
return ebv
def get_catalog_in_wcs(chipwcs,
catsurvey_north,
catsurvey_south=None,
resolve_dec=None,
margin=20):
TT = []
surveys = [(catsurvey_north, True)]
if catsurvey_south is not None:
surveys.append((catsurvey_south, False))
for catsurvey, north in surveys:
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
if resolve_dec is not None:
from astrometry.util.starutil_numpy import radectolb
bricks.gal_l, bricks.gal_b = radectolb(bricks.ra, bricks.dec)
for b in bricks:
# Skip bricks that are entirely on the wrong side of the resolve line (NGC only)
if resolve_dec is not None and b.gal_b > 0:
if north and b.dec2 <= resolve_dec:
continue
if not (north) and b.dec1 >= resolve_dec:
continue
# there is some overlap with this brick... read the catalog.
fn = catsurvey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn,
columns=[
'ra', 'dec', 'brick_primary', 'type', 'release',
'brickid', 'brickname', 'objid', 'fracdev',
'flux_r', 'shapedev_r', 'shapedev_e1',
'shapedev_e2', 'shapeexp_r', 'shapeexp_e1',
'shapeexp_e2', 'ref_epoch', 'pmra', 'pmdec',
'parallax'
])
if resolve_dec is not None and b.gal_b > 0:
if north:
T.cut(T.dec >= resolve_dec)
print('Cut to', len(T), 'north of the resolve line')
else:
T.cut(T.dec < resolve_dec)
print('Cut to', len(T), 'south of the resolve line')
ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
I, = np.nonzero((xx >= -margin) * (xx <= (W + margin)) *
(yy >= -margin) * (yy <= (H + margin)))
T.cut(I)
print('Cut to', len(T), 'sources within image + margin')
T.cut(T.brick_primary)
print('Cut to', len(T), 'on brick_primary')
for col in ['out_of_bounds', 'left_blob']:
if col in T.get_columns():
T.cut(T.get(col) == False)
print('Cut to', len(T), 'on', col)
# drop DUP sources
I, = np.nonzero([t.strip() != 'DUP' for t in T.type])
T.cut(I)
print('Cut to', len(T), 'after removing DUP')
if len(T):
TT.append(T)
if len(TT) == 0:
return None
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
print('Total of', len(T), 'catalog sources')
# Fix up various failure modes:
# FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0)
# -> convert to EXP
I, = np.nonzero([
t == 'COMP' and not np.isfinite(r)
for t, r in zip(T.type, T.shapedev_r)
])
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to EXP')
for i in I:
T.type[i] = 'EXP'
# Same thing with the exp component.
# -> convert to DEV
I, = np.nonzero([
t == 'COMP' and not np.isfinite(r)
for t, r in zip(T.type, T.shapeexp_r)
])
if len(I):
print('Converting', len(I), 'bogus COMP galaxies to DEV')
for i in I:
T.type[i] = 'DEV'
return T
def get_catalog_in_wcs(chipwcs,
survey,
catsurvey_north,
catsurvey_south=None,
resolve_dec=None,
margin=20):
TT = []
surveys = [(catsurvey_north, True)]
if catsurvey_south is not None:
surveys.append((catsurvey_south, False))
columns = [
'ra',
'dec',
'brick_primary',
'type',
'release',
'brickid',
'brickname',
'objid',
'flux_g',
'flux_r',
'flux_z',
'sersic',
'shape_r',
'shape_e1',
'shape_e2',
'ref_epoch',
'pmra',
'pmdec',
'parallax',
'ref_cat',
'ref_id',
]
for catsurvey, north in surveys:
bricks = bricks_touching_wcs(chipwcs, survey=catsurvey)
if resolve_dec is not None:
from astrometry.util.starutil_numpy import radectolb
bricks.gal_l, bricks.gal_b = radectolb(bricks.ra, bricks.dec)
for b in bricks:
# Skip bricks that are entirely on the wrong side of the resolve line (NGC only)
if resolve_dec is not None:
# Northern survey, brick too far south (max dec is below the resolve line)
if north and b.dec2 <= resolve_dec:
continue
# Southern survey, brick too far north (min dec is above the resolve line), but only in the North Galactic Cap
if not (north) and b.dec1 >= resolve_dec and b.gal_b > 0:
continue
# there is some overlap with this brick... read the catalog.
fn = catsurvey.find_file('tractor', brick=b.brickname)
if not os.path.exists(fn):
print('WARNING: catalog', fn, 'does not exist. Skipping!')
continue
print('Reading', fn)
T = fits_table(fn, columns=columns)
if resolve_dec is not None:
if north:
T.cut(T.dec >= resolve_dec)
print('Cut to', len(T), 'north of the resolve line')
elif b.gal_b > 0:
# Northern galactic cap only: cut Southern survey
T.cut(T.dec < resolve_dec)
print('Cut to', len(T), 'south of the resolve line')
_, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec)
W, H = chipwcs.get_width(), chipwcs.get_height()
# Cut to sources that are inside the image+margin
T.cut((xx >= -margin) * (xx <= (W + margin)) * (yy >= -margin) *
(yy <= (H + margin)))
T.cut(T.brick_primary)
#print('Cut to', len(T), 'on brick_primary')
# drop DUP sources
I, = np.nonzero([t.strip() != 'DUP' for t in T.type])
T.cut(I)
#print('Cut to', len(T), 'after removing DUP')
if len(T):
TT.append(T)
if len(TT) == 0:
return None
T = merge_tables(TT, columns='fillzero')
T._header = TT[0]._header
del TT
SGA = find_missing_sga(T, chipwcs, survey, surveys, columns)
if SGA is not None:
## Add 'em in!
T = merge_tables([T, SGA], columns='fillzero')
print('Total of', len(T), 'catalog sources')
return T
def main():
'''
This function generates the plots in the paper.
Some files and directories are assumed to exist in the current directory:
* WISE atlas tiles, from http://unwise.me/data/allsky-atlas.fits
* unwise-neo1-coadds, from http://unwise.me/data/neo1/
* unwise-neo1-coadds-half, unwise-neo1-coadds-quarter: directories
'''
# First, create the WCS into which we want to render
# degrees width to render in galactic coords
# |l| < 60
# |b| < 30
width = 120
# ~2 arcmin per pixel
W = int(width * 60.) / 2
H = W/2
zoom = 360. / width
wcs = anwcs_create_hammer_aitoff(0., 0., zoom, W, H, 0)
# Select WISE tiles that overlap. This atlas table is available
# from http://unwise.me/data/allsky-atlas.fits
# Select WISE tiles that overlap.
T = fits_table('allsky-atlas.fits')
print(len(T), 'tiles total')
T.ll,T.bb = radectolb(T.ra, T.dec)
I = np.flatnonzero(np.logical_or(T.ll < width+1,
T.ll > (360-width-1)) *
(T.bb > -width/2-1) * (T.bb < width/2+1))
T.cut(I)
print(len(I), 'tiles in L,B range')
# Create a coadd for each WISE band
lbpat = 'unwise-neo1-w%i-lb.fits'
imgs = []
for band in [1,2]:
outfn = lbpat % (band)
if os.path.exists(outfn):
print('Exists:', outfn)
img = fitsio.read(outfn)
imgs.append(img)
continue
coimg = np.zeros((H,W), np.float32)
conimg = np.zeros((H,W), np.float32)
for i,brick in enumerate(T.coadd_id):
# We downsample by 2, twice, just to make repeat runs a
# little faster.
# unWISE
fn = os.path.join('unwise-neo1-coadds', brick[:3], brick,
'unwise-%s-w%i-img-u.fits' % (brick, band))
qfn = os.path.join('unwise-neo1-coadds-quarter',
'unwise-%s-w%i.fits' % (brick, band))
hfn = os.path.join('unwise-neo1-coadds-half',
'unwise-%s-w%i.fits' % (brick, band))
if not os.path.exists(qfn):
if not os.path.exists(hfn):
print('Reading', fn)
halfsize(fn, hfn)
halfsize(hfn, qfn)
fn = qfn
print('Reading', fn)
img = fitsio.read(fn)
bwcs = Tan(fn, 0)
bh,bw = img.shape
# Coadd each unWISE pixel into the nearest target pixel.
xx,yy = np.meshgrid(np.arange(bw), np.arange(bh))
rr,dd = bwcs.pixelxy2radec(xx, yy)
ll,bb = radectolb(rr.ravel(), dd.ravel())
ll = ll.reshape(rr.shape)
bb = bb.reshape(rr.shape)
ok,ox,oy = wcs.radec2pixelxy(ll, bb)
ox = np.round(ox - 1).astype(int)
oy = np.round(oy - 1).astype(int)
K = (ox >= 0) * (ox < W) * (oy >= 0) * (oy < H) * ok
#print('ok:', np.unique(ok), 'x', ox.min(), ox.max(), 'y', oy.min(), oy.max())
assert(np.all(np.isfinite(img)))
if np.sum(K) == 0:
# no overlap
print('No overlap')
continue
np.add.at( coimg, (oy[K], ox[K]), img[K])
np.add.at(conimg, (oy[K], ox[K]), 1)
img = coimg / np.maximum(conimg, 1)
# Hack -- write and then read FITS WCS header.
fn = 'wiselb.wcs'
wcs.writeto(fn)
hdr = fitsio.read_header(fn)
hdr['CTYPE1'] = 'GLON-AIT'
hdr['CTYPE2'] = 'GLAT-AIT'
fitsio.write(outfn, img, header=hdr, clobber=True)
fitsio.write(outfn.replace('.fits', '-n.fits'), conimg,
header=hdr, clobber=True)
imgs.append(img)
w1,w2 = imgs
# Get/confirm L,B bounds...
H,W = w1.shape
print('Image size', W, 'x', H)
ok,l1,b1 = wcs.pixelxy2radec(1, (H+1)/2.)
ok,l2,b2 = wcs.pixelxy2radec(W, (H+1)/2.)
ok,l3,b3 = wcs.pixelxy2radec((W+1)/2., 1)
ok,l4,b4 = wcs.pixelxy2radec((W+1)/2., H)
print('L,B', (l1,b1), (l2,b2), (l3,b3), (l4,b4))
llo,lhi = l2,l1+360
blo,bhi = b3,b4
# Set plot sizes
plt.figure(1, figsize=(10,5))
plt.subplots_adjust(left=0.1, right=0.95, bottom=0.1, top=0.95)
plt.figure(2, figsize=(5,5))
plt.subplots_adjust(left=0.11, right=0.96, bottom=0.1, top=0.95)
suffix = '.pdf'
rgb = wise_rgb(w1, w2)
xlo,ylo = 0,0
plt.figure(1)
plt.clf()
plt.imshow(rgb, origin='lower', interpolation='nearest')
lbticks(wcs, xlo, ylo, lticks=[60,30,0,330,300], bticks=[-30,-15,0,15,30])
plt.savefig('xbulge-00' + suffix)
# Compute the median of each row as a crude way of suppressing the
# Galactic plane
medy1 = np.median(w1, axis=1)
medy2 = np.median(w2, axis=1)
rgb = wise_rgb(w1 - medy1[:,np.newaxis],
w2 - medy2[:,np.newaxis])
# Zoom in a bit for Galactic plane subtracted version
lhi,llo,blo,bhi = 40, 320, -20, 20
okxy = np.array([wcs.radec2pixelxy(l,b) for l,b in [
(llo, blo), (llo, bhi), (lhi, blo), (lhi, bhi)]])
xlo = int(np.floor(min(okxy[:,-2])))
xhi = int(np.ceil (max(okxy[:,-2])))
ylo = int(np.floor(min(okxy[:,-1])))
yhi = int(np.ceil (max(okxy[:,-1])))
plt.clf()
plt.imshow(rgb[ylo:yhi, xlo:xhi, :],origin='lower', interpolation='nearest')
#lbticks(wcs, xlo, ylo, lticks=[40,20,0,340,320], bticks=[-20,-10,0,10,20])
lbticks(wcs, xlo, ylo, lticks=[30,15,0,345,330], bticks=[-20,-10,0,10,20])
plt.savefig('xbulge-01' + suffix)
# Zoom in on the core
lhi,llo,blo,bhi = 15, 345, -15, 15
ok,x1,y1 = wcs.radec2pixelxy(llo, blo)
ok,x2,y2 = wcs.radec2pixelxy(llo, bhi)
ok,x3,y3 = wcs.radec2pixelxy(lhi, blo)
ok,x4,y4 = wcs.radec2pixelxy(lhi, bhi)
xlo = int(np.floor(min(x1,x2,x3,x4)))
xhi = int(np.ceil (max(x1,x2,x3,x4)))
ylo = int(np.floor(min(y1,y2,y3,y4)))
yhi = int(np.ceil (max(y1,y2,y3,y4)))
print('xlo,ylo', xlo, ylo)
w1 = w1[ylo:yhi, xlo:xhi]
w2 = w2[ylo:yhi, xlo:xhi]
plt.figure(2)
# Apply color cut
w1mag = -2.5*(np.log10(w1) - 9.)
w2mag = -2.5*(np.log10(w2) - 9.)
cc = w1mag - w2mag
goodcolor = np.isfinite(cc)
mlo,mhi = np.percentile(cc[goodcolor], [5,95])
print('W1 - W2 color masks:', mlo,mhi)
mask = goodcolor * (cc > mlo) * (cc < mhi)
plt.clf()
rgb = wise_rgb(w1, w2)
plt.imshow(rgb, origin='lower', interpolation='nearest')
lbticks(wcs, xlo,ylo)
plt.title('Data')
plt.savefig('xbulge-fit-data' + suffix)
plt.clf()
rgb = wise_rgb(w1 * mask, w2 * mask)
plt.imshow(rgb, origin='lower', interpolation='nearest')
lbticks(wcs, xlo,ylo)
plt.title('Data (masked)')
plt.savefig('xbulge-fit-masked' + suffix)
ie = mask.astype(np.float32)
from tractor import (Image, NCircularGaussianPSF, LinearPhotoCal, Tractor,
PixPos, Fluxes)
from tractor.galaxy import ExpGalaxy, GalaxyShape
# Create Tractor images
tim1 = Image(data=w1 * mask, inverr=ie,
psf=NCircularGaussianPSF([1.],[1.]),
photocal=LinearPhotoCal(1., 'w1'))
tim2 = Image(data=w2 * mask, inverr=ie,
psf=NCircularGaussianPSF([1.],[1.]),
photocal=LinearPhotoCal(1., 'w2'))
H,W = w1.shape
gal = ExpGalaxy(PixPos(W/2, H/2), Fluxes(w1=w1.sum(), w2=w2.sum()),
GalaxyShape(200, 0.4, 90.))
tractor = Tractor([tim1, tim2],[gal])
# fitsio.write('data-w1.fits', w1 * mask, clobber=True)
# fitsio.write('data-w2.fits', w2 * mask, clobber=True)
# fitsio.write('mask.fits', mask.astype(np.uint8), clobber=True)
# Optimize galaxy model
tractor.freezeParam('images')
for step in range(50):
dlnp,x,alpha = tractor.optimize()
print('dlnp', dlnp)
print('x', x)
print('alpha', alpha)
print('Galaxy', gal)
if dlnp == 0:
break
# Get galaxy model images, compute residuals
mod1 = tractor.getModelImage(0)
resid1 = w1 - mod1
mod2 = tractor.getModelImage(1)
resid2 = w2 - mod2
rgb = wise_rgb(mod1, mod2)
plt.clf()
plt.imshow(rgb, origin='lower', interpolation='nearest')
lbticks(wcs, xlo,ylo)
plt.title('Model')
plt.savefig('xbulge-fit-model' + suffix)
rgb = resid_rgb(resid1, resid2)
plt.clf()
plt.imshow(rgb, origin='lower', interpolation='nearest')
lbticks(wcs, xlo,ylo)
plt.title('Residuals')
plt.savefig('xbulge-fit-resid' + suffix)
rgb = resid_rgb(resid1*mask, resid2*mask)
plt.clf()
plt.imshow(rgb, origin='lower', interpolation='nearest')
lbticks(wcs, xlo,ylo)
plt.title('Residuals (masked)')
plt.savefig('xbulge-fit-residmasked' + suffix)
# fitsio.write('resid1.fits', resid1, clobber=True)
# fitsio.write('resid2.fits', resid2, clobber=True)
# Compute median-smoothed residuals
fr1 = np.zeros_like(resid1)
fr2 = np.zeros_like(resid2)
median_smooth(resid1, np.logical_not(mask), 25, fr1)
median_smooth(resid2, np.logical_not(mask), 25, fr2)
rgb = resid_rgb(fr1, fr2)
plt.clf()
plt.imshow(rgb, origin='lower', interpolation='nearest')
lbticks(wcs, xlo,ylo)
plt.title('Residuals (smoothed)')
plt.savefig('xbulge-fit-smooth2' + suffix)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
'-b',
'--brick',
help='Brick name to run; required unless --radec is given')
parser.add_argument(
'--survey-dir',
type=str,
default=None,
help='Override the $LEGACY_SURVEY_DIR environment variable')
parser.add_argument('-d',
'--outdir',
dest='output_dir',
help='Set output base directory, default "."')
parser.add_argument(
'--out',
help='Output filename -- if not set, defaults to path within --outdir.'
)
parser.add_argument('-r',
'--run',
default=None,
help='Set the run type to execute (for images)')
parser.add_argument(
'--catalog',
help=
'Use the given FITS catalog file, rather than reading from a data release directory'
)
parser.add_argument('--catalog-dir',
help='Set LEGACY_SURVEY_DIR to use to read catalogs')
parser.add_argument(
'--catalog-dir-north',
help='Set LEGACY_SURVEY_DIR to use to read Northern catalogs')
parser.add_argument(
'--catalog-dir-south',
help='Set LEGACY_SURVEY_DIR to use to read Southern catalogs')
parser.add_argument(
'--catalog-resolve-dec-ngc',
type=float,
help=
'Dec at which to switch from Northern to Southern catalogs (NGC only)',
default=32.375)
parser.add_argument('-v',
'--verbose',
dest='verbose',
action='count',
default=0,
help='Make more verbose')
opt = parser.parse_args()
if opt.brick is None:
parser.print_help()
return -1
verbose = opt.verbose
if verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
from legacypipe.runs import get_survey
survey = get_survey(opt.run,
survey_dir=opt.survey_dir,
output_dir=opt.output_dir)
columns = [
'release',
'brickid',
'objid',
]
cat = None
catsurvey = survey
if opt.catalog is not None:
cat = fits_table(opt.catalog, columns=columns)
print('Read', len(cat), 'sources from', opt.catalog)
else:
from astrometry.util.starutil_numpy import radectolb
# The "north" and "south" directories often don't have
# 'survey-bricks" files of their own -- use the 'survey' one
# instead.
brick = None
for s in [survey, catsurvey]:
try:
brick = s.get_brick_by_name(opt.brick)
break
except:
import traceback
traceback.print_exc()
pass
l, b = radectolb(brick.ra, brick.dec)
# NGC and above resolve line? -> north
if b > 0 and brick.dec >= opt.catalog_resolve_dec_ngc:
if opt.catalog_dir_north:
catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir_north)
else:
if opt.catalog_dir_south:
catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir_south)
fn = catsurvey.find_file('tractor', brick=opt.brick)
cat = fits_table(fn, columns=columns)
print('Read', len(cat), 'sources from', fn)
program_name = sys.argv[0]
## FIXME -- from catalog?
release = 9999
version_hdr = get_version_header(program_name, opt.survey_dir, release)
from legacypipe.utils import add_bits
from legacypipe.bits import DQ_BITS
add_bits(version_hdr, DQ_BITS, 'DQMASK', 'DQ', 'D')
from legacyzpts.psfzpt_cuts import CCD_CUT_BITS
add_bits(version_hdr, CCD_CUT_BITS, 'CCD_CUTS', 'CC', 'C')
for i, ap in enumerate(apertures_arcsec):
version_hdr.add_record(
dict(name='APRAD%i' % i,
value=ap,
comment='(optical) Aperture radius, in arcsec'))
cat, forced = merge_forced(survey, opt.brick, cat)
units = []
for i, col in enumerate(forced.get_columns()):
units.append(forced._header.get('TUNIT%i' % (i + 1), ''))
cols = forced.get_columns()
if opt.out:
cat.writeto(opt.out, primheader=version_hdr)
forced.writeto(opt.out, append=True, units=units, columns=cols)
else:
with survey.write_output('forced-brick', brick=opt.brick) as out:
cat.writeto(None, fits_object=out.fits, primheader=version_hdr)
forced.writeto(None,
fits_object=out.fits,
append=True,
units=units,
columns=cols)