def read_large_galaxies(survey, targetwcs):
from astrometry.libkd.spherematch import tree_open, tree_search_radec
galfn = survey.find_file('large-galaxies')
radius = 1.
rc, dc = targetwcs.radec_center()
kd = tree_open(galfn, 'largegals')
I = tree_search_radec(kd, rc, dc, radius)
debug(len(I), 'large galaxies within', radius,
'deg of RA,Dec (%.3f, %.3f)' % (rc, dc))
if len(I) == 0:
return None
# Read only the rows within range.
galaxies = fits_table(
galfn,
rows=I,
columns=['ra', 'dec', 'd25', 'mag', 'lslga_id', 'ba', 'pa'])
del kd
# # D25 is diameter in arcmin
galaxies.radius = galaxies.d25 / 2. / 60.
# John told me to do this...
#galaxies.radius *= 1.2 ...and then John taketh away.
galaxies.delete_column('d25')
galaxies.rename('lslga_id', 'ref_id')
galaxies.ref_cat = np.array(['L2'] * len(galaxies))
galaxies.islargegalaxy = np.ones(len(galaxies), bool)
return galaxies
def other_passes(self, tile, tiles):
'''
Given tile number *tile*, return the obstatus rows for the other passes
on this tile center.
Returns: *otherpasses*, table object
'''
if tiles != self.tiles:
from astrometry.libkd.spherematch import match_radec
# Could also use the fact that the passes are offset from each other
# by a fixed index (15872 for decam)...
# Max separation is about 0.6 degrees for DECam...
#### FIXME for Mosaic this is much smaller... and the three passes
#### for a tile are not necessarily relatively close to each other.
I,J,d = match_radec(tile.ra, tile.dec, tiles.ra, tiles.dec, 1.)
# Omit 'tile' itself...
K = np.flatnonzero(tiles.tileid[J] != tile.tileid)
J = J[K]
return tiles[J]
if self.tiletree is None:
from astrometry.libkd.spherematch import tree_build_radec
self.tiletree = tree_build_radec(self.tiles.ra, self.tiles.dec)
from astrometry.libkd.spherematch import tree_search_radec
J = tree_search_radec(self.tiletree, tile.ra, tile.dec, 1.0)
# Omit 'tile' itself...
K = np.flatnonzero(tiles.tileid[J] != tile.tileid)
J = J[K]
return tiles[J]
def get_healpix_rangesearch_catalogs(self, healpixes, rc, dc, rad):
cats = []
for hp in healpixes:
(kd, tab) = self.get_healpix_tree(hp)
I = tree_search_radec(kd, rc, dc, rad)
if len(I):
cats.append(tab[I])
if len(cats) == 1:
return cats[0] #.copy()
return merge_tables(cats)
def cat_kd(req, ver, tag, fn):
tag = 'spec'
ralo = float(req.GET['ralo'])
rahi = float(req.GET['rahi'])
declo = float(req.GET['declo'])
dechi = float(req.GET['dechi'])
ver = int(ver)
if not ver in catversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
import numpy as np
from astrometry.util.fits import fits_table, merge_tables
from astrometry.libkd.spherematch import tree_open, tree_search_radec
from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between
xyz1 = radectoxyz(ralo, declo)
xyz2 = radectoxyz(rahi, dechi)
xyz = (xyz1 + xyz2) / 2.
xyz /= np.sqrt(np.sum(xyz**2))
rc, dc = xyztoradec(xyz)
rc = rc[0]
dc = dc[0]
rad = degrees_between(rc, dc, ralo, declo)
kd = tree_open(fn)
I = tree_search_radec(kd, rc, dc, rad)
print('Matched', len(I), 'from', fn)
if len(I) == 0:
return None
T = fits_table(fn, rows=I)
debug(len(T), 'spectra')
if ralo > rahi:
# RA wrap
T.cut(
np.logical_or(T.ra > ralo, T.ra < rahi) * (T.dec > declo) *
(T.dec < dechi))
else:
T.cut(
(T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi))
debug(len(T), 'in cut')
return T
def bricks_touching_radec_box(self, bricks,
ralo, rahi, declo, dechi):
'''
Returns an index vector of the bricks that touch the given RA,Dec box.
'''
if bricks is None:
bricks = self.get_bricks_readonly()
if self.cache_tree and bricks == self.bricks:
from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec
# Use kdtree
if self.bricktree is None:
self.bricktree = tree_build_radec(bricks.ra, bricks.dec)
# brick size
radius = np.sqrt(2.)/2. * self.bricksize
# + RA,Dec box size
radius = radius + degrees_between(ralo, declo, rahi, dechi) / 2.
dec = (dechi + declo) / 2.
c = (np.cos(np.deg2rad(rahi)) + np.cos(np.deg2rad(ralo))) / 2.
s = (np.sin(np.deg2rad(rahi)) + np.sin(np.deg2rad(ralo))) / 2.
ra = np.rad2deg(np.arctan2(s, c))
J = tree_search_radec(self.bricktree, ra, dec, radius)
I = J[np.nonzero((bricks.ra1[J] <= rahi ) * (bricks.ra2[J] >= ralo) *
(bricks.dec1[J] <= dechi) * (bricks.dec2[J] >= declo))[0]]
return I
if rahi < ralo:
# Wrap-around
print('In Dec slice:', len(np.flatnonzero((bricks.dec1 <= dechi) *
(bricks.dec2 >= declo))))
print('Above RAlo=', ralo, ':', len(np.flatnonzero(bricks.ra2 >= ralo)))
print('Below RAhi=', rahi, ':', len(np.flatnonzero(bricks.ra1 <= rahi)))
print('In RA slice:', len(np.nonzero(np.logical_or(bricks.ra2 >= ralo,
bricks.ra1 <= rahi))))
I, = np.nonzero(np.logical_or(bricks.ra2 >= ralo, bricks.ra1 <= rahi) *
(bricks.dec1 <= dechi) * (bricks.dec2 >= declo))
print('In RA&Dec slice', len(I))
else:
I, = np.nonzero((bricks.ra1 <= rahi ) * (bricks.ra2 >= ralo) *
(bricks.dec1 <= dechi) * (bricks.dec2 >= declo))
return I
def bricks_touching_radec_box(self, bricks,
ralo, rahi, declo, dechi):
'''
Returns an index vector of the bricks that touch the given RA,Dec box.
'''
if bricks is None:
bricks = self.get_bricks_readonly()
if self.cache_tree and bricks == self.bricks:
from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec
# Use kdtree
if self.bricktree is None:
self.bricktree = tree_build_radec(bricks.ra, bricks.dec)
# brick size
radius = np.sqrt(2.)/2. * self.bricksize
# + RA,Dec box size
radius = radius + degrees_between(ralo, declo, rahi, dechi) / 2.
dec = (dechi + declo) / 2.
c = (np.cos(np.deg2rad(rahi)) + np.cos(np.deg2rad(ralo))) / 2.
s = (np.sin(np.deg2rad(rahi)) + np.sin(np.deg2rad(ralo))) / 2.
ra = np.rad2deg(np.arctan2(s, c))
J = tree_search_radec(self.bricktree, ra, dec, radius)
I = J[np.nonzero((bricks.ra1[J] <= rahi ) * (bricks.ra2[J] >= ralo) *
(bricks.dec1[J] <= dechi) * (bricks.dec2[J] >= declo))[0]]
return I
if rahi < ralo:
# Wrap-around
print('In Dec slice:', len(np.flatnonzero((bricks.dec1 <= dechi) *
(bricks.dec2 >= declo))))
print('Above RAlo=', ralo, ':', len(np.flatnonzero(bricks.ra2 >= ralo)))
print('Below RAhi=', rahi, ':', len(np.flatnonzero(bricks.ra1 <= rahi)))
print('In RA slice:', len(np.nonzero(np.logical_or(bricks.ra2 >= ralo,
bricks.ra1 <= rahi))))
I, = np.nonzero(np.logical_or(bricks.ra2 >= ralo, bricks.ra1 <= rahi) *
(bricks.dec1 <= dechi) * (bricks.dec2 >= declo))
print('In RA&Dec slice', len(I))
else:
I, = np.nonzero((bricks.ra1 <= rahi ) * (bricks.ra2 >= ralo) *
(bricks.dec1 <= dechi) * (bricks.dec2 >= declo))
return I
def cat_kd(req, ver, tag, fn):
tag = 'spec'
ralo = float(req.GET['ralo'])
rahi = float(req.GET['rahi'])
declo = float(req.GET['declo'])
dechi = float(req.GET['dechi'])
ver = int(ver)
if not ver in catversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
import numpy as np
from astrometry.util.fits import fits_table, merge_tables
from astrometry.libkd.spherematch import tree_open, tree_search_radec
from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between
xyz1 = radectoxyz(ralo, declo)
xyz2 = radectoxyz(rahi, dechi)
xyz = (xyz1 + xyz2)/2.
xyz /= np.sqrt(np.sum(xyz**2))
rc,dc = xyztoradec(xyz)
rc = rc[0]
dc = dc[0]
rad = degrees_between(rc, dc, ralo, declo)
kd = tree_open(fn)
I = tree_search_radec(kd, rc, dc, rad)
print('Matched', len(I), 'from', fn)
if len(I) == 0:
return None
T = fits_table(fn, rows=I)
debug(len(T), 'spectra')
if ralo > rahi:
# RA wrap
T.cut(np.logical_or(T.ra > ralo, T.ra < rahi) * (T.dec > declo) * (T.dec < dechi))
else:
T.cut((T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi))
debug(len(T), 'in cut')
return T
def cat_targets_drAB(req,
ver,
cats=[],
tag='',
bgs=False,
sky=False,
bright=False,
dark=False,
color_name_func=desitarget_color_names):
'''
color_name_func: function that selects names and colors for targets
(eg based on targeting bit values)
'''
import json
ralo = float(req.GET['ralo'])
rahi = float(req.GET['rahi'])
declo = float(req.GET['declo'])
dechi = float(req.GET['dechi'])
ver = int(ver)
if not ver in catversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
from astrometry.util.fits import fits_table, merge_tables
from astrometry.libkd.spherematch import tree_open, tree_search_radec
import numpy as np
from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between
xyz1 = radectoxyz(ralo, declo)
xyz2 = radectoxyz(rahi, dechi)
xyz = (xyz1 + xyz2) / 2.
xyz /= np.sqrt(np.sum(xyz**2))
rc, dc = xyztoradec(xyz)
rc = rc[0]
dc = dc[0]
rad = degrees_between(rc, dc, ralo, declo)
'''
startree -i /project/projectdirs/desi/target/catalogs/targets-dr4-0.20.0.fits -o data/targets-dr4-0.20.0.kd.fits -P -k -T
'''
TT = []
for fn in cats:
kd = tree_open(fn)
I = tree_search_radec(kd, rc, dc, rad)
print('Matched', len(I), 'from', fn)
if len(I) == 0:
continue
T = fits_table(fn, rows=I)
TT.append(T)
if len(TT) == 0:
return HttpResponse(json.dumps(dict(rd=[], name=[])),
content_type='application/json')
T = merge_tables(TT, columns='fillzero')
if bgs:
T.cut(T.bgs_target > 0)
if bright:
T.cut(np.logical_or(T.bgs_target > 0, T.mws_target > 0))
if dark:
T.cut(T.desi_target > 0)
names = None
colors = None
if color_name_func is not None:
names, colors = color_name_func(T)
if sky:
fluxes = [
dict(g=float(g), r=float(r), z=float(z))
for (g, r, z) in zip(T.apflux_g[:,
0], T.apflux_r[:,
0], T.apflux_z[:,
0])
]
nobs = None
else:
fluxes = [
dict(g=float(g),
r=float(r),
z=float(z),
W1=float(W1),
W2=float(W2))
for (g, r, z, W1,
W2) in zip(T.flux_g, T.flux_r, T.flux_z, T.flux_w1, T.flux_w2)
]
nobs = [
dict(g=int(g), r=int(r), z=int(z))
for g, r, z in zip(T.nobs_g, T.nobs_r, T.nobs_z)
],
rtn = dict(
rd=[(t.ra, t.dec) for t in T],
targetid=[int(t) for t in T.targetid],
fluxes=fluxes,
)
if names is not None:
rtn.update(name=names)
if colors is not None:
rtn.update(color=colors)
if nobs is not None:
rtn.update(nobs=nobs)
return HttpResponse(json.dumps(rtn), content_type='application/json')
def cat_targets_drAB(req, ver, cats=None, tag='', bgs=False, sky=False, bright=False, dark=False, color_name_func=desitarget_color_names):
'''
color_name_func: function that selects names and colors for targets
(eg based on targeting bit values)
'''
if cats is None:
cats = []
import json
ralo = float(req.GET['ralo'])
rahi = float(req.GET['rahi'])
declo = float(req.GET['declo'])
dechi = float(req.GET['dechi'])
ver = int(ver)
if not ver in catversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
from astrometry.util.fits import fits_table, merge_tables
from astrometry.libkd.spherematch import tree_open, tree_search_radec
import numpy as np
from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between
xyz1 = radectoxyz(ralo, declo)
xyz2 = radectoxyz(rahi, dechi)
xyz = (xyz1 + xyz2)/2.
xyz /= np.sqrt(np.sum(xyz**2))
rc,dc = xyztoradec(xyz)
rc = rc[0]
dc = dc[0]
rad = degrees_between(rc, dc, ralo, declo)
'''
startree -i /project/projectdirs/desi/target/catalogs/targets-dr4-0.20.0.fits -o data/targets-dr4-0.20.0.kd.fits -P -k -T
'''
TT = []
for fn in cats:
kd = tree_open(fn)
I = tree_search_radec(kd, rc, dc, rad)
print('Matched', len(I), 'from', fn)
if len(I) == 0:
continue
T = fits_table(fn, rows=I)
TT.append(T)
if len(TT) == 0:
return HttpResponse(json.dumps(dict(rd=[], name=[])),
content_type='application/json')
T = merge_tables(TT, columns='fillzero')
if bgs:
T.cut(T.bgs_target > 0)
if bright:
T.cut(np.logical_or(T.bgs_target > 0, T.mws_target > 0))
if dark:
T.cut(T.desi_target > 0)
names = None
colors = None
if color_name_func is not None:
names,colors = color_name_func(T)
if sky:
fluxes = [dict(g=float(g), r=float(r), z=float(z))
for (g,r,z) in zip(T.apflux_g[:,0], T.apflux_r[:,0], T.apflux_z[:,0])]
nobs = None
else:
fluxes = [dict(g=float(g), r=float(r), z=float(z),
W1=float(W1), W2=float(W2))
for (g,r,z,W1,W2)
in zip(T.flux_g, T.flux_r, T.flux_z, T.flux_w1, T.flux_w2)]
nobs=[dict(g=int(g), r=int(r), z=int(z)) for g,r,z
in zip(T.nobs_g, T.nobs_r, T.nobs_z)],
rtn = dict(rd=[(t.ra, t.dec) for t in T],
targetid=[int(t) for t in T.targetid],
fluxes=fluxes,
)
if names is not None:
rtn.update(name=names)
if colors is not None:
rtn.update(color=colors)
if nobs is not None:
rtn.update(nobs=nobs)
# Convert targetid to string to prevent rounding errors
rtn['targetid'] = [str(s) for s in rtn['targetid']]
return HttpResponse(json.dumps(rtn), content_type='application/json')
def main():
import argparse
parser = argparse.ArgumentParser(
description='This script creates small self-contained data sets that '
'are useful for test cases of the pipeline codes.')
parser.add_argument('ccds', help='CCDs table describing region to grab')
parser.add_argument('outdir', help='Output directory name')
parser.add_argument('brick', help='Brick containing these images')
parser.add_argument('--survey-dir', type=str, default=None)
parser.add_argument('--cache-dir', type=str, default=None,
help='Directory to search for cached files')
parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image')
parser.add_argument('--wise-wcs-hdu', help='For WISE outputs, the HDU to read the WCS from in the file given by --wise.', type=int, default=0)
parser.add_argument('--fpack', action='store_true', default=False)
parser.add_argument('--gzip', action='store_true', default=False)
parser.add_argument('--pad', action='store_true', default=False,
help='Keep original image size, but zero out pixels outside ROI')
args = parser.parse_args()
v = 'SKY_TEMPLATE_DIR'
if v in os.environ:
del os.environ[v]
C = fits_table(args.ccds)
print(len(C), 'CCDs in', args.ccds)
C.camera = np.array([c.strip() for c in C.camera])
survey = LegacySurveyData(cache_dir=args.cache_dir, survey_dir=args.survey_dir)
if ',' in args.brick:
ra,dec = args.brick.split(',')
ra = float(ra)
dec = float(dec)
fakebricks = fits_table()
fakebricks.brickname = np.array([('custom-%06i%s%05i' %
(int(1000*ra), 'm' if dec < 0 else 'p',
int(1000*np.abs(dec))))])
fakebricks.ra = np.array([ra])
fakebricks.dec = np.array([dec])
bricks = fakebricks
outbricks = bricks
else:
bricks = survey.get_bricks_readonly()
outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])]
assert(len(outbricks) == 1)
outsurvey = LegacySurveyData(survey_dir = args.outdir)
trymakedirs(args.outdir)
outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz'))
targetwcs = wcs_for_brick(outbricks[0])
H,W = targetwcs.shape
tycho2fn = survey.find_file('tycho2')
kd = tree_open(tycho2fn, 'stars')
radius = 1.
rc,dc = targetwcs.radec_center()
I = tree_search_radec(kd, rc, dc, radius)
print(len(I), 'Tycho-2 stars within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (rc,dc))
# Read only the rows within range.
tycho = fits_table(tycho2fn, rows=I)
del kd
print('Read', len(tycho), 'Tycho-2 stars')
ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec)
#margin = 100
#tycho.cut(ok * (tx > -margin) * (tx < W+margin) *
# (ty > -margin) * (ty < H+margin))
print('Cut to', len(tycho), 'Tycho-2 stars within brick')
del ok,tx,ty
#tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz'))
f,tfn = tempfile.mkstemp(suffix='.fits')
os.close(f)
tycho.writeto(tfn)
outfn = os.path.join(args.outdir, 'tycho2.kd.fits')
cmd = 'startree -i %s -o %s -P -k -n stars -T' % (tfn, outfn)
print(cmd)
rtn = os.system(cmd)
assert(rtn == 0)
os.unlink(tfn)
from legacypipe.gaiacat import GaiaCatalog
gcat = GaiaCatalog()
# from ps1cat.py:
wcs = targetwcs
step=100.
margin=10.
# Grid the CCD in pixel space
W,H = wcs.get_width(), wcs.get_height()
xx,yy = np.meshgrid(
np.linspace(1-margin, W+margin, 2+int((W+2*margin)/step)),
np.linspace(1-margin, H+margin, 2+int((H+2*margin)/step)))
# Convert to RA,Dec and then to unique healpixes
ra,dec = wcs.pixelxy2radec(xx.ravel(), yy.ravel())
healpixes = set()
for r,d in zip(ra,dec):
healpixes.add(gcat.healpix_for_radec(r, d))
for hp in healpixes:
hpcat = gcat.get_healpix_catalog(hp)
ok,xx,yy = wcs.radec2pixelxy(hpcat.ra, hpcat.dec)
onccd = np.flatnonzero((xx >= 1.-margin) * (xx <= W+margin) *
(yy >= 1.-margin) * (yy <= H+margin))
hpcat.cut(onccd)
if len(hpcat):
outfn = os.path.join(args.outdir, 'gaia', 'chunk-%05d.fits' % hp)
trymakedirs(os.path.join(args.outdir, 'gaia'))
hpcat.writeto(outfn)
outccds = C.copy()
cols = outccds.get_columns()
for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1',
'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1',
'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver',
'psfplver' ]:
if c in cols:
outccds.delete_column(c)
outccds.image_hdu[:] = 1
# Convert to list to avoid truncating filenames
outccds.image_filename = [fn for fn in outccds.image_filename]
for iccd,ccd in enumerate(C):
decam = (ccd.camera.strip() == 'decam')
bok = (ccd.camera.strip() == '90prime')
im = survey.get_image_object(ccd)
print('Got', im)
if survey.cache_dir is not None:
im.check_for_cached_files(survey)
slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1))
psfkwargs = dict(pixPsf=True, gaussPsf=False, hybridPsf=False,
normalizePsf=False)
tim = im.get_tractor_image(slc, pixPsf=True,
subsky=False, nanomaggies=False,
no_remap_invvar=True, old_calibs_ok=True)
print('Tim:', tim.shape)
psfrow = psfhdr = None
if args.pad:
psf = im.read_psf_model(0, 0, w=im.width, h=im.height, **psfkwargs)
psfex = psf.psfex
else:
psf = tim.getPsf()
psfex = psf.psfex
# Did the PSF model come from a merged file?
for fn in [im.merged_psffn, im.psffn] + im.old_merged_psffns:
if not os.path.exists(fn):
continue
T = fits_table(fn)
I, = np.nonzero((T.expnum == im.expnum) *
np.array([c.strip() == im.ccdname for c in T.ccdname]))
if len(I) != 1:
continue
psfrow = T[I]
x0 = ccd.ccd_x0
y0 = ccd.ccd_y0
psfrow.polzero1[0] -= x0
psfrow.polzero2[0] -= y0
#psfhdr = fitsio.read_header(im.merged_psffn)
break
psfex.fwhm = tim.psf_fwhm
#### HACK
#psfrow = None
assert(psfrow is not None)
if psfrow is not None:
print('PSF row:', psfrow)
#else:
# print('PSF:', psf)
# print('PsfEx:', psfex)
skyrow = skyhdr = None
if args.pad:
primhdr = fitsio.read_header(im.imgfn)
imghdr = fitsio.read_header(im.imgfn, hdu=im.hdu)
sky = im.read_sky_model(splinesky=True, primhdr=primhdr, imghdr=imghdr)
#skyhdr = fitsio.read_header(im.splineskyfn)
#msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True)
else:
sky = tim.getSky()
# Did the sky model come from a merged file?
#msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True)
print('merged skyfn:', im.merged_skyfn)
print('single skyfn:', im.skyfn)
print('old merged skyfns:', im.old_merged_skyfns)
for fn in [im.merged_skyfn, im.skyfn] + im.old_merged_skyfns:
if not os.path.exists(fn):
continue
T = fits_table(fn)
I, = np.nonzero((T.expnum == im.expnum) *
np.array([c.strip() == im.ccdname for c in T.ccdname]))
skyrow = T[I]
skyrow.x0[0] = ccd.ccd_x0
skyrow.y0[0] = ccd.ccd_y0
# s_med = skyrow.sky_med[0]
# s_john = skyrow.sky_john[0]
# skyhdr = fitsio.read_header(fn)
assert(skyrow is not None)
### HACK
#skyrow = None
if skyrow is not None:
print('Sky row:', skyrow)
else:
print('Sky:', sky)
# Output filename format:
fn = ccd.image_filename.strip()
ccd.image_filename = os.path.join(os.path.dirname(fn),
'%s.%s.fits' % (os.path.basename(fn).split('.')[0], ccd.ccdname.strip()))
outim = outsurvey.get_image_object(ccd)
print('Output image:', outim)
print('Image filename:', outim.imgfn)
trymakedirs(outim.imgfn, dir=True)
imgdata = tim.getImage()
ivdata = tim.getInvvar()
# Since we remap DQ codes (always with Mosaic and Bok, sometimes with DECam),
# re-read from the FITS file rather than using tim.dq.
print('Reading data quality from', im.dqfn, 'hdu', im.hdu)
dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice)
print('Tim shape:', tim.shape, 'Slice', tim.slice)
print('image shape:', imgdata.shape, 'iv', ivdata.shape, 'DQ', dqdata.shape)
from collections import Counter
dqvals = Counter(dqdata.ravel())
print('DQ pixel counts:')
for k,n in dqvals.most_common():
print(' 0x%x' % k, ':', n)
if args.pad:
# Create zero image of full size, copy in data.
fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype)
fullsize[slc] = imgdata
imgdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype)
fullsize[slc] = dqdata
dqdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype)
fullsize[slc] = ivdata
ivdata = fullsize
else:
# Adjust the header WCS by x0,y0
crpix1 = tim.hdr['CRPIX1']
crpix2 = tim.hdr['CRPIX2']
tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0
tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0
# Add image extension to filename
# fitsio doesn't compress .fz by default, so drop .fz suffix
#outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits')
if args.gzip:
outim.imgfn = outim.imgfn.replace('.fits', '.fits.gz')
#outim.wtfn = outim.wtfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.wtfn = outim.wtfn.replace('.fits.fz', '.fits')
if args.gzip:
outim.wtfn = outim.wtfn.replace('.fits', '.fits.gz')
if outim.dqfn is not None:
#outim.dqfn = outim.dqfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.dqfn = outim.dqfn.replace('.fits.fz', '.fits')
if args.gzip:
outim.dqfn = outim.dqfn.replace('.fits', '.fits.gz')
if bok:
outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname)
ccdfn = outim.imgfn
ccdfn = ccdfn.replace(outsurvey.get_image_dir(), '')
if ccdfn.startswith('/'):
ccdfn = ccdfn[1:]
outccds.image_filename[iccd] = ccdfn
print('Changed output filenames to:')
print(outim.imgfn)
print(outim.dqfn)
ofn = outim.imgfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fits = fitsio.FITS(ofn, 'rw', clobber=True)
fits.write(None, header=tim.primhdr)
fits.write(imgdata, header=tim.hdr, extname=ccd.ccdname)
fits.close()
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
h,w = tim.shape
if not args.pad:
outccds.width[iccd] = w
outccds.height[iccd] = h
outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0
outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0
wcs = Tan(*[float(x) for x in
[ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2,
ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]])
if args.pad:
subwcs = wcs
else:
subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h)
outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center()
print('Weight filename:', outim.wtfn)
wfn = outim.wtfn
trymakedirs(wfn, dir=True)
ofn = wfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fits = fitsio.FITS(ofn, 'rw', clobber=True)
fits.write(None, header=tim.primhdr)
fits.write(ivdata, header=tim.hdr, extname=ccd.ccdname)
fits.close()
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
if outim.dqfn is not None:
print('DQ filename', outim.dqfn)
trymakedirs(outim.dqfn, dir=True)
ofn = outim.dqfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fits = fitsio.FITS(ofn, 'rw', clobber=True)
fits.write(None, header=tim.primhdr)
fits.write(dqdata, header=tim.hdr, extname=ccd.ccdname)
fits.close()
if args.fpack:
cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
psfout = outim.psffn
#if psfrow:
# psfout = outim.merged_psffn
print('PSF output filename:', psfout)
trymakedirs(psfout, dir=True)
if psfrow:
psfrow.writeto(psfout, primhdr=psfhdr)
else:
print('Writing PsfEx:', psfout)
psfex.writeto(psfout)
# update header
F = fitsio.FITS(psfout, 'rw')
F[0].write_keys([dict(name='EXPNUM', value=ccd.expnum),
dict(name='PLVER', value=psf.plver),
dict(name='PROCDATE', value=psf.procdate),
dict(name='PLPROCID', value=psf.plprocid),])
F.close()
skyout = outim.skyfn
#if skyrow:
# skyout = outim.merged_splineskyfn
print('Sky output filename:', skyout)
trymakedirs(skyout, dir=True)
if skyrow is not None:
skyrow.writeto(skyout, primhdr=skyhdr)
else:
primhdr = fitsio.FITSHDR()
primhdr['PLVER'] = sky.plver
primhdr['PLPROCID'] = sky.plprocid
primhdr['PROCDATE'] = sky.procdate
primhdr['EXPNUM'] = ccd.expnum
primhdr['IMGDSUM'] = sky.datasum
primhdr['S_MED'] = s_med
primhdr['S_JOHN'] = s_john
sky.write_fits(skyout, primhdr=primhdr)
# HACK -- check result immediately.
outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))
outsurvey.ccds = None
outC = outsurvey.get_ccds_readonly()
occd = outC[iccd]
outim = outsurvey.get_image_object(occd)
print('Got output image:', outim)
otim = outim.get_tractor_image(pixPsf=True,
hybridPsf=True, old_calibs_ok=True)
print('Got output tim:', otim)
outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))
# WISE
if args.wise is not None:
from wise.forcedphot import unwise_tiles_touching_wcs
from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs,
get_unwise_tractor_image, get_unwise_tile_dir)
# Read WCS...
print('Reading TAN wcs header from', args.wise, 'HDU', args.wise_wcs_hdu)
targetwcs = Tan(args.wise, args.wise_wcs_hdu)
tiles = unwise_tiles_touching_wcs(targetwcs)
print('Cut to', len(tiles), 'unWISE tiles')
H,W = targetwcs.shape
r,d = targetwcs.pixelxy2radec(np.array([1, W, W/2, W/2]),
np.array([H/2, H/2, 1, H ]))
roiradec = [r[0], r[1], d[2], d[3]]
unwise_dir = os.environ['UNWISE_COADDS_DIR']
wise_out = os.path.join(args.outdir, 'images', 'unwise')
print('Will write WISE outputs to', wise_out)
unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR']
wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr')
print('Will write WISE time-resolved outputs to', wise_tr_out)
trymakedirs(wise_tr_out)
W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_atlas.fits'))
print('Read', len(W), 'time-resolved WISE coadd tiles')
W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id]))
print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth')
# Write the time-resolved index subset.
W.writeto(os.path.join(wise_tr_out, 'time_resolved_atlas.fits'))
# this ought to be enough for anyone =)
_,Nepochs = W.epoch_bitmask.shape
print('N epochs in time-resolved atlas:', Nepochs)
wisedata = []
# full depth
for band in [1,2,3,4]:
wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band, True))
# time-resolved
for band in [1,2]:
# W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2)
bitmask = (1 << (band-1))
for e in range(Nepochs):
# Which tiles have images for this epoch?
I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask)
if len(I) == 0:
continue
print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I])
edir = os.path.join(unwise_tr_dir, 'e%03i' % e)
eoutdir = os.path.join(wise_tr_out, 'e%03i' % e)
wisedata.append((edir, eoutdir, tiles.coadd_id[I], band, False))
wrote_masks = set()
model_dir = os.environ.get('UNWISE_MODEL_SKY_DIR')
if model_dir is not None:
model_dir_out = os.path.join(args.outdir, 'images', 'unwise-mod')
trymakedirs(model_dir_out)
for indir, outdir, tiles, band, fulldepth in wisedata:
for tile in tiles:
wanyband = 'w'
tim = get_unwise_tractor_image(indir, tile, band,
bandname=wanyband, roiradecbox=roiradec)
print('Got unWISE tim', tim)
print(tim.shape)
if model_dir is not None and fulldepth and band in [1,2]:
print('ROI', tim.roi)
#0387p575.1.mod.fits
fn = '%s.%i.mod.fits' % (tile, band)
print('Filename', fn)
F = fitsio.FITS(os.path.join(model_dir, fn))
x0,x1,y0,y1 = tim.roi
slc = slice(y0,y1),slice(x0,x1)
phdr = F[0].read_header()
outfn = os.path.join(model_dir_out, fn)
for e,extname in [(1,'MODEL'), (2,'SKY')]:
pix = F[e][slc]
hdr = F[e].read_header()
crpix1 = hdr['CRPIX1']
crpix2 = hdr['CRPIX2']
hdr['CRPIX1'] -= x0
hdr['CRPIX2'] -= y0
#print('mod', mod)
#print('Model', mod.shape)
if e == 1:
fitsio.write(outfn, None, clobber=True, header=phdr)
fitsio.write(outfn, pix, header=hdr, extname=extname)
print('Wrote', outfn)
thisdir = get_unwise_tile_dir(outdir, tile)
print('Directory for this WISE tile:', thisdir)
base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band))
print('Base filename:', base)
masked = True
mu = 'm' if masked else 'u'
imfn = base + 'img-%s.fits' % mu
ivfn = base + 'invvar-%s.fits.gz' % mu
nifn = base + 'n-%s.fits.gz' % mu
nufn = base + 'n-u.fits.gz'
#print('WISE image header:', tim.hdr)
# Adjust the header WCS by x0,y0
wcs = tim.wcs.wcs
tim.hdr['CRPIX1'] = wcs.crpix[0]
tim.hdr['CRPIX2'] = wcs.crpix[1]
H,W = tim.shape
tim.hdr['IMAGEW'] = W
tim.hdr['IMAGEH'] = H
print('WCS:', wcs)
print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2'])
trymakedirs(imfn, dir=True)
fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True)
print('Wrote', imfn)
fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True)
print('Wrote', ivfn)
fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True)
print('Wrote', nifn)
fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True)
print('Wrote', nufn)
if not (indir,tile) in wrote_masks:
print('Looking for mask file for', indir, tile)
# record that we tried this dir/tile combo
wrote_masks.add((indir,tile))
for idir in indir.split(':'):
tdir = get_unwise_tile_dir(idir, tile)
maskfn = 'unwise-%s-msk.fits.gz' % tile
fn = os.path.join(tdir, maskfn)
print('Mask file:', fn)
if os.path.exists(fn):
print('Reading', fn)
(x0,x1,y0,y1) = tim.roi
roislice = (slice(y0,y1), slice(x0,x1))
F = fitsio.FITS(fn)[0]
hdr = F.read_header()
M = F[roislice]
outfn = os.path.join(thisdir, maskfn)
fitsio.write(outfn, M, header=tim.hdr, clobber=True)
print('Wrote', outfn)
break
outC = outsurvey.get_ccds_readonly()
for iccd,ccd in enumerate(outC):
outim = outsurvey.get_image_object(ccd)
print('Got output image:', outim)
otim = outim.get_tractor_image(pixPsf=True,
hybridPsf=True, old_calibs_ok=True)
print('Got output tim:', otim)
print('Kd data:', spherematch.tree_data(kd, np.array([0,3,5]).astype(np.uint32)))
print('Kd data:', kd.get_data(np.array([0,3,5]).astype(np.uint32)))
print('Permute:', spherematch.tree_permute(kd, np.array([3,5,7]).astype(np.int32)))
print('Permute:', kd.permute(np.array([0,99,199]).astype(np.int32)))
ra,dec = np.meshgrid(np.arange(0, 360), np.arange(-90, 91, 1))
ra1 = ra.ravel()
dec1 = dec.ravel()
rdkd1 = spherematch.tree_build_radec(ra1, dec1)
print('RdKd:', rdkd1.n, rdkd1.bbox)
ra2 = np.random.uniform(-10, 10, size=1000)
dec2 = np.random.uniform(-10, 10, size=1000)
rdkd2 = spherematch.tree_build_radec(ra2, dec2)
I = spherematch.tree_search_radec(rdkd1, ra2[0], dec2[0], 2.)
print('search_radec:', I)
I,J,d = spherematch.match_radec(ra1, dec1, ra2, dec2, 1.)
print('Matches:', len(I))
I,J,d = spherematch.match_radec(ra1, dec1, ra2, dec2, 1., nearest=True)
print('Nearest matches:', len(I))
I = spherematch.match_radec(ra1, dec1, ra2, dec2, 1.,
indexlist=True)
print('Index lists matches:', len(I))
def map_decam_depth(req,
ver,
zoom,
x,
y,
savecache=False,
band=None,
ignoreCached=False):
global Tdepth
global Tdepthkd
if band is None:
band = req.GET.get('band')
if not band in ['g', 'r', 'z']:
raise RuntimeError('Invalid band')
tag = 'decam-depth-%s' % band
zoom = int(zoom)
zoomscale = 2.**zoom
x = int(x)
y = int(y)
if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale:
raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom, x, y))
ver = int(ver)
if not ver in tileversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
basedir = settings.DATA_DIR
tilefn = os.path.join(basedir, 'tiles', tag,
'%i/%i/%i/%i.jpg' % (ver, zoom, x, y))
if os.path.exists(tilefn) and not ignoreCached:
print('Cached:', tilefn)
return send_file(tilefn,
'image/jpeg',
expires=oneyear,
modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'))
from astrometry.util.util import Tan
from astrometry.libkd.spherematch import match_radec
from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec
from astrometry.util.fits import fits_table
from astrometry.util.starutil_numpy import degrees_between
import numpy as np
import fitsio
try:
wcs, W, H, zoomscale, zoom, x, y = get_tile_wcs(zoom, x, y)
except RuntimeError as e:
return HttpResponse(e.strerror)
rlo, d = wcs.pixelxy2radec(W, H / 2)[-2:]
rhi, d = wcs.pixelxy2radec(1, H / 2)[-2:]
r, d1 = wcs.pixelxy2radec(W / 2, 1)[-2:]
r, d2 = wcs.pixelxy2radec(W / 2, H)[-2:]
r, d = wcs.pixelxy2radec(W / 2, H / 2)[-2:]
rad = max(degrees_between(r, d, rlo, d1), degrees_between(r, d, rhi, d2))
if Tdepth is None:
T = fits_table(os.path.join(basedir, 'decals-zpt-nondecals.fits'),
columns=[
'ccdra', 'ccddec', 'arawgain', 'avsky', 'ccdzpt',
'filter', 'crpix1', 'crpix2', 'crval1', 'crval2',
'cd1_1', 'cd1_2', 'cd2_1', 'cd2_2', 'naxis1',
'naxis2', 'exptime', 'fwhm'
])
T.rename('ccdra', 'ra')
T.rename('ccddec', 'dec')
Tdepth = {}
Tdepthkd = {}
for b in ['g', 'r', 'z']:
Tdepth[b] = T[T.filter == b]
Tdepthkd[b] = tree_build_radec(Tdepth[b].ra, Tdepth[b].dec)
T = Tdepth[band]
Tkd = Tdepthkd[band]
#I,J,d = match_radec(T.ra, T.dec, r, d, rad + 0.2)
I = tree_search_radec(Tkd, r, d, rad + 0.2)
print(len(I), 'CCDs in range')
if len(I) == 0:
from django.http import HttpResponseRedirect
return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg')
depthiv = np.zeros((H, W), np.float32)
for t in T[I]:
twcs = Tan(*[
float(x) for x in [
t.crval1, t.crval2, t.crpix1, t.crpix2, t.cd1_1, t.cd1_2,
t.cd2_1, t.cd2_2, t.naxis1, t.naxis2
]
])
w, h = t.naxis1, t.naxis2
r, d = twcs.pixelxy2radec([1, 1, w, w], [1, h, h, 1])
ok, x, y = wcs.radec2pixelxy(r, d)
#print 'x,y coords of CCD:', x, y
x0 = int(x.min())
x1 = int(x.max())
y0 = int(y.min())
y1 = int(y.max())
if y1 < 0 or x1 < 0 or x0 >= W or y0 >= H:
continue
readnoise = 10. # e-; 7.0 to 15.0 according to DECam Data Handbook
skysig = np.sqrt(t.avsky * t.arawgain + readnoise**2) / t.arawgain
zpscale = 10.**((t.ccdzpt - 22.5) / 2.5) * t.exptime
sig1 = skysig / zpscale
psf_sigma = t.fwhm / 2.35
# point-source depth
psfnorm = 1. / (2. * np.sqrt(np.pi) * psf_sigma)
detsig1 = sig1 / psfnorm
#print '5-sigma point-source depth:', NanoMaggies.nanomaggiesToMag(detsig1 * 5.)
div = 1 / detsig1**2
depthiv[max(y0, 0):min(y1, H), max(x0, 0):min(x1, W)] += div
ptsrc = -2.5 * (np.log10(np.sqrt(1. / depthiv) * 5) - 9)
ptsrc[depthiv == 0] = 0.
if savecache:
trymakedirs(tilefn)
else:
import tempfile
f, tilefn = tempfile.mkstemp(suffix='.jpg')
os.close(f)
import pylab as plt
plt.imsave(tilefn, ptsrc, vmin=22., vmax=25., cmap='hot') #nipy_spectral')
return send_file(tilefn, 'image/jpeg', unlink=(not savecache))
def map_sdss(req,
ver,
zoom,
x,
y,
savecache=None,
tag='sdss',
get_images=False,
ignoreCached=False,
wcs=None,
forcecache=False,
forcescale=None,
bestOnly=False,
bands='gri',
**kwargs):
from decals import settings
if savecache is None:
savecache = settings.SAVE_CACHE
zoom = int(zoom)
zoomscale = 2.**zoom
x = int(x)
y = int(y)
if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale:
raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom, x, y))
ver = int(ver)
if not ver in tileversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
basedir = settings.DATA_DIR
tilefn = os.path.join(basedir, 'tiles', tag,
'%i/%i/%i/%i.jpg' % (ver, zoom, x, y))
if os.path.exists(tilefn) and not ignoreCached:
if get_images:
return None
return send_file(tilefn,
'image/jpeg',
expires=oneyear,
modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'))
if not savecache:
import tempfile
f, tilefn = tempfile.mkstemp(suffix='.jpg')
os.close(f)
if wcs is None:
try:
wcs, W, H, zoomscale, zoom, x, y = get_tile_wcs(zoom, x, y)
except RuntimeError as e:
if get_images:
return None
return HttpResponse(e.strerror)
else:
W = wcs.get_width()
H = wcs.get_height()
from astrometry.util.fits import fits_table
import numpy as np
from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec
from astrometry.util.starutil_numpy import degrees_between, arcsec_between
from astrometry.util.resample import resample_with_wcs, OverlapError
from astrometry.util.util import Tan, Sip
import fitsio
print('Tile wcs: center', wcs.radec_center(), 'pixel scale',
wcs.pixel_scale())
global w_flist
global w_flist_tree
if w_flist is None:
w_flist = fits_table(
os.path.join(settings.DATA_DIR, 'sdss', 'window_flist.fits'),
columns=['run', 'rerun', 'camcol', 'field', 'ra', 'dec', 'score'])
print('Read', len(w_flist), 'window_flist entries')
w_flist.cut(w_flist.rerun == '301')
print('Cut to', len(w_flist), 'in rerun 301')
w_flist_tree = tree_build_radec(w_flist.ra, w_flist.dec)
# SDSS field size
radius = 1.01 * np.hypot(10., 14.) / 2. / 60.
# leaflet tile size
ra, dec = wcs.pixelxy2radec(W / 2., H / 2.)[-2:]
r0, d0 = wcs.pixelxy2radec(1, 1)[-2:]
r1, d1 = wcs.pixelxy2radec(W, H)[-2:]
radius = radius + max(degrees_between(ra, dec, r0, d0),
degrees_between(ra, dec, r1, d1))
J = tree_search_radec(w_flist_tree, ra, dec, radius)
print(len(J), 'overlapping SDSS fields found')
if len(J) == 0:
if get_images:
return None
if forcecache:
# create symlink to blank.jpg!
trymakedirs(tilefn)
src = os.path.join(settings.STATIC_ROOT, 'blank.jpg')
if os.path.exists(tilefn):
os.unlink(tilefn)
os.symlink(src, tilefn)
print('Symlinked', tilefn, '->', src)
from django.http import HttpResponseRedirect
return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg')
ww = [1, W * 0.25, W * 0.5, W * 0.75, W]
hh = [1, H * 0.25, H * 0.5, H * 0.75, H]
r, d = wcs.pixelxy2radec(
[1] * len(hh) + ww + [W] * len(hh) + list(reversed(ww)),
hh + [1] * len(ww) + list(reversed(hh)) + [H] * len(ww))[-2:]
scaled = 0
scalepat = None
scaledir = 'sdss'
if zoom <= 13 and forcescale is None:
# Get *actual* pixel scales at the top & bottom
r1, d1 = wcs.pixelxy2radec(W / 2., H)[-2:]
r2, d2 = wcs.pixelxy2radec(W / 2., H - 1.)[-2:]
r3, d3 = wcs.pixelxy2radec(W / 2., 1.)[-2:]
r4, d4 = wcs.pixelxy2radec(W / 2., 2.)[-2:]
# Take the min = most zoomed-in
scale = min(arcsec_between(r1, d1, r2, d2),
arcsec_between(r3, d3, r4, d4))
native_scale = 0.396
scaled = int(np.floor(np.log2(scale / native_scale)))
print('Zoom:', zoom, 'x,y', x, y, 'Tile pixel scale:', scale,
'Scale step:', scaled)
scaled = np.clip(scaled, 1, 7)
dirnm = os.path.join(basedir, 'scaled', scaledir)
scalepat = os.path.join(
dirnm, '%(scale)i%(band)s', '%(rerun)s', '%(run)i', '%(camcol)i',
'sdss-%(run)i-%(camcol)i-%(field)i-%(band)s.fits')
if forcescale is not None:
scaled = forcescale
rimgs = [np.zeros((H, W), np.float32) for band in bands]
rns = [np.zeros((H, W), np.float32) for band in bands]
from astrometry.sdss import AsTransWrapper, DR9
sdss = DR9(basedir=settings.SDSS_DIR)
sdss.saveUnzippedFiles(settings.SDSS_DIR)
#sdss.setFitsioReadBZ2()
if settings.SDSS_PHOTOOBJS:
sdss.useLocalTree(photoObjs=settings.SDSS_PHOTOOBJS,
resolve=settings.SDSS_RESOLVE)
for jnum, j in enumerate(J):
print('SDSS field', jnum, 'of', len(J), 'for zoom', zoom, 'x', x, 'y',
y)
im = w_flist[j]
if im.score >= 0.5:
weight = 1.
else:
weight = 0.001
for band, rimg, rn in zip(bands, rimgs, rns):
if im.rerun != '301':
continue
tmpsuff = '.tmp%08i' % np.random.randint(100000000)
basefn = sdss.retrieve('frame',
im.run,
im.camcol,
field=im.field,
band=band,
rerun=im.rerun,
tempsuffix=tmpsuff)
if scaled > 0:
fnargs = dict(band=band,
rerun=im.rerun,
run=im.run,
camcol=im.camcol,
field=im.field)
fn = get_scaled(scalepat,
fnargs,
scaled,
basefn,
read_base_wcs=read_astrans,
read_wcs=read_sip_wcs)
print('get_scaled:', fn)
else:
fn = basefn
frame = None
if fn == basefn:
frame = sdss.readFrame(im.run,
im.camcol,
im.field,
band,
filename=fn)
h, w = frame.getImageShape()
# Trim off the overlapping top of the image
# Wimp out and instead of trimming 128 pix, trim 124!
trim = 124
subh = h - trim
astrans = frame.getAsTrans()
fwcs = AsTransWrapper(astrans, w, subh)
fullimg = frame.getImage()
fullimg = fullimg[:-trim, :]
else:
fwcs = Sip(fn)
fitsimg = fitsio.FITS(fn)[0]
h, w = fitsimg.get_info()['dims']
fullimg = fitsimg.read()
try:
#Yo,Xo,Yi,Xi,nil = resample_with_wcs(wcs, fwcs, [], 3)
Yo, Xo, Yi, Xi, [resamp
] = resample_with_wcs(wcs, fwcs, [fullimg], 2)
except OverlapError:
continue
if len(Xi) == 0:
#print 'No overlap'
continue
if sdssps is not None:
x0 = Xi.min()
x1 = Xi.max()
y0 = Yi.min()
y1 = Yi.max()
slc = (slice(y0, y1 + 1), slice(x0, x1 + 1))
if frame is not None:
img = frame.getImageSlice(slc)
else:
img = fitsimg[slc]
#rimg[Yo,Xo] += img[Yi-y0, Xi-x0]
if bestOnly:
K = np.flatnonzero(weight > rn[Yo, Xo])
print('Updating', len(K), 'of', len(Yo), 'pixels')
if len(K):
rimg[Yo[K], Xo[K]] = resamp[K] * weight
rn[Yo[K], Xo[K]] = weight
else:
rimg[Yo, Xo] += resamp * weight
rn[Yo, Xo] += weight
if sdssps is not None:
# goodpix = np.ones(img.shape, bool)
# fpM = sdss.readFpM(im.run, im.camcol, im.field, band)
# for plane in [ 'INTERP', 'SATUR', 'CR', 'GHOST' ]:
# fpM.setMaskedPixels(plane, goodpix, False, roi=[x0,x1,y0,y1])
plt.clf()
#ima = dict(vmin=-0.05, vmax=0.5)
#ima = dict(vmin=-0.5, vmax=2.)
ima = dict(vmax=np.percentile(img, 99))
plt.subplot(2, 3, 1)
dimshow(img, ticks=False, **ima)
plt.title('image')
rthis = np.zeros_like(rimg)
#rthis[Yo,Xo] += img[Yi-y0, Xi-x0]
rthis[Yo, Xo] += resamp
plt.subplot(2, 3, 2)
dimshow(rthis, ticks=False, **ima)
plt.title('resampled')
# plt.subplot(2,3,3)
# dimshow(goodpix, ticks=False, vmin=0, vmax=1)
# plt.title('good pix')
plt.subplot(2, 3, 4)
dimshow(rimg / np.maximum(rn, 1), ticks=False, **ima)
plt.title('coadd')
plt.subplot(2, 3, 5)
dimshow(rn, vmin=0, ticks=False)
plt.title('coverage: max %i' % rn.max())
plt.subplot(2, 3, 6)
rgb = sdss_rgb(
[rimg / np.maximum(rn, 1) for rimg, rn in zip(rimgs, rns)],
bands)
dimshow(rgb)
plt.suptitle('SDSS %s, R/C/F %i/%i/%i' %
(band, im.run, im.camcol, im.field))
sdssps.savefig()
for rimg, rn in zip(rimgs, rns):
rimg /= np.maximum(rn, 1e-3)
del rns
if get_images:
return rimgs
rgb = sdss_rgb(rimgs, bands)
trymakedirs(tilefn)
save_jpeg(tilefn, rgb)
print('Wrote', tilefn)
return send_file(tilefn, 'image/jpeg', unlink=(not savecache))
def run_tiles(X):
tiles, tag = X
print('Running', tag, '-', len(tiles), 'tiles')
# Aaron's file has all images share the boresight CRVAL, so they have large CRPIX values.
T = fits_table(
'/global/cfs/cdirs/desi/users/ameisner/GFA/gfa_reduce_etc/gfa_wcs+focus.bigtan-zenith.fits'
)
Nbright = 10
tiles_ann = fits_table()
tiles_ann.index = tiles.index
gfa_regions = []
maxr = 0.
#t.cd[0,0], t.cd[0,1], t.cd[1,0], t.cd[1,1],
for t in T:
wcs = Tan(0., 0., t.crpix[0], t.crpix[1], t.cd[0, 0], t.cd[1, 0],
t.cd[0, 1], t.cd[1, 1], float(t.naxis[0]), float(t.naxis[1]))
ctype = t.extname[:5]
cnum = int(t.extname[5])
h, w = wcs.shape
x, y = [1, 1, w, w, 1], [1, h, h, 1, 1]
r, d = wcs.pixelxy2radec(x, y)
dists = degrees_between(0., 0., r, d)
maxr = max(maxr, max(dists))
# x0, y0, x1, y1
rois = []
if ctype == 'FOCUS':
# add the two half-chips.
# wcs.get_subimage moves the CRPIX, but leave CRVAL unchanged, so tx,ty still work unchanged.
# Aaron's WCS templates correct for the overscans
#wcs_subs.append((cstr, cnum, 'a', wcs.get_subimage(0, 0, 1024, h)))
#wcs_subs.append((cstr, cnum, 'b', wcs.get_subimage(1024, 0, 1024, h)))
#all_sub_wcs[(cstr, cnum, 1)] = (tx, ty, wcs.get_subimage(50, 0, 1024, 1032))
#all_sub_wcs[(cstr, cnum, 2)] = (tx, ty, wcs.get_subimage(1174, 0, 1024, 1032))
# Add (negative) margin for donut size and telescope pointing uncertainty.
# ~10" for donuts and ~10" for telescope pointing
#margin = 100
#wcs_subs.append((cstr, cnum, 'a_margin', wcs.get_subimage(margin, margin, 1024-2*margin, h-2*margin)))
#wcs_subs.append((cstr, cnum, 'b_margin', wcs.get_subimage(1024+margin, margin, 1024-2*margin, h-2*margin)))
# Also add a positive margin for bright-star reflections off filters
#margin = 125
#wcs_subs.append((cstr, cnum, 'expanded', wcs.get_subimage(-margin, -margin, w+2*margin, h+2*margin)))
rois.append(('a', 0, 0, 1024, h))
rois.append(('b', 1024, 0, 2048, h))
margin = 100
rois.append(
('a_margin', margin, margin, 1024 - margin, h - margin))
rois.append(
('b_margin', 1024 + margin, margin, 2048 - margin, h - margin))
margin = 125
rois.append(('expanded', -margin, -margin, w + margin, h + margin))
else:
# Guide chips include overscan pixels -- including a blank region in the middle.
#print(cstr,cnum, 'shape', wcs.shape)
#wcs_subs.append((cstr, cnum, 'ccd', wcs))
rois.append(('ccd', 0, 0, w, h))
# Add expanded GUIDE chips -- 25" margin / 0.2"/pix = 125 pix
margin = 125
#wcs_subs.append((cstr, cnum, 'expanded', wcs.get_subimage(-margin, -margin, w+2*margin, h+2*margin)))
rois.append(('expanded', -margin, -margin, w + margin, h + margin))
margin = 125
expwcs = wcs.get_subimage(-margin, -margin, w + 2 * margin,
h + 2 * margin)
newrois = []
for tag, x0, y0, x1, y1 in rois:
name = '%s_%i_%s' % (ctype.lower(), cnum, tag)
arr = np.zeros(len(tiles), (np.float32, Nbright))
tiles_ann.set('brightest_' + name, arr)
# (the rois have zero-indexed x0,y0, and non-inclusive x1,y1!)
newrois.append((name, arr, 1 + x0, 1 + y0, x1, y1))
gfa_regions.append((ctype, cnum, wcs, expwcs, newrois))
# DEBUG WCS
# s = []
# for ctype,cnum,wcs,expwcs,rois in gfa_regions:
# WCS = wcs
# h,w = WCS.shape
# #print('Expwcs:', w, 'x', h)
# x = [1,1,w,w,1]
# y = [1,h,h,1,1]
# r,d = WCS.pixelxy2radec(x, y)
# p = ','.join(['%.4f,%.4f' % (rr,dd) for rr,dd in zip(r,d)])
# s.append(p)
# s = ';'.join(s)
# print('http://legacysurvey.org/viewer/?ra=0&dec=0&poly='+s)
# sys.exit(0)
gaia = CachingGaiaCatalog(columns=[
'ra', 'dec', 'phot_g_mean_mag', 'phot_bp_mean_mag', 'phot_rp_mean_mag',
'astrometric_excess_noise', 'astrometric_params_solved', 'source_id',
'pmra_error', 'pmdec_error', 'parallax_error', 'ra_error', 'dec_error',
'pmra', 'pmdec', 'parallax', 'ref_epoch'
])
tyc2fn = '/global/cfs/cdirs/cosmo/staging/tycho2/tycho2.kd.fits'
tycho_kd = tree_open(tyc2fn)
tycho_cat = fits_table(tyc2fn)
maxrad = maxr * 1.05
for itile, tile in enumerate(tiles):
#if not tile.in_imaging:
# continue
#if tile.centerid % 10 == 0:
#print('tile program', tile.program, 'pass', tile.get('pass'), 'id', tile.centerid, gaia.get_healpix_tree.cache_info())
I = tree_search_radec(tycho_kd, tile.ra, tile.dec, maxrad)
tycstars = tycho_cat[I]
fix_tycho(tycstars)
for cstr, cname, chipwcs, bigwcs, rois in gfa_regions:
h, w = chipwcs.shape
chipwcs.set_crval(tile.ra, tile.dec)
bigwcs.set_crval(tile.ra, tile.dec)
gstars = gaia.get_catalog_in_wcs(bigwcs, step=1032, margin=0)
fix_gaia(gstars)
bh, bw = bigwcs.shape
ok, x, y = bigwcs.radec2pixelxy(tycstars.ra, tycstars.dec)
tstars = tycstars[(x >= 1) * (y >= 1) * (x <= bw) * (y <= bh)]
#print('Tile', tile.program, 'p', tile.get('pass'), tile.centerid,
# 'GFA', cstr, cname, ':', len(gstars), 'Gaia stars', len(tstars), 'Tycho-2 stars')
if len(gstars) + len(tstars) == 0:
print('No stars in tile centerid', tile.centerid, 'chip', name)
continue
if len(gstars) > 0 and len(tstars) > 0:
merge_gaia_tycho(gstars, tstars)
stars = merge_tables([gstars, tstars], columns='fillzero')
elif len(tstars) > 0:
stars = tstars
else:
stars = gstars
ok, x, y = chipwcs.radec2pixelxy(stars.ra, stars.dec)
for name, arr, x0, y0, x1, y1 in rois:
J = np.flatnonzero(
(x >= x0) * (x <= x1) * (y >= y0) * (y <= y1))
mags = stars.mag[J]
#print(' ', len(mags), 'in name')
K = np.argsort(mags)
K = K[:Nbright]
arr[itile, :len(K)] = mags[K]
#tiles.add_columns_from(tiles_ann)
return tiles_ann
print('Kd data:', spherematch.tree_data(kd, np.array([0,3,5]).astype(np.uint32)))
print('Kd data:', kd.get_data(np.array([0,3,5]).astype(np.uint32)))
print('Permute:', spherematch.tree_permute(kd, np.array([3,5,7]).astype(np.int32)))
print('Permute:', kd.permute(np.array([0,99,199]).astype(np.int32)))
ra,dec = np.meshgrid(np.arange(0, 360), np.arange(-90, 91, 1))
ra1 = ra.ravel()
dec1 = dec.ravel()
rdkd1 = spherematch.tree_build_radec(ra1, dec1)
print('RdKd:', rdkd1.n, rdkd1.bbox)
ra2 = np.random.uniform(-10, 10, size=1000)
dec2 = np.random.uniform(-10, 10, size=1000)
rdkd2 = spherematch.tree_build_radec(ra2, dec2)
I = spherematch.tree_search_radec(rdkd1, ra2[0], dec2[0], 2.)
print('search_radec:', I)
I,J,d = spherematch.match_radec(ra1, dec1, ra2, dec2, 1.)
print('Matches:', len(I))
I,J,d = spherematch.match_radec(ra1, dec1, ra2, dec2, 1., nearest=True)
print('Nearest matches:', len(I))
I = spherematch.match_radec(ra1, dec1, ra2, dec2, 1.,
indexlist=True)
print('Index lists matches:', len(I))
def map_sdss(req, ver, zoom, x, y, savecache=None, tag='sdss',
get_images=False,
ignoreCached=False,
wcs=None,
forcecache=False,
forcescale=None,
**kwargs):
from decals import settings
if savecache is None:
savecache = settings.SAVE_CACHE
zoom = int(zoom)
zoomscale = 2.**zoom
x = int(x)
y = int(y)
if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale:
raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom,x,y))
ver = int(ver)
if not ver in tileversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
basedir = settings.DATA_DIR
tilefn = os.path.join(basedir, 'tiles', tag,
'%i/%i/%i/%i.jpg' % (ver, zoom, x, y))
if os.path.exists(tilefn) and not ignoreCached:
if get_images:
return None
return send_file(tilefn, 'image/jpeg', expires=oneyear,
modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'))
if not savecache:
import tempfile
f,tilefn = tempfile.mkstemp(suffix='.jpg')
os.close(f)
if wcs is None:
try:
wcs, W, H, zoomscale, zoom,x,y = get_tile_wcs(zoom, x, y)
except RuntimeError as e:
if get_images:
return None
return HttpResponse(e.strerror)
else:
W = wcs.get_width()
H = wcs.get_height()
from astrometry.util.fits import fits_table
import numpy as np
from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec
from astrometry.util.starutil_numpy import degrees_between, arcsec_between
from astrometry.util.resample import resample_with_wcs, OverlapError
from astrometry.util.util import Tan, Sip
import fitsio
print 'Tile wcs: center', wcs.radec_center(), 'pixel scale', wcs.pixel_scale()
global w_flist
global w_flist_tree
if w_flist is None:
w_flist = fits_table(os.path.join(settings.DATA_DIR, 'sdss',
'window_flist.fits'),
columns=['run','rerun','camcol','field','ra','dec','score'])
print 'Read', len(w_flist), 'window_flist entries'
w_flist.cut(w_flist.rerun == '301')
print 'Cut to', len(w_flist), 'in rerun 301'
w_flist_tree = tree_build_radec(w_flist.ra, w_flist.dec)
# SDSS field size
radius = 1.01 * np.hypot(10., 14.)/2. / 60.
# leaflet tile size
ra,dec = wcs.pixelxy2radec(W/2., H/2.)[-2:]
r0,d0 = wcs.pixelxy2radec(1, 1)[-2:]
r1,d1 = wcs.pixelxy2radec(W, H)[-2:]
radius = radius + max(degrees_between(ra,dec, r0,d0), degrees_between(ra,dec, r1,d1))
J = tree_search_radec(w_flist_tree, ra, dec, radius)
print len(J), 'overlapping SDSS fields found'
if len(J) == 0:
if get_images:
return None
if forcecache:
# create symlink to blank.jpg!
trymakedirs(tilefn)
src = os.path.join(settings.STATIC_ROOT, 'blank.jpg')
if os.path.exists(tilefn):
os.unlink(tilefn)
os.symlink(src, tilefn)
print 'Symlinked', tilefn, '->', src
from django.http import HttpResponseRedirect
return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg')
ww = [1, W*0.25, W*0.5, W*0.75, W]
hh = [1, H*0.25, H*0.5, H*0.75, H]
r,d = wcs.pixelxy2radec(
[1]*len(hh) + ww + [W]*len(hh) + list(reversed(ww)),
hh + [1]*len(ww) + list(reversed(hh)) + [H]*len(ww))[-2:]
scaled = 0
scalepat = None
scaledir = 'sdss'
if zoom <= 13 and forcescale is None:
# Get *actual* pixel scales at the top & bottom
r1,d1 = wcs.pixelxy2radec(W/2., H)[-2:]
r2,d2 = wcs.pixelxy2radec(W/2., H-1.)[-2:]
r3,d3 = wcs.pixelxy2radec(W/2., 1.)[-2:]
r4,d4 = wcs.pixelxy2radec(W/2., 2.)[-2:]
# Take the min = most zoomed-in
scale = min(arcsec_between(r1,d1, r2,d2), arcsec_between(r3,d3, r4,d4))
native_scale = 0.396
scaled = int(np.floor(np.log2(scale / native_scale)))
print 'Zoom:', zoom, 'x,y', x,y, 'Tile pixel scale:', scale, 'Scale step:', scaled
scaled = np.clip(scaled, 1, 7)
dirnm = os.path.join(basedir, 'scaled', scaledir)
scalepat = os.path.join(dirnm, '%(scale)i%(band)s', '%(rerun)s', '%(run)i', '%(camcol)i', 'sdss-%(run)i-%(camcol)i-%(field)i-%(band)s.fits')
if forcescale is not None:
scaled = forcescale
bands = 'gri'
rimgs = [np.zeros((H,W), np.float32) for band in bands]
rns = [np.zeros((H,W), np.float32) for band in bands]
from astrometry.sdss import AsTransWrapper, DR9
sdss = DR9(basedir=settings.SDSS_DIR)
sdss.saveUnzippedFiles(settings.SDSS_DIR)
#sdss.setFitsioReadBZ2()
if settings.SDSS_PHOTOOBJS:
sdss.useLocalTree(photoObjs=settings.SDSS_PHOTOOBJS,
resolve=settings.SDSS_RESOLVE)
for jnum,j in enumerate(J):
print 'SDSS field', jnum, 'of', len(J), 'for zoom', zoom, 'x', x, 'y', y
im = w_flist[j]
if im.score >= 0.5:
weight = 1.
else:
weight = 0.001
for band,rimg,rn in zip(bands, rimgs, rns):
if im.rerun != '301':
continue
tmpsuff = '.tmp%08i' % np.random.randint(100000000)
basefn = sdss.retrieve('frame', im.run, im.camcol, field=im.field,
band=band, rerun=im.rerun, tempsuffix=tmpsuff)
if scaled > 0:
fnargs = dict(band=band, rerun=im.rerun, run=im.run,
camcol=im.camcol, field=im.field)
fn = get_scaled(scalepat, fnargs, scaled, basefn,
read_base_wcs=read_astrans, read_wcs=_read_sip_wcs)
print 'get_scaled:', fn
else:
fn = basefn
frame = None
if fn == basefn:
frame = sdss.readFrame(im.run, im.camcol, im.field, band,
filename=fn)
h,w = frame.getImageShape()
# Trim off the overlapping top of the image
# Wimp out and instead of trimming 128 pix, trim 124!
trim = 124
subh = h - trim
astrans = frame.getAsTrans()
fwcs = AsTransWrapper(astrans, w, subh)
fullimg = frame.getImage()
fullimg = fullimg[:-trim,:]
else:
fwcs = Sip(fn)
fitsimg = fitsio.FITS(fn)[0]
h,w = fitsimg.get_info()['dims']
fullimg = fitsimg.read()
try:
#Yo,Xo,Yi,Xi,nil = resample_with_wcs(wcs, fwcs, [], 3)
Yo,Xo,Yi,Xi,[resamp] = resample_with_wcs(wcs, fwcs, [fullimg], 2)
except OverlapError:
continue
if len(Xi) == 0:
#print 'No overlap'
continue
if sdssps is not None:
x0 = Xi.min()
x1 = Xi.max()
y0 = Yi.min()
y1 = Yi.max()
slc = (slice(y0,y1+1), slice(x0,x1+1))
if frame is not None:
img = frame.getImageSlice(slc)
else:
img = fitsimg[slc]
#rimg[Yo,Xo] += img[Yi-y0, Xi-x0]
rimg[Yo,Xo] += resamp * weight
rn [Yo,Xo] += weight
if sdssps is not None:
# goodpix = np.ones(img.shape, bool)
# fpM = sdss.readFpM(im.run, im.camcol, im.field, band)
# for plane in [ 'INTERP', 'SATUR', 'CR', 'GHOST' ]:
# fpM.setMaskedPixels(plane, goodpix, False, roi=[x0,x1,y0,y1])
plt.clf()
#ima = dict(vmin=-0.05, vmax=0.5)
#ima = dict(vmin=-0.5, vmax=2.)
ima = dict(vmax=np.percentile(img, 99))
plt.subplot(2,3,1)
dimshow(img, ticks=False, **ima)
plt.title('image')
rthis = np.zeros_like(rimg)
#rthis[Yo,Xo] += img[Yi-y0, Xi-x0]
rthis[Yo,Xo] += resamp
plt.subplot(2,3,2)
dimshow(rthis, ticks=False, **ima)
plt.title('resampled')
# plt.subplot(2,3,3)
# dimshow(goodpix, ticks=False, vmin=0, vmax=1)
# plt.title('good pix')
plt.subplot(2,3,4)
dimshow(rimg / np.maximum(rn, 1), ticks=False, **ima)
plt.title('coadd')
plt.subplot(2,3,5)
dimshow(rn, vmin=0, ticks=False)
plt.title('coverage: max %i' % rn.max())
plt.subplot(2,3,6)
rgb = sdss_rgb([rimg/np.maximum(rn,1)
for rimg,rn in zip(rimgs,rns)], bands)
dimshow(rgb)
plt.suptitle('SDSS %s, R/C/F %i/%i/%i' % (band, im.run, im.camcol, im.field))
sdssps.savefig()
for rimg,rn in zip(rimgs, rns):
rimg /= np.maximum(rn, 1e-3)
del rns
if get_images:
return rimgs
rgb = sdss_rgb(rimgs, bands)
trymakedirs(tilefn)
save_jpeg(tilefn, rgb)
print 'Wrote', tilefn
return send_file(tilefn, 'image/jpeg', unlink=(not savecache))
def read_tycho2(survey, targetwcs):
from astrometry.libkd.spherematch import tree_open, tree_search_radec
tycho2fn = survey.find_file('tycho2')
radius = 1.
ra, dec = targetwcs.radec_center()
# John added the "isgalaxy" flag 2018-05-10, from the Metz &
# Geffert (04) catalog.
# Eddie added the "zguess" column 2019-03-06, by matching with
# 2MASS and estimating z based on APASS.
# The "tycho2.kd.fits" file read here was produced by:
#
# fitscopy ~schlafly/legacysurvey/tycho-isgalaxyflag-2mass.fits"[col \
# tyc1;tyc2;tyc3;ra;dec;sigma_ra;sigma_dec;mean_ra;mean_dec;pm_ra;pm_dec; \
# sigma_pm_ra;sigma_pm_dec;epoch_ra;epoch_dec;mag_bt;mag_vt;mag_hp; \
# isgalaxy;Jmag;Hmag;Kmag,zguess]" /tmp/tycho2-astrom.fits
# startree -P -k -n stars -T -i /tmp/tycho2-astrom.fits \
# -o /global/project/projectdirs/cosmo/staging/tycho2/tycho2.kd.fits
kd = tree_open(tycho2fn, 'stars')
I = tree_search_radec(kd, ra, dec, radius)
debug(len(I), 'Tycho-2 stars within', radius,
'deg of RA,Dec (%.3f, %.3f)' % (ra, dec))
if len(I) == 0:
return None
# Read only the rows within range.
tycho = fits_table(tycho2fn, rows=I)
del kd
if 'isgalaxy' in tycho.get_columns():
tycho.cut(tycho.isgalaxy == 0)
debug('Cut to', len(tycho), 'Tycho-2 stars on isgalaxy==0')
else:
print('Warning: no "isgalaxy" column in Tycho-2 catalog')
tycho.ref_cat = np.array(['T2'] * len(tycho))
# tyc1: [1,9537], tyc2: [1,12121], tyc3: [1,3]
tycho.ref_id = (tycho.tyc1.astype(np.int64) * 1000000 +
tycho.tyc2.astype(np.int64) * 10 +
tycho.tyc3.astype(np.int64))
with np.errstate(divide='ignore'):
tycho.pmra_ivar = 1. / tycho.sigma_pm_ra**2
tycho.pmdec_ivar = 1. / tycho.sigma_pm_dec**2
tycho.ra_ivar = 1. / tycho.sigma_ra**2
tycho.dec_ivar = 1. / tycho.sigma_dec**2
tycho.rename('pm_ra', 'pmra')
tycho.rename('pm_dec', 'pmdec')
for c in ['pmra', 'pmdec', 'pmra_ivar', 'pmdec_ivar']:
X = tycho.get(c)
X[np.logical_not(np.isfinite(X))] = 0.
tycho.mag = tycho.mag_vt
# Patch missing mag values...
tycho.mag[tycho.mag == 0] = tycho.mag_hp[tycho.mag == 0]
tycho.mag[tycho.mag == 0] = tycho.mag_bt[tycho.mag == 0]
# Per discussion in issue #306 -- cut on mag < 13. This drops only 13k/2.5M stars
tycho.cut(tycho.mag < 13.)
# See note on gaia.radius above -- don't change the 0.262 to
# targetwcs.pixel_scale()!
tycho.radius = np.minimum(1800., 150. * 2.5**(
(11. - tycho.mag) / 3.)) * 0.262 / 3600.
for c in [
'tyc1', 'tyc2', 'tyc3', 'mag_bt', 'mag_vt', 'mag_hp', 'mean_ra',
'mean_dec', 'sigma_pm_ra', 'sigma_pm_dec', 'sigma_ra', 'sigma_dec'
]:
tycho.delete_column(c)
# add Gaia-style columns
# No parallaxes in Tycho-2
tycho.parallax = np.zeros(len(tycho), np.float32)
# Tycho-2 has separate epoch_ra and epoch_dec.
# Move source to the mean epoch.
tycho.ref_epoch = (tycho.epoch_ra + tycho.epoch_dec) / 2.
cosdec = np.cos(np.deg2rad(tycho.dec))
tycho.ra += (tycho.ref_epoch -
tycho.epoch_ra) * tycho.pmra / 3600. / cosdec
tycho.dec += (tycho.ref_epoch - tycho.epoch_dec) * tycho.pmdec / 3600.
# Tycho-2 proper motions are in arcsec/yr; Gaia are mas/yr.
tycho.pmra *= 1000.
tycho.pmdec *= 1000.
# We already cut on John's "isgalaxy" flag
tycho.pointsource = np.ones(len(tycho), bool)
# phot_g_mean_mag -- for initial brightness of source
tycho.phot_g_mean_mag = tycho.mag
tycho.delete_column('epoch_ra')
tycho.delete_column('epoch_dec')
tycho.isbright = np.ones(len(tycho), bool)
tycho.ismedium = np.ones(len(tycho), bool)
return tycho
def read_large_galaxies(survey,
targetwcs,
bands,
clean_columns=True,
max_radius=2.):
# Note, max_radius must include the brick radius!
from astrometry.libkd.spherematch import tree_open, tree_search_radec
galfn = survey.find_file('large-galaxies')
if galfn is None:
debug('No large-galaxies catalog file')
return None
radius = max_radius
rc, dc = targetwcs.radec_center()
debug('Reading', galfn)
try:
kd = tree_open(galfn, 'stars')
except:
kd = tree_open(galfn, 'largegals')
I = tree_search_radec(kd, rc, dc, radius)
debug('%i large galaxies within %.3g deg of RA,Dec (%.3f, %.3f)' %
(len(I), radius, rc, dc))
if len(I) == 0:
return None
# Read only the rows within range.
galaxies = fits_table(galfn, rows=I)
del kd
refcat, preburn = get_large_galaxy_version(galfn)
debug('Large galaxies version: "%s", preburned?' % refcat, preburn)
if preburn:
# SGA ellipse catalog
# NOTE: fields such as ref_cat, preburned, etc, already exist in the
# "galaxies" catalog read from disk.
# The galaxies we want to appear in MASKBITS get
# 'islargegalaxy' set. This includes both pre-burned
# galaxies, and ones where the preburning failed and we want
# to fall back to the SGA-parent ellipse for masking.
galaxies.islargegalaxy = ((galaxies.ref_cat == refcat) *
(galaxies.sga_id > -1))
# The pre-fit galaxies whose parameters will stay fixed
galaxies.freezeparams = (galaxies.preburned * galaxies.freeze)
else:
# SGA parent catalog
galaxies.ref_cat = np.array([refcat] * len(galaxies))
galaxies.islargegalaxy = np.ones(len(galaxies), bool)
galaxies.freezeparams = np.zeros(len(galaxies), bool)
galaxies.preburned = np.zeros(len(galaxies), bool)
galaxies.rename('sga_id', 'ref_id')
galaxies.rename('mag_leda', 'mag')
# Pre-burned, frozen but non-SGA sources have diam=-1.
galaxies.radius = np.maximum(0., galaxies.diam / 2. / 60.) # [degree]
galaxies.keep_radius = 2. * galaxies.radius
galaxies.sources = np.empty(len(galaxies), object)
galaxies.sources[:] = None
if bands is not None:
galaxies.sources[:] = get_galaxy_sources(galaxies, bands)
if clean_columns:
keep_columns = [
'ra', 'dec', 'radius', 'mag', 'ref_cat', 'ref_id', 'ba', 'pa',
'sources', 'islargegalaxy', 'freezeparams', 'keep_radius'
]
for c in galaxies.get_columns():
if not c in keep_columns:
galaxies.delete_column(c)
return galaxies
def read_tycho2(survey, targetwcs, bands):
from astrometry.libkd.spherematch import tree_open, tree_search_radec
from legacypipe.survey import GaiaSource
tycho2fn = survey.find_file('tycho2')
radius = 1.
ra, dec = targetwcs.radec_center()
# John added the "isgalaxy" flag 2018-05-10, from the Metz &
# Geffert (04) catalog.
# Eddie added the "zguess" column 2019-03-06, by matching with
# 2MASS and estimating z based on APASS.
# The "tycho2.kd.fits" file read here was produced by:
#
# fitscopy ~schlafly/legacysurvey/tycho-isgalaxyflag-2mass.fits"[col \
# tyc1;tyc2;tyc3;ra;dec;sigma_ra;sigma_dec;mean_ra;mean_dec;pm_ra;pm_dec; \
# sigma_pm_ra;sigma_pm_dec;epoch_ra;epoch_dec;mag_bt;mag_vt;mag_hp; \
# isgalaxy;Jmag;Hmag;Kmag,zguess]" /tmp/tycho2-astrom.fits
# startree -P -k -n stars -T -i /tmp/tycho2-astrom.fits \
# -o /global/project/projectdirs/cosmo/staging/tycho2/tycho2.kd.fits
kd = tree_open(tycho2fn, 'stars')
I = tree_search_radec(kd, ra, dec, radius)
debug(len(I), 'Tycho-2 stars within', radius,
'deg of RA,Dec (%.3f, %.3f)' % (ra, dec))
if len(I) == 0:
return None
# Read only the rows within range.
tycho = fits_table(tycho2fn, rows=I)
del kd
if 'isgalaxy' in tycho.get_columns():
tycho.cut(tycho.isgalaxy == 0)
debug('Cut to', len(tycho), 'Tycho-2 stars on isgalaxy==0')
else:
print('Warning: no "isgalaxy" column in Tycho-2 catalog')
tycho.ref_cat = np.array(['T2'] * len(tycho))
# tyc1: [1,9537], tyc2: [1,12121], tyc3: [1,3]
tycho.ref_id = (tycho.tyc1.astype(np.int64) * 1000000 +
tycho.tyc2.astype(np.int64) * 10 +
tycho.tyc3.astype(np.int64))
with np.errstate(divide='ignore'):
# In our Tycho-2 catalog, sigma_pm_* are in *arcsec/yr*, Gaia is in mas/yr.
tycho.pmra_ivar = 1. / (tycho.sigma_pm_ra * 1000.)**2
tycho.pmdec_ivar = 1. / (tycho.sigma_pm_dec * 1000.)**2
tycho.ra_ivar = 1. / tycho.sigma_ra**2
tycho.dec_ivar = 1. / tycho.sigma_dec**2
tycho.rename('pm_ra', 'pmra')
tycho.rename('pm_dec', 'pmdec')
for c in ['pmra', 'pmdec', 'pmra_ivar', 'pmdec_ivar']:
X = tycho.get(c)
X[np.logical_not(np.isfinite(X))] = 0.
tycho.mag = tycho.mag_vt
# Patch missing mag values...
tycho.mag[tycho.mag == 0] = tycho.mag_hp[tycho.mag == 0]
tycho.mag[tycho.mag == 0] = tycho.mag_bt[tycho.mag == 0]
# Use zguess
tycho.mask_mag = tycho.mag
with np.errstate(invalid='ignore'):
I = np.flatnonzero(
np.isfinite(tycho.zguess) * (tycho.zguess + 1. < tycho.mag))
tycho.mask_mag[I] = tycho.zguess[I]
# Per discussion in issue #306 -- cut on mag < 13.
# This drops only 13k/2.5M stars.
tycho.cut(tycho.mask_mag < 13.)
tycho.radius = mask_radius_for_mag(tycho.mask_mag)
tycho.keep_radius = 2. * tycho.radius
for c in [
'tyc1', 'tyc2', 'tyc3', 'mag_bt', 'mag_vt', 'mag_hp', 'mean_ra',
'mean_dec', 'sigma_pm_ra', 'sigma_pm_dec', 'sigma_ra', 'sigma_dec'
]:
tycho.delete_column(c)
# add Gaia-style columns
# No parallaxes in Tycho-2
tycho.parallax = np.zeros(len(tycho), np.float32)
# Tycho-2 "supplement" stars, from Hipparcos and Tycho-1 catalogs, have
# ref_epoch = 0. Fill in with the 1991.25 epoch of those catalogs.
tycho.epoch_ra[tycho.epoch_ra == 0] = 1991.25
tycho.epoch_dec[tycho.epoch_dec == 0] = 1991.25
# Tycho-2 has separate epoch_ra and epoch_dec.
# Move source to the mean epoch.
tycho.ref_epoch = (tycho.epoch_ra + tycho.epoch_dec) / 2.
cosdec = np.cos(np.deg2rad(tycho.dec))
tycho.ra += (tycho.ref_epoch -
tycho.epoch_ra) * tycho.pmra / 3600. / cosdec
tycho.dec += (tycho.ref_epoch - tycho.epoch_dec) * tycho.pmdec / 3600.
# Tycho-2 proper motions are in arcsec/yr; Gaia are mas/yr.
tycho.pmra *= 1000.
tycho.pmdec *= 1000.
# We already cut on John's "isgalaxy" flag
tycho.pointsource = np.ones(len(tycho), bool)
# phot_g_mean_mag -- for initial brightness of source
tycho.phot_g_mean_mag = tycho.mag
tycho.delete_column('epoch_ra')
tycho.delete_column('epoch_dec')
tycho.istycho = np.ones(len(tycho), bool)
tycho.isbright = np.ones(len(tycho), bool)
tycho.ismedium = np.ones(len(tycho), bool)
tycho.donotfit = np.zeros(len(tycho), bool)
tycho.sources = np.empty(len(tycho), object)
if bands is not None:
for i, t in enumerate(tycho):
tycho.sources[i] = GaiaSource.from_catalog(t, bands)
return tycho
def map_decam_depth(req, ver, zoom, x, y, savecache=False, band=None,
ignoreCached=False):
global Tdepth
global Tdepthkd
if band is None:
band = req.GET.get('band')
if not band in ['g','r','z']:
raise RuntimeError('Invalid band')
tag = 'decam-depth-%s' % band
zoom = int(zoom)
zoomscale = 2.**zoom
x = int(x)
y = int(y)
if zoom < 0 or x < 0 or y < 0 or x >= zoomscale or y >= zoomscale:
raise RuntimeError('Invalid zoom,x,y %i,%i,%i' % (zoom,x,y))
ver = int(ver)
if not ver in tileversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
basedir = settings.DATA_DIR
tilefn = os.path.join(basedir, 'tiles', tag,
'%i/%i/%i/%i.jpg' % (ver, zoom, x, y))
if os.path.exists(tilefn) and not ignoreCached:
print 'Cached:', tilefn
return send_file(tilefn, 'image/jpeg', expires=oneyear,
modsince=req.META.get('HTTP_IF_MODIFIED_SINCE'))
from astrometry.util.util import Tan
from astrometry.libkd.spherematch import match_radec
from astrometry.libkd.spherematch import tree_build_radec, tree_search_radec
from astrometry.util.fits import fits_table
from astrometry.util.starutil_numpy import degrees_between
import numpy as np
import fitsio
try:
wcs, W, H, zoomscale, zoom,x,y = get_tile_wcs(zoom, x, y)
except RuntimeError as e:
return HttpResponse(e.strerror)
rlo,d = wcs.pixelxy2radec(W, H/2)[-2:]
rhi,d = wcs.pixelxy2radec(1, H/2)[-2:]
r,d1 = wcs.pixelxy2radec(W/2, 1)[-2:]
r,d2 = wcs.pixelxy2radec(W/2, H)[-2:]
r,d = wcs.pixelxy2radec(W/2, H/2)[-2:]
rad = max(degrees_between(r, d, rlo, d1),
degrees_between(r, d, rhi, d2))
if Tdepth is None:
T = fits_table(os.path.join(basedir, 'decals-zpt-nondecals.fits'),
columns=['ccdra','ccddec','arawgain', 'avsky',
'ccdzpt', 'filter', 'crpix1','crpix2',
'crval1','crval2','cd1_1','cd1_2',
'cd2_1','cd2_2', 'naxis1', 'naxis2', 'exptime', 'fwhm'])
T.rename('ccdra', 'ra')
T.rename('ccddec', 'dec')
Tdepth = {}
Tdepthkd = {}
for b in ['g','r','z']:
Tdepth[b] = T[T.filter == b]
Tdepthkd[b] = tree_build_radec(Tdepth[b].ra, Tdepth[b].dec)
T = Tdepth[band]
Tkd = Tdepthkd[band]
#I,J,d = match_radec(T.ra, T.dec, r, d, rad + 0.2)
I = tree_search_radec(Tkd, r, d, rad + 0.2)
print len(I), 'CCDs in range'
if len(I) == 0:
from django.http import HttpResponseRedirect
return HttpResponseRedirect(settings.STATIC_URL + 'blank.jpg')
depthiv = np.zeros((H,W), np.float32)
for t in T[I]:
twcs = Tan(*[float(x) for x in [
t.crval1, t.crval2, t.crpix1, t.crpix2,
t.cd1_1, t.cd1_2, t.cd2_1, t.cd2_2, t.naxis1, t.naxis2]])
w,h = t.naxis1, t.naxis2
r,d = twcs.pixelxy2radec([1,1,w,w], [1,h,h,1])
ok,x,y = wcs.radec2pixelxy(r, d)
#print 'x,y coords of CCD:', x, y
x0 = int(x.min())
x1 = int(x.max())
y0 = int(y.min())
y1 = int(y.max())
if y1 < 0 or x1 < 0 or x0 >= W or y0 >= H:
continue
readnoise = 10. # e-; 7.0 to 15.0 according to DECam Data Handbook
skysig = np.sqrt(t.avsky * t.arawgain + readnoise**2) / t.arawgain
zpscale = 10.**((t.ccdzpt - 22.5)/2.5) * t.exptime
sig1 = skysig / zpscale
psf_sigma = t.fwhm / 2.35
# point-source depth
psfnorm = 1./(2. * np.sqrt(np.pi) * psf_sigma)
detsig1 = sig1 / psfnorm
#print '5-sigma point-source depth:', NanoMaggies.nanomaggiesToMag(detsig1 * 5.)
div = 1 / detsig1**2
depthiv[max(y0,0):min(y1,H), max(x0,0):min(x1,W)] += div
ptsrc = -2.5 * (np.log10(np.sqrt(1./depthiv) * 5) - 9)
ptsrc[depthiv == 0] = 0.
if savecache:
trymakedirs(tilefn)
else:
import tempfile
f,tilefn = tempfile.mkstemp(suffix='.jpg')
os.close(f)
import pylab as plt
plt.imsave(tilefn, ptsrc, vmin=22., vmax=25., cmap='hot')#nipy_spectral')
return send_file(tilefn, 'image/jpeg', unlink=(not savecache))