def read_large_galaxies(survey, targetwcs):
from astrometry.libkd.spherematch import tree_open, tree_search_radec
galfn = survey.find_file('large-galaxies')
radius = 1.
rc, dc = targetwcs.radec_center()
kd = tree_open(galfn, 'largegals')
I = tree_search_radec(kd, rc, dc, radius)
debug(len(I), 'large galaxies within', radius,
'deg of RA,Dec (%.3f, %.3f)' % (rc, dc))
if len(I) == 0:
return None
# Read only the rows within range.
galaxies = fits_table(
galfn,
rows=I,
columns=['ra', 'dec', 'd25', 'mag', 'lslga_id', 'ba', 'pa'])
del kd
# # D25 is diameter in arcmin
galaxies.radius = galaxies.d25 / 2. / 60.
# John told me to do this...
#galaxies.radius *= 1.2 ...and then John taketh away.
galaxies.delete_column('d25')
galaxies.rename('lslga_id', 'ref_id')
galaxies.ref_cat = np.array(['L2'] * len(galaxies))
galaxies.islargegalaxy = np.ones(len(galaxies), bool)
return galaxies
def match_kdtree_catalog(wcs, catfn):
from astrometry.libkd.spherematch import (
tree_open,
tree_close,
tree_build_radec,
tree_free,
trees_match,
tree_permute,
)
from astrometry.libkd import spherematch_c
from astrometry.util.starutil_numpy import deg2dist, xyztoradec
import numpy as np
import sys
rc, dc = wcs.get_center()
rr = wcs.get_radius()
kd = tree_open(catfn)
kd2 = tree_build_radec(np.array([rc]), np.array([dc]))
r = deg2dist(rr)
I, J, nil = trees_match(kd, kd2, r, permuted=False)
# HACK
# I2,J,d = trees_match(kd, kd2, r)
xyz = spherematch_c.kdtree_get_positions(kd, I.astype(np.uint32))
# print 'I', I
I2 = tree_permute(kd, I)
# print 'I2', I2
tree_free(kd2)
tree_close(kd)
tra, tdec = xyztoradec(xyz)
return tra, tdec, I2
def get_sky_template_filename(self, old_calibs_ok=False):
import os
from astrometry.util.fits import fits_table
dirnm = os.environ.get('SKY_TEMPLATE_DIR', None)
if dirnm is None:
info('decam: no SKY_TEMPLATE_DIR environment variable set.')
return None
'''
# Create this sky-scales.kd.fits file via:
python legacypipe/create-sky-template-kdtree.py skyscales_ccds.fits \
sky-scales.kd.fits
'''
fn = os.path.join(dirnm, 'sky-scales.kd.fits')
if not os.path.exists(fn):
info('decam: no $SKY_TEMPLATE_DIR/sky-scales.kd.fits file.')
return None
from astrometry.libkd.spherematch import tree_open
kd = tree_open(fn, 'expnum')
I = kd.search(np.array([self.expnum]), 0.5, 0, 0)
if len(I) == 0:
info('decam: expnum %i not found in file %s' % (self.expnum, fn))
return None
# Read only the CCD-table rows within range.
S = fits_table(fn, rows=I)
S.cut(np.array([c.strip() == self.ccdname for c in S.ccdname]))
if len(S) == 0:
info('decam: ccdname %s, expnum %i not found in file %s' %
(self.ccdname, self.expnum, fn))
return None
assert (len(S) == 1)
sky = S[0]
if sky.run == -1:
debug('sky template: run=-1 for expnum %i, ccdname %s' %
(self.expnum, self.ccdname))
return None
# Check PLPROCID only
if not validate_version(
fn, 'table', self.expnum, None, self.plprocid, data=S):
txt = (
'Sky template for expnum=%i, ccdname=%s did not pass consistency validation (EXPNUM, PLPROCID) -- image %i,%s vs template table %i,%s'
% (self.expnum, self.ccdname, self.expnum, self.plprocid,
sky.expnum, sky.plprocid))
if old_calibs_ok:
info('Warning:', txt, '-- but old_calibs_ok')
else:
raise RuntimeError(txt)
assert (self.band == sky.filter)
tfn = os.path.join(dirnm, 'sky_templates',
'sky_template_%s_%i.fits.fz' % (self.band, sky.run))
if not os.path.exists(tfn):
info('WARNING: Sky template file %s does not exist' % tfn)
return None
return dict(template_filename=tfn,
sky_template_dir=dirnm,
sky_obj=sky,
skyscales_fn=fn)
def match_kdtree_catalog(wcs, catfn):
from astrometry.libkd.spherematch import tree_open, tree_close, tree_build_radec, tree_free, trees_match, tree_permute
from astrometry.libkd import spherematch_c
from astrometry.util.starutil_numpy import deg2dist, xyztoradec
import numpy as np
import sys
rc,dc = wcs.get_center()
rr = wcs.get_radius()
kd = tree_open(catfn)
kd2 = tree_build_radec(np.array([rc]), np.array([dc]))
r = deg2dist(rr)
I,J,nil = trees_match(kd, kd2, r, permuted=False)
del kd2
xyz = kd.get_data(I.astype(np.uint32))
I = kd.permute(I)
del kd
tra,tdec = xyztoradec(xyz)
return tra, tdec, I
def match_kdtree_catalog(wcs, catfn):
from astrometry.libkd.spherematch import tree_open, tree_close, tree_build_radec, tree_free, trees_match, tree_permute
from astrometry.libkd import spherematch_c
from astrometry.util.starutil_numpy import deg2dist, xyztoradec
import numpy as np
import sys
rc,dc = wcs.get_center()
rr = wcs.get_radius()
kd = tree_open(catfn)
kd2 = tree_build_radec(np.array([rc]), np.array([dc]))
r = deg2dist(rr)
I,J,nil = trees_match(kd, kd2, r, permuted=False)
del kd2
xyz = kd.get_data(I.astype(np.uint32))
I = kd.permute(I)
del kd
tra,tdec = xyztoradec(xyz)
return tra, tdec, I
def cat_kd(req, ver, tag, fn):
tag = 'spec'
ralo = float(req.GET['ralo'])
rahi = float(req.GET['rahi'])
declo = float(req.GET['declo'])
dechi = float(req.GET['dechi'])
ver = int(ver)
if not ver in catversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
import numpy as np
from astrometry.util.fits import fits_table, merge_tables
from astrometry.libkd.spherematch import tree_open, tree_search_radec
from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between
xyz1 = radectoxyz(ralo, declo)
xyz2 = radectoxyz(rahi, dechi)
xyz = (xyz1 + xyz2) / 2.
xyz /= np.sqrt(np.sum(xyz**2))
rc, dc = xyztoradec(xyz)
rc = rc[0]
dc = dc[0]
rad = degrees_between(rc, dc, ralo, declo)
kd = tree_open(fn)
I = tree_search_radec(kd, rc, dc, rad)
print('Matched', len(I), 'from', fn)
if len(I) == 0:
return None
T = fits_table(fn, rows=I)
debug(len(T), 'spectra')
if ralo > rahi:
# RA wrap
T.cut(
np.logical_or(T.ra > ralo, T.ra < rahi) * (T.dec > declo) *
(T.dec < dechi))
else:
T.cut(
(T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi))
debug(len(T), 'in cut')
return T
def cat_kd(req, ver, tag, fn):
tag = 'spec'
ralo = float(req.GET['ralo'])
rahi = float(req.GET['rahi'])
declo = float(req.GET['declo'])
dechi = float(req.GET['dechi'])
ver = int(ver)
if not ver in catversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
import numpy as np
from astrometry.util.fits import fits_table, merge_tables
from astrometry.libkd.spherematch import tree_open, tree_search_radec
from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between
xyz1 = radectoxyz(ralo, declo)
xyz2 = radectoxyz(rahi, dechi)
xyz = (xyz1 + xyz2)/2.
xyz /= np.sqrt(np.sum(xyz**2))
rc,dc = xyztoradec(xyz)
rc = rc[0]
dc = dc[0]
rad = degrees_between(rc, dc, ralo, declo)
kd = tree_open(fn)
I = tree_search_radec(kd, rc, dc, rad)
print('Matched', len(I), 'from', fn)
if len(I) == 0:
return None
T = fits_table(fn, rows=I)
debug(len(T), 'spectra')
if ralo > rahi:
# RA wrap
T.cut(np.logical_or(T.ra > ralo, T.ra < rahi) * (T.dec > declo) * (T.dec < dechi))
else:
T.cut((T.ra > ralo) * (T.ra < rahi) * (T.dec > declo) * (T.dec < dechi))
debug(len(T), 'in cut')
return T
def match_kdtree_catalog(wcs, catfn):
from astrometry.libkd.spherematch import tree_open, tree_close, tree_build_radec, tree_free, trees_match, tree_permute
from astrometry.libkd import spherematch_c
from astrometry.util.starutil_numpy import deg2dist, xyztoradec
import numpy as np
import sys
rc, dc = wcs.get_center()
rr = wcs.get_radius()
kd = tree_open(catfn)
kd2 = tree_build_radec(np.array([rc]), np.array([dc]))
r = deg2dist(rr)
I, J, nil = trees_match(kd, kd2, r, permuted=False)
# HACK
#I2,J,d = trees_match(kd, kd2, r)
xyz = spherematch_c.kdtree_get_positions(kd, I.astype(np.uint32))
#print 'I', I
I2 = tree_permute(kd, I)
#print 'I2', I2
tree_free(kd2)
tree_close(kd)
tra, tdec = xyztoradec(xyz)
return tra, tdec, I2
def plot_wcs_outline(wcsfn,
plotfn,
W=256,
H=256,
width=36,
zoom=True,
zoomwidth=3.6,
grid=10,
hd=False,
hd_labels=False,
tycho2=False):
anutil.log_init(3)
#anutil.log_set_level(3)
wcs = anutil.Tan(wcsfn, 0)
ra, dec = wcs.radec_center()
plot = ps.Plotstuff(outformat='png', size=(W, H), rdw=(ra, dec, width))
plot.linestep = 1.
plot.color = 'verydarkblue'
plot.plot('fill')
plot.fontsize = 12
#plot.color = 'gray'
# dark gray
plot.rgb = (0.3, 0.3, 0.3)
if grid is not None:
plot.plot_grid(*([grid] * 4))
plot.rgb = (0.4, 0.6, 0.4)
ann = plot.annotations
ann.NGC = ann.bright = ann.HD = 0
ann.constellations = 1
ann.constellation_labels = 1
ann.constellation_labels_long = 1
plot.plot('annotations')
plot.stroke()
ann.constellation_labels = 0
ann.constellation_labels_long = 0
ann.constellation_lines = 0
ann.constellation_markers = 1
plot.markersize = 3
plot.rgb = (0.4, 0.6, 0.4)
plot.plot('annotations')
plot.fill()
ann.constellation_markers = 0
ann.bright_labels = False
ann.bright = True
plot.markersize = 2
if zoom >= 2:
ann.bright_labels = True
plot.plot('annotations')
ann.bright = False
ann.bright_labels = False
plot.fill()
if hd:
ann.HD = True
ann.HD_labels = hd_labels
ps.plot_annotations_set_hd_catalog(ann, settings.HENRY_DRAPER_CAT)
plot.plot('annotations')
plot.stroke()
ann.HD = False
ann.HD_labels = False
if tycho2 and settings.TYCHO2_KD:
from astrometry.libkd.spherematch import tree_open, tree_close, tree_build_radec, tree_free, trees_match
from astrometry.libkd import spherematch_c
from astrometry.util.starutil_numpy import deg2dist, xyztoradec
import numpy as np
import sys
kd = tree_open(settings.TYCHO2_KD)
# this is a bit silly: build a tree with a single point, then do match()
kd2 = tree_build_radec(np.array([ra]), np.array([dec]))
r = deg2dist(width * np.sqrt(2.) / 2.)
#r = deg2dist(wcs.radius())
I, nil, nil = trees_match(kd, kd2, r, permuted=False)
del nil
#print 'Matched', len(I)
xyz = spherematch_c.kdtree_get_positions(kd, I)
del I
tree_free(kd2)
tree_close(kd)
#print >>sys.stderr, 'Got', xyz.shape, xyz
tra, tdec = xyztoradec(xyz)
#print >>sys.stderr, 'RA,Dec', ra,dec
plot.apply_settings()
for r, d in zip(tra, tdec):
plot.marker_radec(r, d)
plot.fill()
ann.NGC = 1
plot.plot('annotations')
ann.NGC = 0
plot.color = 'white'
plot.lw = 3
out = plot.outline
out.wcs_file = wcsfn
plot.plot('outline')
if zoom:
# MAGIC width, height are arbitrary
zoomwcs = anutil.anwcs_create_box(ra, dec, zoomwidth, 1000, 1000)
out.wcs = zoomwcs
plot.lw = 1
plot.dashed(3)
plot.plot('outline')
plot.write(plotfn)
def plot_wcs_outline(wcsfn, plotfn, W=256, H=256, width=36, zoom=True,
zoomwidth=3.6, grid=10, hd=False, hd_labels=False,
tycho2=False):
anutil.log_init(3)
#anutil.log_set_level(3)
wcs = anutil.Tan(wcsfn, 0)
ra,dec = wcs.radec_center()
plot = ps.Plotstuff(outformat='png', size=(W, H), rdw=(ra,dec,width))
plot.linestep = 1.
plot.color = 'verydarkblue'
plot.plot('fill')
plot.fontsize = 12
#plot.color = 'gray'
# dark gray
plot.rgb = (0.3,0.3,0.3)
if grid is not None:
plot.plot_grid(*([grid]*4))
plot.rgb = (0.4, 0.6, 0.4)
ann = plot.annotations
ann.NGC = ann.bright = ann.HD = 0
ann.constellations = 1
ann.constellation_labels = 1
ann.constellation_labels_long = 1
plot.plot('annotations')
plot.stroke()
ann.constellation_labels = 0
ann.constellation_labels_long = 0
ann.constellation_lines = 0
ann.constellation_markers = 1
plot.markersize = 3
plot.rgb = (0.4, 0.6, 0.4)
plot.plot('annotations')
plot.fill()
ann.constellation_markers = 0
ann.bright_labels = False
ann.bright = True
plot.markersize = 2
if zoom >= 2:
ann.bright_labels = True
plot.plot('annotations')
ann.bright = False
ann.bright_labels = False
plot.fill()
if hd:
ann.HD = True
ann.HD_labels = hd_labels
ps.plot_annotations_set_hd_catalog(ann, settings.HENRY_DRAPER_CAT)
plot.plot('annotations')
plot.stroke()
ann.HD = False
ann.HD_labels = False
if tycho2:
from astrometry.libkd.spherematch import tree_open, tree_close, tree_build_radec, tree_free, trees_match
from astrometry.libkd import spherematch_c
from astrometry.util.starutil_numpy import deg2dist, xyztoradec
import numpy as np
import sys
kd = tree_open(settings.TYCHO2_KD)
# this is a bit silly: build a tree with a single point, then do match()
kd2 = tree_build_radec(np.array([ra]), np.array([dec]))
r = deg2dist(width * np.sqrt(2.) / 2.)
#r = deg2dist(wcs.radius())
I,J,d = trees_match(kd, kd2, r, permuted=False)
del J
del d
#print 'Matched', len(I)
xyz = spherematch_c.kdtree_get_positions(kd, I)
del I
tree_free(kd2)
tree_close(kd)
#print >>sys.stderr, 'Got', xyz.shape, xyz
tra,tdec = xyztoradec(xyz)
#print >>sys.stderr, 'RA,Dec', ra,dec
plot.apply_settings()
for r,d in zip(tra,tdec):
plot.marker_radec(r,d)
plot.fill()
ann.NGC = 1
plot.plot('annotations')
ann.NGC = 0
plot.color = 'white'
plot.lw = 3
out = plot.outline
out.wcs_file = wcsfn
plot.plot('outline')
if zoom:
# MAGIC width, height are arbitrary
zoomwcs = anutil.anwcs_create_box(ra, dec, zoomwidth, 1000,1000)
out.wcs = zoomwcs
plot.lw = 1
plot.dashed(3)
plot.plot('outline')
plot.write(plotfn)
def cat_targets_drAB(req,
ver,
cats=[],
tag='',
bgs=False,
sky=False,
bright=False,
dark=False,
color_name_func=desitarget_color_names):
'''
color_name_func: function that selects names and colors for targets
(eg based on targeting bit values)
'''
import json
ralo = float(req.GET['ralo'])
rahi = float(req.GET['rahi'])
declo = float(req.GET['declo'])
dechi = float(req.GET['dechi'])
ver = int(ver)
if not ver in catversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
from astrometry.util.fits import fits_table, merge_tables
from astrometry.libkd.spherematch import tree_open, tree_search_radec
import numpy as np
from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between
xyz1 = radectoxyz(ralo, declo)
xyz2 = radectoxyz(rahi, dechi)
xyz = (xyz1 + xyz2) / 2.
xyz /= np.sqrt(np.sum(xyz**2))
rc, dc = xyztoradec(xyz)
rc = rc[0]
dc = dc[0]
rad = degrees_between(rc, dc, ralo, declo)
'''
startree -i /project/projectdirs/desi/target/catalogs/targets-dr4-0.20.0.fits -o data/targets-dr4-0.20.0.kd.fits -P -k -T
'''
TT = []
for fn in cats:
kd = tree_open(fn)
I = tree_search_radec(kd, rc, dc, rad)
print('Matched', len(I), 'from', fn)
if len(I) == 0:
continue
T = fits_table(fn, rows=I)
TT.append(T)
if len(TT) == 0:
return HttpResponse(json.dumps(dict(rd=[], name=[])),
content_type='application/json')
T = merge_tables(TT, columns='fillzero')
if bgs:
T.cut(T.bgs_target > 0)
if bright:
T.cut(np.logical_or(T.bgs_target > 0, T.mws_target > 0))
if dark:
T.cut(T.desi_target > 0)
names = None
colors = None
if color_name_func is not None:
names, colors = color_name_func(T)
if sky:
fluxes = [
dict(g=float(g), r=float(r), z=float(z))
for (g, r, z) in zip(T.apflux_g[:,
0], T.apflux_r[:,
0], T.apflux_z[:,
0])
]
nobs = None
else:
fluxes = [
dict(g=float(g),
r=float(r),
z=float(z),
W1=float(W1),
W2=float(W2))
for (g, r, z, W1,
W2) in zip(T.flux_g, T.flux_r, T.flux_z, T.flux_w1, T.flux_w2)
]
nobs = [
dict(g=int(g), r=int(r), z=int(z))
for g, r, z in zip(T.nobs_g, T.nobs_r, T.nobs_z)
],
rtn = dict(
rd=[(t.ra, t.dec) for t in T],
targetid=[int(t) for t in T.targetid],
fluxes=fluxes,
)
if names is not None:
rtn.update(name=names)
if colors is not None:
rtn.update(color=colors)
if nobs is not None:
rtn.update(nobs=nobs)
return HttpResponse(json.dumps(rtn), content_type='application/json')
def in_footprint(parent, nside=2048, dr='dr9'):
"""Find all galaxies in the DESI footprint.
"""
import time
import healpy as hp
import legacyhalos.misc
#tiles = SGA.io.read_desi_tiles(verbose=verbose)
#indesi = SGA.misc.is_point_in_desi(tiles, parent['RA'], parent['DEC']).astype(bool)
parentpix = legacyhalos.misc.radec2pix(nside, parent['RA'], parent['DEC'])
#parentpix = np.hstack((parentpix, hp.pixelfunc.get_all_neighbours(nside, parentpix, nest=True).flatten()))
drdir = os.path.join(sample_dir(), dr)
bands = ('g', 'r', 'z')
camera = ('90prime', 'mosaic', 'decam')
indesi = dict()
for cam in camera:
for band in bands:
indesi.update(
{'{}_{}'.format(cam, band): np.zeros(len(parent), dtype=bool)})
#indesi = np.zeros(len(parent), dtype=bool)
t0 = time.time()
for cam, radius in zip(camera, (0.44, 0.21, 0.17)):
if False:
from astrometry.libkd.spherematch import trees_match, tree_open
kdccds = tree_open(
os.path.join(drdir,
'survey-ccds-{}-{}.kd.fits'.format(cam, dr)))
I, J, dd = trees_match(kdparent, kdccds,
np.radians(radius)) #, nearest=True)
else:
ccdsfile = os.path.join(
drdir, 'survey-ccds-{}-{}.kd.fits'.format(cam, dr))
ccds = fitsio.read(ccdsfile)
ccds = ccds[ccds['ccd_cuts'] == 0]
print('Read {} CCDs from {}'.format(len(ccds), ccdsfile))
for band in bands:
ww = ccds['filter'] == band
if np.sum(ww) > 0:
# add the neighboring healpixels to protect against edge effects
ccdpix = legacyhalos.misc.radec2pix(
nside, ccds['ra'][ww], ccds['dec'][ww])
ccdpix = np.hstack(
(ccdpix,
hp.pixelfunc.get_all_neighbours(nside,
ccdpix,
nest=True).flatten()))
if np.sum(
ccdpix == -1
) > 0: # remove the "no neighbors" healpixel, if it exists
ccdpix = np.delete(ccdpix, np.where(ccdpix == -1)[0])
I = np.isin(parentpix, ccdpix)
indesi['{}_{}'.format(cam, band)][I] = True
else:
I = [False]
#print('Found {} galaxies in {} {} footprint in {:.1f} sec'.format(np.sum(I), cam, time.time() - t0))
print(' Found {} galaxies in {} {} footprint.'.format(
np.sum(I), cam, band))
print('Total time to find galaxies in footprint = {:.1f} sec'.format(
time.time() - t0))
parent['IN_FOOTPRINT_NORTH'] = indesi['90prime_g'] | indesi[
'90prime_r'] | indesi['mosaic_z']
parent['IN_FOOTPRINT_NORTH_GRZ'] = indesi['90prime_g'] & indesi[
'90prime_r'] & indesi['mosaic_z']
parent['IN_FOOTPRINT_SOUTH'] = indesi['decam_g'] | indesi[
'decam_r'] | indesi['decam_z']
parent['IN_FOOTPRINT_SOUTH_GRZ'] = indesi['decam_g'] & indesi[
'decam_r'] & indesi['decam_z']
parent['IN_FOOTPRINT'] = parent['IN_FOOTPRINT_NORTH'] | parent[
'IN_FOOTPRINT_SOUTH']
parent['IN_FOOTPRINT_GRZ'] = parent['IN_FOOTPRINT_NORTH_GRZ'] | parent[
'IN_FOOTPRINT_SOUTH_GRZ']
#plt.scatter(parent['RA'], parent['DEC'], s=1)
#plt.scatter(parent['RA'][indesi], parent['DEC'][indesi], s=1)
#plt.xlim(360, 0)
#plt.show()
#bb = parent[parent['IN_FOOTPRINT_NORTH_GRZ'] & parent['IN_FOOTPRINT_SOUTH_GRZ']]
#plt.scatter(bb['RA'], bb['DEC'], s=1)
#plt.xlim(300, 90) ; plt.ylim(30, 36)
#plt.axhline(y=32.375, color='k')
#plt.xlabel('RA') ; plt.ylabel('Dec')
#plt.show()
print(
' Identified {}/{} ({:.2f}%) galaxies inside and {}/{} ({:.2f}%) galaxies outside the DESI footprint.'
.format(np.sum(parent['IN_FOOTPRINT']), len(parent),
100 * np.sum(parent['IN_FOOTPRINT']) / len(parent),
np.sum(~parent['IN_FOOTPRINT']), len(parent),
100 * np.sum(~parent['IN_FOOTPRINT']) / len(parent)))
return parent
T = fits_table(opt.abellcat) for i in range(len(T)): if not plot.wcs.is_inside(T.ra[i], T.dec[i]): continue ann.add_target(T.ra[i], T.dec[i], 'Abell %i' % T.aco[i]) if opt.t2cat: from astrometry.libkd.spherematch import tree_open, tree_close, tree_build_radec, tree_free, trees_match from astrometry.libkd import spherematch_c from astrometry.util.starutil_numpy import deg2dist, xyztoradec import numpy as np import sys wcs = plot.wcs rc,dc = wcs.get_center() rr = wcs.get_radius() kd = tree_open(opt.t2cat) kd2 = tree_build_radec(np.array([rc]), np.array([dc])) r = deg2dist(rr) I,J,d = trees_match(kd, kd2, r, permuted=False) # HACK I2,J,d = trees_match(kd, kd2, r) xyz = spherematch_c.kdtree_get_positions(kd, I) tree_free(kd2) tree_close(kd) tra,tdec = xyztoradec(xyz) T = fits_table(opt.t2cat, hdu=6) for r,d,t1,t2,t3 in zip(tra,tdec, T.tyc1[I2], T.tyc2[I2], T.tyc3[I2]): if not plot.wcs.is_inside(r, d): continue ann.add_target(r, d, 'Tycho-2 %i-%i-%i' % (t1,t2,t3))
def main():
import argparse
parser = argparse.ArgumentParser(
description='This script creates small self-contained data sets that '
'are useful for test cases of the pipeline codes.')
parser.add_argument('ccds', help='CCDs table describing region to grab')
parser.add_argument('outdir', help='Output directory name')
parser.add_argument('brick', help='Brick containing these images')
parser.add_argument('--survey-dir', type=str, default=None)
parser.add_argument('--cache-dir', type=str, default=None,
help='Directory to search for cached files')
parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image')
parser.add_argument('--wise-wcs-hdu', help='For WISE outputs, the HDU to read the WCS from in the file given by --wise.', type=int, default=0)
parser.add_argument('--fpack', action='store_true', default=False)
parser.add_argument('--gzip', action='store_true', default=False)
parser.add_argument('--pad', action='store_true', default=False,
help='Keep original image size, but zero out pixels outside ROI')
args = parser.parse_args()
v = 'SKY_TEMPLATE_DIR'
if v in os.environ:
del os.environ[v]
C = fits_table(args.ccds)
print(len(C), 'CCDs in', args.ccds)
C.camera = np.array([c.strip() for c in C.camera])
survey = LegacySurveyData(cache_dir=args.cache_dir, survey_dir=args.survey_dir)
if ',' in args.brick:
ra,dec = args.brick.split(',')
ra = float(ra)
dec = float(dec)
fakebricks = fits_table()
fakebricks.brickname = np.array([('custom-%06i%s%05i' %
(int(1000*ra), 'm' if dec < 0 else 'p',
int(1000*np.abs(dec))))])
fakebricks.ra = np.array([ra])
fakebricks.dec = np.array([dec])
bricks = fakebricks
outbricks = bricks
else:
bricks = survey.get_bricks_readonly()
outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])]
assert(len(outbricks) == 1)
outsurvey = LegacySurveyData(survey_dir = args.outdir)
trymakedirs(args.outdir)
outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz'))
targetwcs = wcs_for_brick(outbricks[0])
H,W = targetwcs.shape
tycho2fn = survey.find_file('tycho2')
kd = tree_open(tycho2fn, 'stars')
radius = 1.
rc,dc = targetwcs.radec_center()
I = tree_search_radec(kd, rc, dc, radius)
print(len(I), 'Tycho-2 stars within', radius, 'deg of RA,Dec (%.3f, %.3f)' % (rc,dc))
# Read only the rows within range.
tycho = fits_table(tycho2fn, rows=I)
del kd
print('Read', len(tycho), 'Tycho-2 stars')
ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec)
#margin = 100
#tycho.cut(ok * (tx > -margin) * (tx < W+margin) *
# (ty > -margin) * (ty < H+margin))
print('Cut to', len(tycho), 'Tycho-2 stars within brick')
del ok,tx,ty
#tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz'))
f,tfn = tempfile.mkstemp(suffix='.fits')
os.close(f)
tycho.writeto(tfn)
outfn = os.path.join(args.outdir, 'tycho2.kd.fits')
cmd = 'startree -i %s -o %s -P -k -n stars -T' % (tfn, outfn)
print(cmd)
rtn = os.system(cmd)
assert(rtn == 0)
os.unlink(tfn)
from legacypipe.gaiacat import GaiaCatalog
gcat = GaiaCatalog()
# from ps1cat.py:
wcs = targetwcs
step=100.
margin=10.
# Grid the CCD in pixel space
W,H = wcs.get_width(), wcs.get_height()
xx,yy = np.meshgrid(
np.linspace(1-margin, W+margin, 2+int((W+2*margin)/step)),
np.linspace(1-margin, H+margin, 2+int((H+2*margin)/step)))
# Convert to RA,Dec and then to unique healpixes
ra,dec = wcs.pixelxy2radec(xx.ravel(), yy.ravel())
healpixes = set()
for r,d in zip(ra,dec):
healpixes.add(gcat.healpix_for_radec(r, d))
for hp in healpixes:
hpcat = gcat.get_healpix_catalog(hp)
ok,xx,yy = wcs.radec2pixelxy(hpcat.ra, hpcat.dec)
onccd = np.flatnonzero((xx >= 1.-margin) * (xx <= W+margin) *
(yy >= 1.-margin) * (yy <= H+margin))
hpcat.cut(onccd)
if len(hpcat):
outfn = os.path.join(args.outdir, 'gaia', 'chunk-%05d.fits' % hp)
trymakedirs(os.path.join(args.outdir, 'gaia'))
hpcat.writeto(outfn)
outccds = C.copy()
cols = outccds.get_columns()
for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1',
'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1',
'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver',
'psfplver' ]:
if c in cols:
outccds.delete_column(c)
outccds.image_hdu[:] = 1
# Convert to list to avoid truncating filenames
outccds.image_filename = [fn for fn in outccds.image_filename]
for iccd,ccd in enumerate(C):
decam = (ccd.camera.strip() == 'decam')
bok = (ccd.camera.strip() == '90prime')
im = survey.get_image_object(ccd)
print('Got', im)
if survey.cache_dir is not None:
im.check_for_cached_files(survey)
slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1))
psfkwargs = dict(pixPsf=True, gaussPsf=False, hybridPsf=False,
normalizePsf=False)
tim = im.get_tractor_image(slc, pixPsf=True,
subsky=False, nanomaggies=False,
no_remap_invvar=True, old_calibs_ok=True)
print('Tim:', tim.shape)
psfrow = psfhdr = None
if args.pad:
psf = im.read_psf_model(0, 0, w=im.width, h=im.height, **psfkwargs)
psfex = psf.psfex
else:
psf = tim.getPsf()
psfex = psf.psfex
# Did the PSF model come from a merged file?
for fn in [im.merged_psffn, im.psffn] + im.old_merged_psffns:
if not os.path.exists(fn):
continue
T = fits_table(fn)
I, = np.nonzero((T.expnum == im.expnum) *
np.array([c.strip() == im.ccdname for c in T.ccdname]))
if len(I) != 1:
continue
psfrow = T[I]
x0 = ccd.ccd_x0
y0 = ccd.ccd_y0
psfrow.polzero1[0] -= x0
psfrow.polzero2[0] -= y0
#psfhdr = fitsio.read_header(im.merged_psffn)
break
psfex.fwhm = tim.psf_fwhm
#### HACK
#psfrow = None
assert(psfrow is not None)
if psfrow is not None:
print('PSF row:', psfrow)
#else:
# print('PSF:', psf)
# print('PsfEx:', psfex)
skyrow = skyhdr = None
if args.pad:
primhdr = fitsio.read_header(im.imgfn)
imghdr = fitsio.read_header(im.imgfn, hdu=im.hdu)
sky = im.read_sky_model(splinesky=True, primhdr=primhdr, imghdr=imghdr)
#skyhdr = fitsio.read_header(im.splineskyfn)
#msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True)
else:
sky = tim.getSky()
# Did the sky model come from a merged file?
#msky = im.read_merged_splinesky_model(slc=slc, old_calibs_ok=True)
print('merged skyfn:', im.merged_skyfn)
print('single skyfn:', im.skyfn)
print('old merged skyfns:', im.old_merged_skyfns)
for fn in [im.merged_skyfn, im.skyfn] + im.old_merged_skyfns:
if not os.path.exists(fn):
continue
T = fits_table(fn)
I, = np.nonzero((T.expnum == im.expnum) *
np.array([c.strip() == im.ccdname for c in T.ccdname]))
skyrow = T[I]
skyrow.x0[0] = ccd.ccd_x0
skyrow.y0[0] = ccd.ccd_y0
# s_med = skyrow.sky_med[0]
# s_john = skyrow.sky_john[0]
# skyhdr = fitsio.read_header(fn)
assert(skyrow is not None)
### HACK
#skyrow = None
if skyrow is not None:
print('Sky row:', skyrow)
else:
print('Sky:', sky)
# Output filename format:
fn = ccd.image_filename.strip()
ccd.image_filename = os.path.join(os.path.dirname(fn),
'%s.%s.fits' % (os.path.basename(fn).split('.')[0], ccd.ccdname.strip()))
outim = outsurvey.get_image_object(ccd)
print('Output image:', outim)
print('Image filename:', outim.imgfn)
trymakedirs(outim.imgfn, dir=True)
imgdata = tim.getImage()
ivdata = tim.getInvvar()
# Since we remap DQ codes (always with Mosaic and Bok, sometimes with DECam),
# re-read from the FITS file rather than using tim.dq.
print('Reading data quality from', im.dqfn, 'hdu', im.hdu)
dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice)
print('Tim shape:', tim.shape, 'Slice', tim.slice)
print('image shape:', imgdata.shape, 'iv', ivdata.shape, 'DQ', dqdata.shape)
from collections import Counter
dqvals = Counter(dqdata.ravel())
print('DQ pixel counts:')
for k,n in dqvals.most_common():
print(' 0x%x' % k, ':', n)
if args.pad:
# Create zero image of full size, copy in data.
fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype)
fullsize[slc] = imgdata
imgdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype)
fullsize[slc] = dqdata
dqdata = fullsize
fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype)
fullsize[slc] = ivdata
ivdata = fullsize
else:
# Adjust the header WCS by x0,y0
crpix1 = tim.hdr['CRPIX1']
crpix2 = tim.hdr['CRPIX2']
tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0
tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0
# Add image extension to filename
# fitsio doesn't compress .fz by default, so drop .fz suffix
#outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits')
if args.gzip:
outim.imgfn = outim.imgfn.replace('.fits', '.fits.gz')
#outim.wtfn = outim.wtfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.wtfn = outim.wtfn.replace('.fits.fz', '.fits')
if args.gzip:
outim.wtfn = outim.wtfn.replace('.fits', '.fits.gz')
if outim.dqfn is not None:
#outim.dqfn = outim.dqfn.replace('.fits', '-%s.fits' % im.ccdname)
if not args.fpack:
outim.dqfn = outim.dqfn.replace('.fits.fz', '.fits')
if args.gzip:
outim.dqfn = outim.dqfn.replace('.fits', '.fits.gz')
if bok:
outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname)
ccdfn = outim.imgfn
ccdfn = ccdfn.replace(outsurvey.get_image_dir(), '')
if ccdfn.startswith('/'):
ccdfn = ccdfn[1:]
outccds.image_filename[iccd] = ccdfn
print('Changed output filenames to:')
print(outim.imgfn)
print(outim.dqfn)
ofn = outim.imgfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fits = fitsio.FITS(ofn, 'rw', clobber=True)
fits.write(None, header=tim.primhdr)
fits.write(imgdata, header=tim.hdr, extname=ccd.ccdname)
fits.close()
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
h,w = tim.shape
if not args.pad:
outccds.width[iccd] = w
outccds.height[iccd] = h
outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0
outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0
wcs = Tan(*[float(x) for x in
[ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2,
ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]])
if args.pad:
subwcs = wcs
else:
subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h)
outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center()
print('Weight filename:', outim.wtfn)
wfn = outim.wtfn
trymakedirs(wfn, dir=True)
ofn = wfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fits = fitsio.FITS(ofn, 'rw', clobber=True)
fits.write(None, header=tim.primhdr)
fits.write(ivdata, header=tim.hdr, extname=ccd.ccdname)
fits.close()
if args.fpack:
cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
if outim.dqfn is not None:
print('DQ filename', outim.dqfn)
trymakedirs(outim.dqfn, dir=True)
ofn = outim.dqfn
if args.fpack:
f,ofn = tempfile.mkstemp(suffix='.fits')
os.close(f)
fits = fitsio.FITS(ofn, 'rw', clobber=True)
fits.write(None, header=tim.primhdr)
fits.write(dqdata, header=tim.hdr, extname=ccd.ccdname)
fits.close()
if args.fpack:
cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn)
print('Running:', cmd)
rtn = os.system(cmd)
assert(rtn == 0)
psfout = outim.psffn
#if psfrow:
# psfout = outim.merged_psffn
print('PSF output filename:', psfout)
trymakedirs(psfout, dir=True)
if psfrow:
psfrow.writeto(psfout, primhdr=psfhdr)
else:
print('Writing PsfEx:', psfout)
psfex.writeto(psfout)
# update header
F = fitsio.FITS(psfout, 'rw')
F[0].write_keys([dict(name='EXPNUM', value=ccd.expnum),
dict(name='PLVER', value=psf.plver),
dict(name='PROCDATE', value=psf.procdate),
dict(name='PLPROCID', value=psf.plprocid),])
F.close()
skyout = outim.skyfn
#if skyrow:
# skyout = outim.merged_splineskyfn
print('Sky output filename:', skyout)
trymakedirs(skyout, dir=True)
if skyrow is not None:
skyrow.writeto(skyout, primhdr=skyhdr)
else:
primhdr = fitsio.FITSHDR()
primhdr['PLVER'] = sky.plver
primhdr['PLPROCID'] = sky.plprocid
primhdr['PROCDATE'] = sky.procdate
primhdr['EXPNUM'] = ccd.expnum
primhdr['IMGDSUM'] = sky.datasum
primhdr['S_MED'] = s_med
primhdr['S_JOHN'] = s_john
sky.write_fits(skyout, primhdr=primhdr)
# HACK -- check result immediately.
outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))
outsurvey.ccds = None
outC = outsurvey.get_ccds_readonly()
occd = outC[iccd]
outim = outsurvey.get_image_object(occd)
print('Got output image:', outim)
otim = outim.get_tractor_image(pixPsf=True,
hybridPsf=True, old_calibs_ok=True)
print('Got output tim:', otim)
outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz'))
# WISE
if args.wise is not None:
from wise.forcedphot import unwise_tiles_touching_wcs
from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs,
get_unwise_tractor_image, get_unwise_tile_dir)
# Read WCS...
print('Reading TAN wcs header from', args.wise, 'HDU', args.wise_wcs_hdu)
targetwcs = Tan(args.wise, args.wise_wcs_hdu)
tiles = unwise_tiles_touching_wcs(targetwcs)
print('Cut to', len(tiles), 'unWISE tiles')
H,W = targetwcs.shape
r,d = targetwcs.pixelxy2radec(np.array([1, W, W/2, W/2]),
np.array([H/2, H/2, 1, H ]))
roiradec = [r[0], r[1], d[2], d[3]]
unwise_dir = os.environ['UNWISE_COADDS_DIR']
wise_out = os.path.join(args.outdir, 'images', 'unwise')
print('Will write WISE outputs to', wise_out)
unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR']
wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr')
print('Will write WISE time-resolved outputs to', wise_tr_out)
trymakedirs(wise_tr_out)
W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_atlas.fits'))
print('Read', len(W), 'time-resolved WISE coadd tiles')
W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id]))
print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth')
# Write the time-resolved index subset.
W.writeto(os.path.join(wise_tr_out, 'time_resolved_atlas.fits'))
# this ought to be enough for anyone =)
_,Nepochs = W.epoch_bitmask.shape
print('N epochs in time-resolved atlas:', Nepochs)
wisedata = []
# full depth
for band in [1,2,3,4]:
wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band, True))
# time-resolved
for band in [1,2]:
# W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2)
bitmask = (1 << (band-1))
for e in range(Nepochs):
# Which tiles have images for this epoch?
I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask)
if len(I) == 0:
continue
print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I])
edir = os.path.join(unwise_tr_dir, 'e%03i' % e)
eoutdir = os.path.join(wise_tr_out, 'e%03i' % e)
wisedata.append((edir, eoutdir, tiles.coadd_id[I], band, False))
wrote_masks = set()
model_dir = os.environ.get('UNWISE_MODEL_SKY_DIR')
if model_dir is not None:
model_dir_out = os.path.join(args.outdir, 'images', 'unwise-mod')
trymakedirs(model_dir_out)
for indir, outdir, tiles, band, fulldepth in wisedata:
for tile in tiles:
wanyband = 'w'
tim = get_unwise_tractor_image(indir, tile, band,
bandname=wanyband, roiradecbox=roiradec)
print('Got unWISE tim', tim)
print(tim.shape)
if model_dir is not None and fulldepth and band in [1,2]:
print('ROI', tim.roi)
#0387p575.1.mod.fits
fn = '%s.%i.mod.fits' % (tile, band)
print('Filename', fn)
F = fitsio.FITS(os.path.join(model_dir, fn))
x0,x1,y0,y1 = tim.roi
slc = slice(y0,y1),slice(x0,x1)
phdr = F[0].read_header()
outfn = os.path.join(model_dir_out, fn)
for e,extname in [(1,'MODEL'), (2,'SKY')]:
pix = F[e][slc]
hdr = F[e].read_header()
crpix1 = hdr['CRPIX1']
crpix2 = hdr['CRPIX2']
hdr['CRPIX1'] -= x0
hdr['CRPIX2'] -= y0
#print('mod', mod)
#print('Model', mod.shape)
if e == 1:
fitsio.write(outfn, None, clobber=True, header=phdr)
fitsio.write(outfn, pix, header=hdr, extname=extname)
print('Wrote', outfn)
thisdir = get_unwise_tile_dir(outdir, tile)
print('Directory for this WISE tile:', thisdir)
base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band))
print('Base filename:', base)
masked = True
mu = 'm' if masked else 'u'
imfn = base + 'img-%s.fits' % mu
ivfn = base + 'invvar-%s.fits.gz' % mu
nifn = base + 'n-%s.fits.gz' % mu
nufn = base + 'n-u.fits.gz'
#print('WISE image header:', tim.hdr)
# Adjust the header WCS by x0,y0
wcs = tim.wcs.wcs
tim.hdr['CRPIX1'] = wcs.crpix[0]
tim.hdr['CRPIX2'] = wcs.crpix[1]
H,W = tim.shape
tim.hdr['IMAGEW'] = W
tim.hdr['IMAGEH'] = H
print('WCS:', wcs)
print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2'])
trymakedirs(imfn, dir=True)
fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True)
print('Wrote', imfn)
fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True)
print('Wrote', ivfn)
fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True)
print('Wrote', nifn)
fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True)
print('Wrote', nufn)
if not (indir,tile) in wrote_masks:
print('Looking for mask file for', indir, tile)
# record that we tried this dir/tile combo
wrote_masks.add((indir,tile))
for idir in indir.split(':'):
tdir = get_unwise_tile_dir(idir, tile)
maskfn = 'unwise-%s-msk.fits.gz' % tile
fn = os.path.join(tdir, maskfn)
print('Mask file:', fn)
if os.path.exists(fn):
print('Reading', fn)
(x0,x1,y0,y1) = tim.roi
roislice = (slice(y0,y1), slice(x0,x1))
F = fitsio.FITS(fn)[0]
hdr = F.read_header()
M = F[roislice]
outfn = os.path.join(thisdir, maskfn)
fitsio.write(outfn, M, header=tim.hdr, clobber=True)
print('Wrote', outfn)
break
outC = outsurvey.get_ccds_readonly()
for iccd,ccd in enumerate(outC):
outim = outsurvey.get_image_object(ccd)
print('Got output image:', outim)
otim = outim.get_tractor_image(pixPsf=True,
hybridPsf=True, old_calibs_ok=True)
print('Got output tim:', otim)
def run_tiles(X):
tiles, tag = X
print('Running', tag, '-', len(tiles), 'tiles')
# Aaron's file has all images share the boresight CRVAL, so they have large CRPIX values.
T = fits_table(
'/global/cfs/cdirs/desi/users/ameisner/GFA/gfa_reduce_etc/gfa_wcs+focus.bigtan-zenith.fits'
)
Nbright = 10
tiles_ann = fits_table()
tiles_ann.index = tiles.index
gfa_regions = []
maxr = 0.
#t.cd[0,0], t.cd[0,1], t.cd[1,0], t.cd[1,1],
for t in T:
wcs = Tan(0., 0., t.crpix[0], t.crpix[1], t.cd[0, 0], t.cd[1, 0],
t.cd[0, 1], t.cd[1, 1], float(t.naxis[0]), float(t.naxis[1]))
ctype = t.extname[:5]
cnum = int(t.extname[5])
h, w = wcs.shape
x, y = [1, 1, w, w, 1], [1, h, h, 1, 1]
r, d = wcs.pixelxy2radec(x, y)
dists = degrees_between(0., 0., r, d)
maxr = max(maxr, max(dists))
# x0, y0, x1, y1
rois = []
if ctype == 'FOCUS':
# add the two half-chips.
# wcs.get_subimage moves the CRPIX, but leave CRVAL unchanged, so tx,ty still work unchanged.
# Aaron's WCS templates correct for the overscans
#wcs_subs.append((cstr, cnum, 'a', wcs.get_subimage(0, 0, 1024, h)))
#wcs_subs.append((cstr, cnum, 'b', wcs.get_subimage(1024, 0, 1024, h)))
#all_sub_wcs[(cstr, cnum, 1)] = (tx, ty, wcs.get_subimage(50, 0, 1024, 1032))
#all_sub_wcs[(cstr, cnum, 2)] = (tx, ty, wcs.get_subimage(1174, 0, 1024, 1032))
# Add (negative) margin for donut size and telescope pointing uncertainty.
# ~10" for donuts and ~10" for telescope pointing
#margin = 100
#wcs_subs.append((cstr, cnum, 'a_margin', wcs.get_subimage(margin, margin, 1024-2*margin, h-2*margin)))
#wcs_subs.append((cstr, cnum, 'b_margin', wcs.get_subimage(1024+margin, margin, 1024-2*margin, h-2*margin)))
# Also add a positive margin for bright-star reflections off filters
#margin = 125
#wcs_subs.append((cstr, cnum, 'expanded', wcs.get_subimage(-margin, -margin, w+2*margin, h+2*margin)))
rois.append(('a', 0, 0, 1024, h))
rois.append(('b', 1024, 0, 2048, h))
margin = 100
rois.append(
('a_margin', margin, margin, 1024 - margin, h - margin))
rois.append(
('b_margin', 1024 + margin, margin, 2048 - margin, h - margin))
margin = 125
rois.append(('expanded', -margin, -margin, w + margin, h + margin))
else:
# Guide chips include overscan pixels -- including a blank region in the middle.
#print(cstr,cnum, 'shape', wcs.shape)
#wcs_subs.append((cstr, cnum, 'ccd', wcs))
rois.append(('ccd', 0, 0, w, h))
# Add expanded GUIDE chips -- 25" margin / 0.2"/pix = 125 pix
margin = 125
#wcs_subs.append((cstr, cnum, 'expanded', wcs.get_subimage(-margin, -margin, w+2*margin, h+2*margin)))
rois.append(('expanded', -margin, -margin, w + margin, h + margin))
margin = 125
expwcs = wcs.get_subimage(-margin, -margin, w + 2 * margin,
h + 2 * margin)
newrois = []
for tag, x0, y0, x1, y1 in rois:
name = '%s_%i_%s' % (ctype.lower(), cnum, tag)
arr = np.zeros(len(tiles), (np.float32, Nbright))
tiles_ann.set('brightest_' + name, arr)
# (the rois have zero-indexed x0,y0, and non-inclusive x1,y1!)
newrois.append((name, arr, 1 + x0, 1 + y0, x1, y1))
gfa_regions.append((ctype, cnum, wcs, expwcs, newrois))
# DEBUG WCS
# s = []
# for ctype,cnum,wcs,expwcs,rois in gfa_regions:
# WCS = wcs
# h,w = WCS.shape
# #print('Expwcs:', w, 'x', h)
# x = [1,1,w,w,1]
# y = [1,h,h,1,1]
# r,d = WCS.pixelxy2radec(x, y)
# p = ','.join(['%.4f,%.4f' % (rr,dd) for rr,dd in zip(r,d)])
# s.append(p)
# s = ';'.join(s)
# print('http://legacysurvey.org/viewer/?ra=0&dec=0&poly='+s)
# sys.exit(0)
gaia = CachingGaiaCatalog(columns=[
'ra', 'dec', 'phot_g_mean_mag', 'phot_bp_mean_mag', 'phot_rp_mean_mag',
'astrometric_excess_noise', 'astrometric_params_solved', 'source_id',
'pmra_error', 'pmdec_error', 'parallax_error', 'ra_error', 'dec_error',
'pmra', 'pmdec', 'parallax', 'ref_epoch'
])
tyc2fn = '/global/cfs/cdirs/cosmo/staging/tycho2/tycho2.kd.fits'
tycho_kd = tree_open(tyc2fn)
tycho_cat = fits_table(tyc2fn)
maxrad = maxr * 1.05
for itile, tile in enumerate(tiles):
#if not tile.in_imaging:
# continue
#if tile.centerid % 10 == 0:
#print('tile program', tile.program, 'pass', tile.get('pass'), 'id', tile.centerid, gaia.get_healpix_tree.cache_info())
I = tree_search_radec(tycho_kd, tile.ra, tile.dec, maxrad)
tycstars = tycho_cat[I]
fix_tycho(tycstars)
for cstr, cname, chipwcs, bigwcs, rois in gfa_regions:
h, w = chipwcs.shape
chipwcs.set_crval(tile.ra, tile.dec)
bigwcs.set_crval(tile.ra, tile.dec)
gstars = gaia.get_catalog_in_wcs(bigwcs, step=1032, margin=0)
fix_gaia(gstars)
bh, bw = bigwcs.shape
ok, x, y = bigwcs.radec2pixelxy(tycstars.ra, tycstars.dec)
tstars = tycstars[(x >= 1) * (y >= 1) * (x <= bw) * (y <= bh)]
#print('Tile', tile.program, 'p', tile.get('pass'), tile.centerid,
# 'GFA', cstr, cname, ':', len(gstars), 'Gaia stars', len(tstars), 'Tycho-2 stars')
if len(gstars) + len(tstars) == 0:
print('No stars in tile centerid', tile.centerid, 'chip', name)
continue
if len(gstars) > 0 and len(tstars) > 0:
merge_gaia_tycho(gstars, tstars)
stars = merge_tables([gstars, tstars], columns='fillzero')
elif len(tstars) > 0:
stars = tstars
else:
stars = gstars
ok, x, y = chipwcs.radec2pixelxy(stars.ra, stars.dec)
for name, arr, x0, y0, x1, y1 in rois:
J = np.flatnonzero(
(x >= x0) * (x <= x1) * (y >= y0) * (y <= y1))
mags = stars.mag[J]
#print(' ', len(mags), 'in name')
K = np.argsort(mags)
K = K[:Nbright]
arr[itile, :len(K)] = mags[K]
#tiles.add_columns_from(tiles_ann)
return tiles_ann
def read_tycho2(survey, targetwcs, bands):
from astrometry.libkd.spherematch import tree_open, tree_search_radec
from legacypipe.survey import GaiaSource
tycho2fn = survey.find_file('tycho2')
radius = 1.
ra, dec = targetwcs.radec_center()
# John added the "isgalaxy" flag 2018-05-10, from the Metz &
# Geffert (04) catalog.
# Eddie added the "zguess" column 2019-03-06, by matching with
# 2MASS and estimating z based on APASS.
# The "tycho2.kd.fits" file read here was produced by:
#
# fitscopy ~schlafly/legacysurvey/tycho-isgalaxyflag-2mass.fits"[col \
# tyc1;tyc2;tyc3;ra;dec;sigma_ra;sigma_dec;mean_ra;mean_dec;pm_ra;pm_dec; \
# sigma_pm_ra;sigma_pm_dec;epoch_ra;epoch_dec;mag_bt;mag_vt;mag_hp; \
# isgalaxy;Jmag;Hmag;Kmag,zguess]" /tmp/tycho2-astrom.fits
# startree -P -k -n stars -T -i /tmp/tycho2-astrom.fits \
# -o /global/project/projectdirs/cosmo/staging/tycho2/tycho2.kd.fits
kd = tree_open(tycho2fn, 'stars')
I = tree_search_radec(kd, ra, dec, radius)
debug(len(I), 'Tycho-2 stars within', radius,
'deg of RA,Dec (%.3f, %.3f)' % (ra, dec))
if len(I) == 0:
return None
# Read only the rows within range.
tycho = fits_table(tycho2fn, rows=I)
del kd
if 'isgalaxy' in tycho.get_columns():
tycho.cut(tycho.isgalaxy == 0)
debug('Cut to', len(tycho), 'Tycho-2 stars on isgalaxy==0')
else:
print('Warning: no "isgalaxy" column in Tycho-2 catalog')
tycho.ref_cat = np.array(['T2'] * len(tycho))
# tyc1: [1,9537], tyc2: [1,12121], tyc3: [1,3]
tycho.ref_id = (tycho.tyc1.astype(np.int64) * 1000000 +
tycho.tyc2.astype(np.int64) * 10 +
tycho.tyc3.astype(np.int64))
with np.errstate(divide='ignore'):
# In our Tycho-2 catalog, sigma_pm_* are in *arcsec/yr*, Gaia is in mas/yr.
tycho.pmra_ivar = 1. / (tycho.sigma_pm_ra * 1000.)**2
tycho.pmdec_ivar = 1. / (tycho.sigma_pm_dec * 1000.)**2
tycho.ra_ivar = 1. / tycho.sigma_ra**2
tycho.dec_ivar = 1. / tycho.sigma_dec**2
tycho.rename('pm_ra', 'pmra')
tycho.rename('pm_dec', 'pmdec')
for c in ['pmra', 'pmdec', 'pmra_ivar', 'pmdec_ivar']:
X = tycho.get(c)
X[np.logical_not(np.isfinite(X))] = 0.
tycho.mag = tycho.mag_vt
# Patch missing mag values...
tycho.mag[tycho.mag == 0] = tycho.mag_hp[tycho.mag == 0]
tycho.mag[tycho.mag == 0] = tycho.mag_bt[tycho.mag == 0]
# Use zguess
tycho.mask_mag = tycho.mag
with np.errstate(invalid='ignore'):
I = np.flatnonzero(
np.isfinite(tycho.zguess) * (tycho.zguess + 1. < tycho.mag))
tycho.mask_mag[I] = tycho.zguess[I]
# Per discussion in issue #306 -- cut on mag < 13.
# This drops only 13k/2.5M stars.
tycho.cut(tycho.mask_mag < 13.)
tycho.radius = mask_radius_for_mag(tycho.mask_mag)
tycho.keep_radius = 2. * tycho.radius
for c in [
'tyc1', 'tyc2', 'tyc3', 'mag_bt', 'mag_vt', 'mag_hp', 'mean_ra',
'mean_dec', 'sigma_pm_ra', 'sigma_pm_dec', 'sigma_ra', 'sigma_dec'
]:
tycho.delete_column(c)
# add Gaia-style columns
# No parallaxes in Tycho-2
tycho.parallax = np.zeros(len(tycho), np.float32)
# Tycho-2 "supplement" stars, from Hipparcos and Tycho-1 catalogs, have
# ref_epoch = 0. Fill in with the 1991.25 epoch of those catalogs.
tycho.epoch_ra[tycho.epoch_ra == 0] = 1991.25
tycho.epoch_dec[tycho.epoch_dec == 0] = 1991.25
# Tycho-2 has separate epoch_ra and epoch_dec.
# Move source to the mean epoch.
tycho.ref_epoch = (tycho.epoch_ra + tycho.epoch_dec) / 2.
cosdec = np.cos(np.deg2rad(tycho.dec))
tycho.ra += (tycho.ref_epoch -
tycho.epoch_ra) * tycho.pmra / 3600. / cosdec
tycho.dec += (tycho.ref_epoch - tycho.epoch_dec) * tycho.pmdec / 3600.
# Tycho-2 proper motions are in arcsec/yr; Gaia are mas/yr.
tycho.pmra *= 1000.
tycho.pmdec *= 1000.
# We already cut on John's "isgalaxy" flag
tycho.pointsource = np.ones(len(tycho), bool)
# phot_g_mean_mag -- for initial brightness of source
tycho.phot_g_mean_mag = tycho.mag
tycho.delete_column('epoch_ra')
tycho.delete_column('epoch_dec')
tycho.istycho = np.ones(len(tycho), bool)
tycho.isbright = np.ones(len(tycho), bool)
tycho.ismedium = np.ones(len(tycho), bool)
tycho.donotfit = np.zeros(len(tycho), bool)
tycho.sources = np.empty(len(tycho), object)
if bands is not None:
for i, t in enumerate(tycho):
tycho.sources[i] = GaiaSource.from_catalog(t, bands)
return tycho
def read_tycho2(survey, targetwcs):
from astrometry.libkd.spherematch import tree_open, tree_search_radec
tycho2fn = survey.find_file('tycho2')
radius = 1.
ra, dec = targetwcs.radec_center()
# John added the "isgalaxy" flag 2018-05-10, from the Metz &
# Geffert (04) catalog.
# Eddie added the "zguess" column 2019-03-06, by matching with
# 2MASS and estimating z based on APASS.
# The "tycho2.kd.fits" file read here was produced by:
#
# fitscopy ~schlafly/legacysurvey/tycho-isgalaxyflag-2mass.fits"[col \
# tyc1;tyc2;tyc3;ra;dec;sigma_ra;sigma_dec;mean_ra;mean_dec;pm_ra;pm_dec; \
# sigma_pm_ra;sigma_pm_dec;epoch_ra;epoch_dec;mag_bt;mag_vt;mag_hp; \
# isgalaxy;Jmag;Hmag;Kmag,zguess]" /tmp/tycho2-astrom.fits
# startree -P -k -n stars -T -i /tmp/tycho2-astrom.fits \
# -o /global/project/projectdirs/cosmo/staging/tycho2/tycho2.kd.fits
kd = tree_open(tycho2fn, 'stars')
I = tree_search_radec(kd, ra, dec, radius)
debug(len(I), 'Tycho-2 stars within', radius,
'deg of RA,Dec (%.3f, %.3f)' % (ra, dec))
if len(I) == 0:
return None
# Read only the rows within range.
tycho = fits_table(tycho2fn, rows=I)
del kd
if 'isgalaxy' in tycho.get_columns():
tycho.cut(tycho.isgalaxy == 0)
debug('Cut to', len(tycho), 'Tycho-2 stars on isgalaxy==0')
else:
print('Warning: no "isgalaxy" column in Tycho-2 catalog')
tycho.ref_cat = np.array(['T2'] * len(tycho))
# tyc1: [1,9537], tyc2: [1,12121], tyc3: [1,3]
tycho.ref_id = (tycho.tyc1.astype(np.int64) * 1000000 +
tycho.tyc2.astype(np.int64) * 10 +
tycho.tyc3.astype(np.int64))
with np.errstate(divide='ignore'):
tycho.pmra_ivar = 1. / tycho.sigma_pm_ra**2
tycho.pmdec_ivar = 1. / tycho.sigma_pm_dec**2
tycho.ra_ivar = 1. / tycho.sigma_ra**2
tycho.dec_ivar = 1. / tycho.sigma_dec**2
tycho.rename('pm_ra', 'pmra')
tycho.rename('pm_dec', 'pmdec')
for c in ['pmra', 'pmdec', 'pmra_ivar', 'pmdec_ivar']:
X = tycho.get(c)
X[np.logical_not(np.isfinite(X))] = 0.
tycho.mag = tycho.mag_vt
# Patch missing mag values...
tycho.mag[tycho.mag == 0] = tycho.mag_hp[tycho.mag == 0]
tycho.mag[tycho.mag == 0] = tycho.mag_bt[tycho.mag == 0]
# Per discussion in issue #306 -- cut on mag < 13. This drops only 13k/2.5M stars
tycho.cut(tycho.mag < 13.)
# See note on gaia.radius above -- don't change the 0.262 to
# targetwcs.pixel_scale()!
tycho.radius = np.minimum(1800., 150. * 2.5**(
(11. - tycho.mag) / 3.)) * 0.262 / 3600.
for c in [
'tyc1', 'tyc2', 'tyc3', 'mag_bt', 'mag_vt', 'mag_hp', 'mean_ra',
'mean_dec', 'sigma_pm_ra', 'sigma_pm_dec', 'sigma_ra', 'sigma_dec'
]:
tycho.delete_column(c)
# add Gaia-style columns
# No parallaxes in Tycho-2
tycho.parallax = np.zeros(len(tycho), np.float32)
# Tycho-2 has separate epoch_ra and epoch_dec.
# Move source to the mean epoch.
tycho.ref_epoch = (tycho.epoch_ra + tycho.epoch_dec) / 2.
cosdec = np.cos(np.deg2rad(tycho.dec))
tycho.ra += (tycho.ref_epoch -
tycho.epoch_ra) * tycho.pmra / 3600. / cosdec
tycho.dec += (tycho.ref_epoch - tycho.epoch_dec) * tycho.pmdec / 3600.
# Tycho-2 proper motions are in arcsec/yr; Gaia are mas/yr.
tycho.pmra *= 1000.
tycho.pmdec *= 1000.
# We already cut on John's "isgalaxy" flag
tycho.pointsource = np.ones(len(tycho), bool)
# phot_g_mean_mag -- for initial brightness of source
tycho.phot_g_mean_mag = tycho.mag
tycho.delete_column('epoch_ra')
tycho.delete_column('epoch_dec')
tycho.isbright = np.ones(len(tycho), bool)
tycho.ismedium = np.ones(len(tycho), bool)
return tycho
def read_large_galaxies(survey,
targetwcs,
bands,
clean_columns=True,
max_radius=2.):
# Note, max_radius must include the brick radius!
from astrometry.libkd.spherematch import tree_open, tree_search_radec
galfn = survey.find_file('large-galaxies')
if galfn is None:
debug('No large-galaxies catalog file')
return None
radius = max_radius
rc, dc = targetwcs.radec_center()
debug('Reading', galfn)
try:
kd = tree_open(galfn, 'stars')
except:
kd = tree_open(galfn, 'largegals')
I = tree_search_radec(kd, rc, dc, radius)
debug('%i large galaxies within %.3g deg of RA,Dec (%.3f, %.3f)' %
(len(I), radius, rc, dc))
if len(I) == 0:
return None
# Read only the rows within range.
galaxies = fits_table(galfn, rows=I)
del kd
refcat, preburn = get_large_galaxy_version(galfn)
debug('Large galaxies version: "%s", preburned?' % refcat, preburn)
if preburn:
# SGA ellipse catalog
# NOTE: fields such as ref_cat, preburned, etc, already exist in the
# "galaxies" catalog read from disk.
# The galaxies we want to appear in MASKBITS get
# 'islargegalaxy' set. This includes both pre-burned
# galaxies, and ones where the preburning failed and we want
# to fall back to the SGA-parent ellipse for masking.
galaxies.islargegalaxy = ((galaxies.ref_cat == refcat) *
(galaxies.sga_id > -1))
# The pre-fit galaxies whose parameters will stay fixed
galaxies.freezeparams = (galaxies.preburned * galaxies.freeze)
else:
# SGA parent catalog
galaxies.ref_cat = np.array([refcat] * len(galaxies))
galaxies.islargegalaxy = np.ones(len(galaxies), bool)
galaxies.freezeparams = np.zeros(len(galaxies), bool)
galaxies.preburned = np.zeros(len(galaxies), bool)
galaxies.rename('sga_id', 'ref_id')
galaxies.rename('mag_leda', 'mag')
# Pre-burned, frozen but non-SGA sources have diam=-1.
galaxies.radius = np.maximum(0., galaxies.diam / 2. / 60.) # [degree]
galaxies.keep_radius = 2. * galaxies.radius
galaxies.sources = np.empty(len(galaxies), object)
galaxies.sources[:] = None
if bands is not None:
galaxies.sources[:] = get_galaxy_sources(galaxies, bands)
if clean_columns:
keep_columns = [
'ra', 'dec', 'radius', 'mag', 'ref_cat', 'ref_id', 'ba', 'pa',
'sources', 'islargegalaxy', 'freezeparams', 'keep_radius'
]
for c in galaxies.get_columns():
if not c in keep_columns:
galaxies.delete_column(c)
return galaxies
inds = inds[order]
dists = dists[order]
ok = np.array_equal(pairs, inds)
print('Indices equal:', ok)
print('Build uint64 tree...')
kd = spherematch.tree_build(ux1)
kd.print()
data = kd.get_data(np.array([0,3,5]).astype(np.uint32))
assert(data.dtype == np.uint64)
print('Kd data:', data.dtype, data)
kd.write('kd-u64.fits')
kd2 = spherematch.tree_open('kd-u64.fits')
data2 = kd2.get_data(np.array([0,3,5]).astype(np.uint32))
assert(data2.dtype == np.uint64)
print('Kd data2:', data2.dtype, data2)
assert(np.all(data == data2))
del kd
del kd2
###
t0 = time()
(inds,dists) = spherematch.nearest(x1, x2, r)
dt = time() - t0
def cat_targets_drAB(req, ver, cats=None, tag='', bgs=False, sky=False, bright=False, dark=False, color_name_func=desitarget_color_names):
'''
color_name_func: function that selects names and colors for targets
(eg based on targeting bit values)
'''
if cats is None:
cats = []
import json
ralo = float(req.GET['ralo'])
rahi = float(req.GET['rahi'])
declo = float(req.GET['declo'])
dechi = float(req.GET['dechi'])
ver = int(ver)
if not ver in catversions[tag]:
raise RuntimeError('Invalid version %i for tag %s' % (ver, tag))
from astrometry.util.fits import fits_table, merge_tables
from astrometry.libkd.spherematch import tree_open, tree_search_radec
import numpy as np
from astrometry.util.starutil_numpy import radectoxyz, xyztoradec, degrees_between
xyz1 = radectoxyz(ralo, declo)
xyz2 = radectoxyz(rahi, dechi)
xyz = (xyz1 + xyz2)/2.
xyz /= np.sqrt(np.sum(xyz**2))
rc,dc = xyztoradec(xyz)
rc = rc[0]
dc = dc[0]
rad = degrees_between(rc, dc, ralo, declo)
'''
startree -i /project/projectdirs/desi/target/catalogs/targets-dr4-0.20.0.fits -o data/targets-dr4-0.20.0.kd.fits -P -k -T
'''
TT = []
for fn in cats:
kd = tree_open(fn)
I = tree_search_radec(kd, rc, dc, rad)
print('Matched', len(I), 'from', fn)
if len(I) == 0:
continue
T = fits_table(fn, rows=I)
TT.append(T)
if len(TT) == 0:
return HttpResponse(json.dumps(dict(rd=[], name=[])),
content_type='application/json')
T = merge_tables(TT, columns='fillzero')
if bgs:
T.cut(T.bgs_target > 0)
if bright:
T.cut(np.logical_or(T.bgs_target > 0, T.mws_target > 0))
if dark:
T.cut(T.desi_target > 0)
names = None
colors = None
if color_name_func is not None:
names,colors = color_name_func(T)
if sky:
fluxes = [dict(g=float(g), r=float(r), z=float(z))
for (g,r,z) in zip(T.apflux_g[:,0], T.apflux_r[:,0], T.apflux_z[:,0])]
nobs = None
else:
fluxes = [dict(g=float(g), r=float(r), z=float(z),
W1=float(W1), W2=float(W2))
for (g,r,z,W1,W2)
in zip(T.flux_g, T.flux_r, T.flux_z, T.flux_w1, T.flux_w2)]
nobs=[dict(g=int(g), r=int(r), z=int(z)) for g,r,z
in zip(T.nobs_g, T.nobs_r, T.nobs_z)],
rtn = dict(rd=[(t.ra, t.dec) for t in T],
targetid=[int(t) for t in T.targetid],
fluxes=fluxes,
)
if names is not None:
rtn.update(name=names)
if colors is not None:
rtn.update(color=colors)
if nobs is not None:
rtn.update(nobs=nobs)
# Convert targetid to string to prevent rounding errors
rtn['targetid'] = [str(s) for s in rtn['targetid']]
return HttpResponse(json.dumps(rtn), content_type='application/json')
