def test_readwrite_tractor(self):
tractorfile = io.list_tractorfiles(self.datadir)[0]
sweepfile = io.list_sweepfiles(self.datadir)[0]
data = io.read_tractor(sweepfile)
self.assertEqual(len(data), 6) #- test data has 6 objects per file
data = io.read_tractor(tractorfile)
self.assertEqual(len(data), 6) #- test data has 6 objects per file
data, hdr = io.read_tractor(tractorfile, header=True)
self.assertEqual(len(data), 6) #- test data has 6 objects per file
#ADM check PHOTSYS got added in writing targets
io.write_targets(self.testfile, data, indir=self.datadir)
#ADM use fits read wrapper in io to correctly handle whitespace
d2, h2 = io.whitespace_fits_read(self.testfile, header=True)
self.assertEqual(
list(data.dtype.names) + ["PHOTSYS"], list(d2.dtype.names))
#ADM check PHOTSYS and HPXPIXEL got added writing targets with NSIDE request
io.write_targets(self.testfile, data, nside=64, indir=self.datadir)
#ADM use fits read wrapper in io to correctly handle whitespace
d2, h2 = io.whitespace_fits_read(self.testfile, header=True)
self.assertEqual(
list(data.dtype.names) + ["HPXPIXEL", "PHOTSYS"],
list(d2.dtype.names))
for column in data.dtype.names:
self.assertTrue(np.all(data[column] == d2[column]))
def test_cuts_basic(self):
#- Cuts work with either data or filenames
desi, bgs, mws = cuts.apply_cuts(self.tractorfiles[0])
desi, bgs, mws = cuts.apply_cuts(self.sweepfiles[0])
data = io.read_tractor(self.tractorfiles[0])
desi, bgs, mws = cuts.apply_cuts(data)
data = io.read_tractor(self.sweepfiles[0])
desi, bgs, mws = cuts.apply_cuts(data)
def test_tractor_columns(self):
tscolumns = list(io.tsdatamodel.dtype.names) + [
'BRICK_PRIMARY',
]
tractorfile = io.list_tractorfiles(self.datadir)[0]
data = io.read_tractor(tractorfile)
self.assertEqual(set(data.dtype.names), set(tscolumns))
# columns = ['BX', 'BY']
columns = ['RA', 'DEC']
data = io.read_tractor(tractorfile, columns=columns)
self.assertEqual(set(data.dtype.names), set(columns))
data = io.read_tractor(tractorfile, columns=tuple(columns))
self.assertEqual(set(data.dtype.names), set(columns))
def setUp(self):
#ADM at this nskymin you always seem to get at least 1 bad position
self.nskymin = 5000000
self.navoid = 2.
self.psfsize = psfsize
#ADM location of input test file
self.datadir = resource_filename('lvmtarget.test', 't')
self.sweepfile = self.datadir + '/sweep-320m005-330p000.fits'
#ADM read in the test sweeps file
self.objs = io.read_tractor(self.sweepfile)
#ADM need to ensure that one object has a large enough half-light radius
#ADM to cover matching sky positions to larger objects
self.objs['SHAPEDEV_R'][0] = (self.psfsize * self.navoid) + 1e-8
self.objs['SHAPEDEV_E1'][0] = -0.22389728
self.objs['SHAPEDEV_E2'][0] = 0.42635256
#ADM create a "maximum" search distance that is as large as the
#ADM diagonal across all objects in the test sweeps file
#ADM note that this is only close to correct because the test
#ADM file is near 0o Declination
cmax = SkyCoord(
max(self.objs["RA"]) * u.degree,
max(self.objs["DEC"]) * u.degree)
cmin = SkyCoord(
min(self.objs["RA"]) * u.degree,
min(self.objs["DEC"]) * u.degree)
self.maxrad = cmax.separation(cmin).arcsec
def _select_sandbox_targets_file(filename):
'''Returns targets in filename that pass the sandbox cuts'''
from lvmtarget.sandbox.cuts import apply_sandbox_cuts
objects = io.read_tractor(filename)
desi_target, bgs_target, mws_target = apply_sandbox_cuts(objects,FoMthresh,Method)
return _finalize_targets(objects, desi_target, bgs_target, mws_target)
def _get_bright_stars(filename):
'''Retrieves bright stars from a sweeps/Tractor file'''
objs = io.read_tractor(filename)
#ADM write the fluxes as an array instead of as named columns
fluxes = objs[bandnames].view(
objs[bandnames].dtype[0]).reshape(objs[bandnames].shape + (-1, ))
#ADM Retain rows for which ANY band is brighter than maglim
w = np.where(np.any(fluxes > fluxlim, axis=1))
if len(w[0]) > 0:
return objs[w]
def test_unextinct_fluxes(self):
targets = io.read_tractor(self.tractorfiles[0])
t1 = cuts.unextinct_fluxes(targets)
self.assertTrue(isinstance(t1, np.ndarray))
t2 = cuts.unextinct_fluxes(Table(targets))
self.assertTrue(isinstance(t2, Table))
for col in ['GFLUX', 'RFLUX', 'ZFLUX', 'W1FLUX', 'W2FLUX']:
self.assertIn(col, t1.dtype.names)
self.assertIn(col, t2.dtype.names)
self.assertTrue(np.all(t1[col] == t2[col]))
def test_fix_dr1(self):
'''test the DR1 TYPE dype fix (make everything S4)'''
#- First, break it
files = io.list_sweepfiles(self.datadir)
objects = io.read_tractor(files[0])
dt = objects.dtype.descr
for i in range(len(dt)):
if dt[i][0] == 'TYPE':
dt[i] = ('TYPE', 'S10')
break
badobjects = objects.astype(np.dtype(dt))
newobjects = io.fix_tractor_dr1_dtype(badobjects)
self.assertEqual(newobjects['TYPE'].dtype, np.dtype('S4'))
def _check_input_files(filename):
'''Check for corrupted values in a file'''
from functools import partial
from os.path import getsize
#ADM read in Tractor or sweeps files
objects = io.read_tractor(filename)
#ADM if everything is OK the default meassage will be "OK"
filemessageroot = 'OK'
filemessageend = ''
#ADM columns that shouldn't have zero values
cols = [
'BRICKID',
# 'RA_IVAR', 'DEC_IVAR',
'MW_TRANSMISSION_G', 'MW_TRANSMISSION_R', 'MW_TRANSMISSION_Z',
# 'WISE_FLUX',
# 'WISE_MW_TRANSMISSION','DCHISQ'
]
#ADM for each of these columnes that shouldn't have zero values,
#ADM loop through and look for zero values
for colname in cols:
if np.min(objects[colname]) == 0:
filemessageroot = "WARNING...some values are zero for"
filemessageend += " "+colname
#ADM now, loop through entries in the file and search for 4096-byte
#ADM blocks that are all zeros (a sign of corruption in file-writing)
#ADM Note that fits files are padded by 2880 bytes, so we only want to
#ADM process the file length (in bytes) - 2880
bytestop = getsize(filename) -2880
with open(filename, 'rb') as f:
for block_number, data in enumerate(iter(partial(f.read, 4096), b'')):
if not any(data):
if block_number*4096 < bytestop:
filemessageroot = "WARNING...some values are zero for"
filemessageend += ' 4096-byte-block-#{0}'.format(block_number)
return [filename,filemessageroot+filemessageend]
def calculate_separations(objs, navoid=2.):
"""Generate an array of separations (in arcseconds) for a set of objects
Parameters
----------
objs : :class:`~numpy.ndarray`
numpy structured array with UPPERCASE columns, OR a string
tractor/sweep filename. Must contain at least the columns
"RA", "DEC", "SHAPEDEV_R", "SHAPEEXP_R"
navoid : :class:`float`, optional, defaults to 2.
the number of times the galaxy half-light radius (or seeing) to avoid
objects out to when placing sky fibers
Returns
-------
:class:`float`
an array of maximum separations (in arcseconds) based on
de Vaucouleurs, Exponential or point-source half-light radii
"""
#ADM check if input objs is a filename or the actual data
if isinstance(objs, str):
objs = io.read_tractor(objs)
nobjs = len(objs)
#ADM possible choices for separation based on de Vaucouleurs and Exponential profiles
#ADM or a minimum of psfsize arcseconds for point sources ("the seeing")
#ADM the default psfsize is supplied at the top of this code
sepchoices = np.vstack(
[objs["SHAPEDEV_R"], objs["SHAPEEXP_R"],
np.ones(nobjs) * psfsize]).T
#ADM the maximum separation from de Vaucoulers/exponential/PSF choices
sep = navoid * np.max(sepchoices, axis=1)
return sep
def make_sky_targets(objs, navoid=2., nskymin=None):
"""Generate sky targets and translate them into the typical format for DESI targets
Parameters
----------
objs : :class:`~numpy.ndarray`
numpy structured array with UPPERCASE columns, OR a string
tractor/sweep filename. Must contain at least the columns
"RA", "DEC", "SHAPEDEV_R", "SHAPEEXP_R"
navoid : :class:`float`, optional, defaults to 2.
the number of times the galaxy half-light radius (or seeing) to avoid
objects out to when placing sky fibers
nskymin : :class:`float`, optional, defaults to reading from lvmmodel.io
the minimum DENSITY of sky fibers to generate
Returns
-------
:class:`~numpy.ndarray`
a structured array of good and bad sky positions in the DESI target format
"""
#ADM initialize the default logger
from lvmutil.log import get_logger, DEBUG
log = get_logger(DEBUG)
start = time()
#ADM check if input objs is a filename or the actual data
if isinstance(objs, str):
objs = io.read_tractor(objs)
log.info('Generating sky positions...t = {:.1f}s'.format(time() - start))
#ADM generate arrays of good and bad objects for this sweeps file (or set)
ragood, decgood, rabad, decbad = generate_sky_positions(objs,
navoid=navoid,
nskymin=nskymin)
ngood = len(ragood)
nbad = len(rabad)
nskies = ngood + nbad
#ADM retrieve the standard DESI target array
dt = io.tsdatamodel.dtype
skies = np.zeros(nskies, dtype=dt)
#ADM populate the output recarray with the RA/Dec of the good and bad sky locations
skies["RA"][0:ngood], skies["DEC"][0:ngood] = ragood, decgood
skies["RA"][ngood:nskies], skies["DEC"][ngood:nskies] = rabad, decbad
#ADM create an array of target bits with the SKY information set
desi_target = np.zeros(nskies, dtype='>i8')
desi_target[0:ngood] |= desi_mask.SKY
desi_target[ngood:nskies] |= desi_mask.BADSKY
log.info('Looking up brick information...t = {:.1f}s'.format(time() -
start))
#ADM add the brick information for the sky targets
b = brick.Bricks(bricksize=0.25)
skies["BRICKID"] = b.brickid(skies["RA"], skies["DEC"])
skies["BRICKNAME"] = b.brickname(skies["RA"], skies["DEC"])
#ADM set the data release from the passed sweeps objects
dr = np.max(objs['RELEASE'])
#ADM check the passed sweeps objects have the same release
checker = np.min(objs['RELEASE'])
if dr != checker:
raise IOError(
'Multiple data releases present in same input sweeps objects?!')
skies["RELEASE"] = dr
#ADM set the objid (just use a sequential number as setting skies
#ADM to 1 in the TARGETID will make these unique
#ADM *MAKE SURE TO SET THE BRIGHT STAR SAFE LOCATIONS OF THE MAXIMUM SKY OBJID*!!!
skies["OBJID"] = np.arange(nskies)
log.info('Finalizing target bits...t = {:.1f}s'.format(time() - start))
#ADM add target bit columns to the output array, note that mws_target
#ADM and bgs_target should be zeros for all sky objects
dum = np.zeros_like(desi_target)
skies = finalize(skies, desi_target, dum, dum, sky=1)
log.info('Done...t = {:.1f}s'.format(time() - start))
return skies
def plot_sky_positions(ragood,
decgood,
rabad,
decbad,
objs,
navoid=2.,
limits=None,
plotname=None):
"""plot an example set of sky positions to check if they avoid real objects
Parameters
----------
ragood : :class:`~numpy.array`
array of RA coordinates for good sky positions
decgood : :class:`~numpy.array`
array of Dec coordinates for good sky positions
rabad : :class:`~numpy.array`
array of RA coordinates for bad sky positions, i.e. positions that
ARE within the avoidance zones of the "objs"
decbad : :class:`~numpy.array`
array of Dec coordinates for bad sky positions, i.e. positions that
ARE within the avoidance zones of the "objs"
objs : :class:`~numpy.ndarray`
numpy structured array with UPPERCASE columns, OR a string
tractor/sweep filename. Must contain at least the columns
"RA", "DEC", "SHAPEDEV_R", "SHAPEEXP_R"
navoid : :class:`float`, optional, defaults to 2.
the number of times the galaxy half-light radius (or seeing) that
objects (objs) were avoided out to when generating sky positions
limits : :class:`~numpy.array`, optional, defaults to None
plot limits in the form [ramin, ramax, decmin, decmax] if None
is passed, then the entire area is plotted
plotname : :class:`str`, defaults to None
If a name is passed use matplotlib's savefig command to save the
plot to that file name. Otherwise, display the plot
Returns
-------
Nothing
"""
import matplotlib.pyplot as plt
from lvmtarget.brightstar import ellipses
from matplotlib.patches import Polygon
from matplotlib.collections import PatchCollection
#ADM initialize the default logger
from lvmutil.log import get_logger, DEBUG
log = get_logger(DEBUG)
start = time()
#ADM set up the figure and the axis labels
fig, ax = plt.subplots(figsize=(8, 8))
ax.set_xlabel('RA (o)')
ax.set_ylabel('Dec (o)')
#ADM check if input objs is a filename or the actual data
if isinstance(objs, str):
objs = io.read_tractor(objs)
#ADM coordinate limits for the passed objs
ramin, ramax = np.min(objs["RA"]), np.max(objs["RA"])
decmin, decmax = np.min(objs["DEC"]), np.max(objs["DEC"])
#ADM guard against wraparound bug (which should never be an issue for the sweeps, anyway)
if ramax - ramin > 180.:
ramax -= 360.
#ADM the avoidance separation (in arcseconds) for each object based on
#ADM its half-light radius/profile
sep = calculate_separations(objs, navoid)
#ADM the maximum such separation for any object in the passed set IN DEGREES
maxrad = max(sep) / 3600.
#ADM limit the figure range based on the passed objs
if limits is None:
ralo, rahi = ramin, ramax
declo, dechi = decmin, decmax
else:
ralo, rahi, declo, dechi = limits
dum = plt.axis([ralo, rahi, declo, dechi])
#ADM plot good and bad sky positions
ax.scatter(ragood, decgood, marker='.', facecolors='none', edgecolors='k')
ax.scatter(rabad, decbad, marker='.', facecolors='r')
#ADM the size that defines a PSF versus an elliptical avoidance zone
sepsplit = (psfsize * navoid) + 1e-8
smallsepw = np.where(sep <= sepsplit)[0]
bigsepw = np.where(sep > sepsplit)[0]
#ADM first the PSF or "small separation objects"...
#ADM set up the ellipse shapes based on sizes of the past avoidance zones
#ADM remembering to stretch by the COS term to de-project the sky
minoraxis = sep / 3600.
majoraxis = minoraxis / np.cos(np.radians(objs["DEC"]))
log.info('Plotting avoidance zones...t = {:.1f}s'.format(time() - start))
#ADM plot the avoidance zones as circles, stretched by their DEC position
out = ellipses(objs[smallsepw]["RA"],
objs[smallsepw]["DEC"],
2 * majoraxis[smallsepw],
2 * minoraxis[smallsepw],
alpha=0.4,
edgecolor='none')
#ADM now the elliptical or "large separation objects"...
#ADM loop through the DEV and EXP shapes, and create polygons
#ADM of them to plot (where they're defined)
patches = []
for i, valobj in enumerate(bigsepw):
if objs[valobj]["SHAPEEXP_R"] > 0:
#ADM points on the ellipse boundary for EXP objects
ras, decs = ellipse_boundary(objs[valobj]["RA"],
objs[valobj]["DEC"],
objs[valobj]["SHAPEEXP_R"] * navoid,
objs[valobj]["SHAPEEXP_E1"],
objs[valobj]["SHAPEEXP_E2"])
polygon = Polygon(np.array(list(zip(ras, decs))), True)
patches.append(polygon)
if objs[valobj]["SHAPEDEV_R"] > 0:
#ADM points on the ellipse boundary for DEV objects
ras, decs = ellipse_boundary(objs[valobj]["RA"],
objs[valobj]["DEC"],
objs[valobj]["SHAPEDEV_R"] * navoid,
objs[valobj]["SHAPEDEV_E1"],
objs[valobj]["SHAPEDEV_E2"])
polygon = Polygon(np.array(list(zip(ras, decs))), True)
patches.append(polygon)
p = PatchCollection(patches, alpha=0.4, facecolors='b', edgecolors='b')
ax.add_collection(p)
#ADM display the plot, if requested
if plotname is None:
log.info('Displaying plot...t = {:.1f}s'.format(time() - start))
plt.show()
else:
plt.savefig(plotname)
log.info('Done...t = {:.1f}s'.format(time() - start))
return
def generate_sky_positions(objs, navoid=2., nskymin=None):
"""Use a basic avoidance-of-other-objects approach to generate sky positions
Parameters
----------
objs : :class:`~numpy.ndarray`
numpy structured array with UPPERCASE columns, OR a string
tractor/sweep filename. Must contain at least the columns
"RA", "DEC", "SHAPEDEV_R", "SHAPEEXP_R"
navoid : :class:`float`, optional, defaults to 2.
the number of times the galaxy half-light radius (or seeing) to avoid
objects out to when placing sky fibers
nskymin : :class:`float`, optional, defaults to reading from lvmmodel.io
the minimum DENSITY of sky fibers to generate
Returns
-------
ragood : :class:`~numpy.array`
array of RA coordinates for good sky positions
decgood : :class:`~numpy.array`
array of Dec coordinates for good sky positions
rabad : :class:`~numpy.array`
array of RA coordinates for bad sky positions, i.e. positions that
ARE within navoid half-light radii of a galaxy (or navoid*psfsize
arcseconds for a PSF object)
decbad : :class:`~numpy.array`
array of Dec coordinates for bad sky positions, i.e. positions that
ARE within navoid half-light radii of a galaxy (or navoid*psfsize
arcseconds for a PSF object)
"""
#ADM set up the default log
from lvmutil.log import get_logger, DEBUG
log = get_logger(DEBUG)
start = time()
#ADM if needed, determine the minimum density of sky fibers to generate
if nskymin is None:
nskymin = density_of_sky_fibers()
log.info(
'Generating sky positions at a density of {} per sq. deg....t = {:.1f}s'
.format(nskymin,
time() - start))
#ADM check if input objs is a filename or the actual data
if isinstance(objs, str):
objs = io.read_tractor(objs)
nobjs = len(objs)
#ADM an avoidance separation (in arcseconds) for each
#ADM object based on its half-light radius/profile
log.info('Calculating avoidance zones...t = {:.1f}s'.format(time() -
start))
sep = calculate_separations(objs, navoid)
#ADM the maximum such separation for any object in the passed set in arcsec
maxrad = max(sep)
#ADM the coordinate limits and corresponding area of the passed objs
ramin, ramax = np.min(objs["RA"]), np.max(objs["RA"])
#ADM guard against the wraparound bug (should never be an issue for the sweeps, anyway)
if ramax - ramin > 180.:
ramax -= 360.
decmin, decmax = np.min(objs["DEC"]), np.max(objs["DEC"])
sindecmin, sindecmax = np.sin(np.radians(decmin)), np.sin(
np.radians(decmax))
spharea = (ramax - ramin) * np.degrees(sindecmax - sindecmin)
log.info('Area covered by passed objects is {:.3f} sq. deg....t = {:.1f}s'.
format(spharea,
time() - start))
#ADM how many sky positions we need to generate to meet the minimum density requirements
nskies = int(spharea * nskymin)
#ADM how many sky positions to generate, given that we'll reject objects close to bad
#ADM sources. The factor of 1.2 was derived by trial-and-error...but this doesn't need
#ADM to be optimal as this algorithm should become more sophisticated
nchunk = int(nskies * 1.2)
#ADM arrays of GOOD sky positions to populate with coordinates
ragood, decgood = np.empty(nskies), np.empty(nskies)
#ADM lists of BAD sky positions to populate with coordinates
rabad, decbad = [], []
#ADM ngenerate will become zero when we generate enough GOOD sky positions
ngenerate = nskies
while ngenerate:
#ADM generate random points in RA and Dec (correctly distributed on the sphere)
log.info('Generated {} test positions...t = {:.1f}s'.format(
nchunk,
time() - start))
ra = np.random.uniform(ramin, ramax, nchunk)
dec = np.degrees(
np.arcsin(1. -
np.random.uniform(1 - sindecmax, 1 - sindecmin, nchunk)))
#ADM set up the coordinate objects
cskies = SkyCoord(ra * u.degree, dec * u.degree)
cobjs = SkyCoord(objs["RA"] * u.degree, objs["DEC"] * u.degree)
#ADM split the objects up using a separation of just larger than psfsize*navoid
#ADM arcseconds in order to speed up the coordinate matching when we have some
#ADM objects with large radii
sepsplit = (psfsize * navoid) + 1e-8
bigsepw = np.where(sep > sepsplit)[0]
smallsepw = np.where(sep <= sepsplit)[0]
#ADM set up a list of skies that don't match an object
goodskies = np.ones(len(cskies), dtype=bool)
#ADM guard against the case where there are no objects with small radii
if len(smallsepw) > 0:
#ADM match the small-separation objects and flag any skies that match such an object
log.info(
'Match positions out to {:.1f} arcsec...t = {:.1f}s'.format(
sepsplit,
time() - start))
idskies, idobjs, d2d, _ = cobjs[smallsepw].search_around_sky(
cskies, sepsplit * u.arcsec)
w = np.where(d2d.arcsec < sep[smallsepw[idobjs]])
#ADM remember to guard against the case with no bad positions
if len(w[0]) > 0:
goodskies[idskies[w]] = False
#ADM guard against the case where there are no objects with large radii
if len(bigsepw) > 0:
#ADM match the large-separation objects and flag any skies that match such an object
log.info(
'(Elliptically) Match additional positions out to {:.1f} arcsec...t = {:.1f}s'
.format(maxrad,
time() - start))
#ADM the transformation matrixes (shapes) for DEV and EXP objects
#ADM with a factor of navoid in the half-light-radius
TDEV = ellipse_matrix(objs[bigsepw]["SHAPEDEV_R"] * navoid,
objs[bigsepw]["SHAPEDEV_E1"],
objs[bigsepw]["SHAPEDEV_E2"])
TEXP = ellipse_matrix(objs[bigsepw]["SHAPEEXP_R"] * navoid,
objs[bigsepw]["SHAPEEXP_E1"],
objs[bigsepw]["SHAPEEXP_E2"])
#ADM loop through the DEV and EXP shapes, and where they are defined
#ADM (radius > tiny), determine if any sky positions occupy them
for i, valobj in enumerate(bigsepw):
if i % 1000 == 999:
log.info('Done {}/{}...t = {:.1f}s'.format(
i, len(bigsepw),
time() - start))
is_in = np.array(np.zeros(nchunk), dtype='bool')
if objs[valobj]["SHAPEEXP_R"] > 1e-16:
is_in += is_in_ellipse_matrix(cskies.ra.deg,
cskies.dec.deg,
cobjs[valobj].ra.deg,
cobjs[valobj].dec.deg,
TEXP[..., i])
if objs[valobj]["SHAPEDEV_R"] > 1e-16:
is_in += is_in_ellipse_matrix(cskies.ra.deg,
cskies.dec.deg,
cobjs[valobj].ra.deg,
cobjs[valobj].dec.deg,
TDEV[..., i])
w = np.where(is_in)
if len(w[0]) > 0:
goodskies[w] = False
#ADM good sky positions we found
wgood = np.where(goodskies)[0]
n1 = nskies - ngenerate
ngenerate = max(0, ngenerate - len(wgood))
n2 = nskies - ngenerate
ragood[n1:n2] = ra[wgood[:n2 - n1]]
decgood[n1:n2] = dec[wgood[:n2 - n1]]
log.info(
'Need to generate a further {} positions...t = {:.1f}s'.format(
ngenerate,
time() - start))
#ADM bad sky positions we found
wbad = np.where(~goodskies)[0]
rabad.append(list(ra[wbad]))
decbad.append(list(dec[wbad]))
#ADM we potentially created nested lists for the bad skies, so need to flatten
#ADM also we can't need more bad sky positions than total sky positions
rabad = np.array([item for sublist in rabad for item in sublist])[:nskies]
decbad = np.array([item for sublist in decbad
for item in sublist])[:nskies]
log.info('Done...t = {:.1f}s'.format(time() - start))
return ragood, decgood, rabad, decbad
def apply_sandbox_cuts(objects, FoMthresh=None, MethodELG='XD'):
"""Perform target selection on objects, returning target mask arrays
Args:
objects: numpy structured array with UPPERCASE columns needed for
target selection, OR a string tractor/sweep filename
FoMthresh: If this is passed, then run apply_XD_globalerror and
return the Figure of Merits calculated for the ELGs in a file
"FoM.fits" in the current working directory.
MethodELG: Three methods available for ELGs
XD: Extreme deconvolution
RF_spectro: Random Forest trained with spectro z (VIPERS and DEEP2)
RF_photo: Random Forest trained with photo z (HSC)
Returns:
(desi_target, bgs_target, mws_target) where each element is
an ndarray of target selection bitmask flags for each object
If FoMthresh is passed
where FoM are the Figure of Merit values calculated by apply_XD_globalerror
Bugs:
If objects is a astropy Table with lowercase column names, this
converts them to UPPERCASE in-place, thus modifying the input table.
To avoid this, pass in objects.copy() instead.
See lvmtarget.targetmask for the definition of each bit
"""
#- Check if objects is a filename instead of the actual data
if isinstance(objects, str):
from lvmtarget import io
objects = io.read_tractor(objects)
#- ensure uppercase column names if astropy Table
if isinstance(objects, (Table, Row)):
for col in list(objects.columns.values()):
if not col.name.isupper():
col.name = col.name.upper()
#- undo Milky Way extinction
flux = unextinct_fluxes(objects)
gflux = flux['GFLUX']
rflux = flux['RFLUX']
zflux = flux['ZFLUX']
w1flux = flux['W1FLUX']
w2flux = flux['W2FLUX']
objtype = objects['TYPE']
gflux_ivar = objects['FLUX_IVAR_G']
rflux_ivar = objects['FLUX_IVAR_R']
zflux_ivar = objects['FLUX_IVAR_Z']
gflux_snr = objects['FLUX_G'] * np.sqrt(objects['FLUX_IVAR_G'])
rflux_snr = objects['FLUX_R'] * np.sqrt(objects['FLUX_IVAR_R'])
zflux_snr = objects['FLUX_Z'] * np.sqrt(objects['FLUX_IVAR_Z'])
w1flux_snr = objects['FLUX_W1'] * np.sqrt(objects['FLUX_IVAR_W1'])
#- DR1 has targets off the edge of the brick; trim to just this brick
try:
primary = objects['BRICK_PRIMARY']
except (KeyError, ValueError):
if _is_row(objects):
primary = True
else:
primary = np.ones_like(objects, dtype=bool)
lrg = isLRG_2016v3(gflux=gflux,
rflux=rflux,
zflux=zflux,
w1flux=w1flux,
gflux_ivar=gflux_ivar,
rflux_snr=rflux_snr,
zflux_snr=zflux_snr,
w1flux_snr=w1flux_snr,
primary=primary)
if FoMthresh is not None:
if (MethodELG == 'XD'):
elg, FoM = apply_XD_globalerror(
objects,
FoMthresh,
glim=23.8,
rlim=23.4,
zlim=22.4,
gr_ref=0.5,
rz_ref=0.5,
reg_r=1e-4 / (0.025**2 * 0.05),
f_i=[1., 1., 0., 0.25, 0., 0.25, 0.],
gmin=21.,
gmax=24.)
elif (MethodELG == 'RF_photo'):
elg, FoM = isELG_randomforest(pcut=abs(FoMthresh),
primary=primary,
zflux=zflux,
rflux=rflux,
gflux=gflux,
w1flux=w1flux,
w2flux=w2flux,
training='photo')
elif (MethodELG == 'RF_spectro'):
elg, FoM = isELG_randomforest(pcut=abs(FoMthresh),
primary=primary,
zflux=zflux,
rflux=rflux,
gflux=gflux,
w1flux=w1flux,
w2flux=w2flux,
training='spectro')
#- construct the targetflag bits
#- Currently our only cuts are DECam based (i.e. South)
desi_target = lrg * desi_mask.LRG_SOUTH
#desi_target |= elg * desi_mask.ELG_SOUTH
#desi_target |= qso * desi_mask.QSO_SOUTH
desi_target |= lrg * desi_mask.LRG
if FoMthresh is not None:
desi_target |= elg * desi_mask.ELG
#desi_target |= qso * desi_mask.QSO
#desi_target |= fstd * desi_mask.STD_FSTAR
bgs_target = np.zeros_like(desi_target)
#bgs_target = bgs_bright * bgs_mask.BGS_BRIGHT
#bgs_target |= bgs_bright * bgs_mask.BGS_BRIGHT_SOUTH
#bgs_target |= bgs_faint * bgs_mask.BGS_FAINT
#bgs_target |= bgs_faint * bgs_mask.BGS_FAINT_SOUTH
#- nothing for MWS yet; will be GAIA-based
#if isinstance(bgs_target, numbers.Integral):
# mws_target = 0
#else:
# mws_target = np.zeros_like(bgs_target)
mws_target = np.zeros_like(desi_target)
#- Are any BGS or MWS bit set? Tell desi_target too.
desi_target |= (bgs_target != 0) * desi_mask.BGS_ANY
desi_target |= (mws_target != 0) * desi_mask.MWS_ANY
if FoMthresh is not None:
keep = (desi_target != 0)
write_fom_targets(objects[keep], FoM[keep], desi_target[keep],
bgs_target[keep], mws_target[keep])
return desi_target, bgs_target, mws_target
def _select_targets_file(filename):
'''Returns targets in filename that pass the cuts'''
objects = io.read_tractor(filename)
desi_target, bgs_target, mws_target = apply_cuts(objects, qso_selection)
return _finalize_targets(objects, desi_target, bgs_target, mws_target)
def apply_cuts(objects, qso_selection='randomforest'):
"""Perform target selection on objects, returning target mask arrays
Args:
objects: numpy structured array with UPPERCASE columns needed for
target selection, OR a string tractor/sweep filename
Options:
qso_selection : algorithm to use for QSO selection; valid options
are 'colorcuts' and 'randomforest'
Returns:
(desi_target, bgs_target, mws_target) where each element is
an ndarray of target selection bitmask flags for each object
Bugs:
If objects is a astropy Table with lowercase column names, this
converts them to UPPERCASE in-place, thus modifying the input table.
To avoid this, pass in objects.copy() instead.
See lvmtarget.targetmask for the definition of each bit
"""
#- Check if objects is a filename instead of the actual data
if isinstance(objects, str):
objects = io.read_tractor(objects)
#- ensure uppercase column names if astropy Table
if isinstance(objects, (Table, Row)):
for col in list(objects.columns.values()):
if not col.name.isupper():
col.name = col.name.upper()
obs_rflux = objects['FLUX_R'] # observed r-band flux (used for F standards, below)
#- undo Milky Way extinction
flux = unextinct_fluxes(objects)
gflux = flux['GFLUX']
rflux = flux['RFLUX']
zflux = flux['ZFLUX']
w1flux = flux['W1FLUX']
w2flux = flux['W2FLUX']
objtype = objects['TYPE']
release = objects['RELEASE']
gfluxivar = objects['FLUX_IVAR_G']
gfracflux = objects['FRACFLUX_G'].T # note transpose
rfracflux = objects['FRACFLUX_R'].T # note transpose
zfracflux = objects['FRACFLUX_Z'].T # note transpose
gsnr = objects['FLUX_G'] * np.sqrt(objects['FLUX_IVAR_G'])
rsnr = objects['FLUX_R'] * np.sqrt(objects['FLUX_IVAR_R'])
zsnr = objects['FLUX_Z'] * np.sqrt(objects['FLUX_IVAR_Z'])
w1snr = objects['FLUX_W1'] * np.sqrt(objects['FLUX_IVAR_W1'])
w2snr = objects['FLUX_W2'] * np.sqrt(objects['FLUX_IVAR_W2'])
# Delta chi2 between PSF and SIMP morphologies; note the sign....
dchisq = objects['DCHISQ']
deltaChi2 = dchisq[...,0] - dchisq[...,1]
#ADM remove handful of NaN values from DCHISQ values and make them unselectable
w = np.where(deltaChi2 != deltaChi2)
#ADM this is to catch the single-object case for unit tests
if len(w[0]) > 0:
deltaChi2[w] = -1e6
#- DR1 has targets off the edge of the brick; trim to just this brick
try:
primary = objects['BRICK_PRIMARY']
except (KeyError, ValueError):
if _is_row(objects):
primary = True
else:
primary = np.ones_like(objects, dtype=bool)
lrg, lrg1pass, lrg2pass = isLRGpass(primary=primary, gflux=gflux, rflux=rflux,
zflux=zflux, w1flux=w1flux, gflux_ivar=gfluxivar,
rflux_snr=rsnr, zflux_snr=zsnr, w1flux_snr=w1snr)
elg = isELG(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux)
bgs_bright = isBGS_bright(primary=primary, rflux=rflux, objtype=objtype)
bgs_faint = isBGS_faint(primary=primary, rflux=rflux, objtype=objtype)
if qso_selection=='colorcuts' :
qso = isQSO_cuts(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux,
w1flux=w1flux, w2flux=w2flux, deltaChi2=deltaChi2, objtype=objtype,
w1snr=w1snr, w2snr=w2snr, release=release)
elif qso_selection == 'randomforest':
qso = isQSO_randomforest(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux,
w1flux=w1flux, w2flux=w2flux, deltaChi2=deltaChi2, objtype=objtype, release=release)
else:
raise ValueError('Unknown qso_selection {}; valid options are {}'.format(qso_selection,
qso_selection_options))
#ADM Make sure to pass all of the needed columns! At one point we stopped
#ADM passing objtype, which meant no standards were being returned.
fstd = isFSTD(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux,
gfracflux=gfracflux, rfracflux=rfracflux, zfracflux=zfracflux,
gsnr=gsnr, rsnr=rsnr, zsnr=zsnr,
obs_rflux=obs_rflux, objtype=objtype)
fstd_bright = isFSTD(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux,
gfracflux=gfracflux, rfracflux=rfracflux, zfracflux=zfracflux,
gsnr=gsnr, rsnr=rsnr, zsnr=zsnr,
obs_rflux=obs_rflux, objtype=objtype, bright=True)
# Construct the targetflag bits; currently our only cuts are DECam based
# (i.e. South). This should really be refactored into a dedicated function.
desi_target = lrg * desi_mask.LRG_SOUTH
desi_target |= elg * desi_mask.ELG_SOUTH
desi_target |= qso * desi_mask.QSO_SOUTH
desi_target |= lrg * desi_mask.LRG
desi_target |= elg * desi_mask.ELG
desi_target |= qso * desi_mask.QSO
#ADM add the per-pass information
desi_target |= lrg1pass * desi_mask.LRG_1PASS
desi_target |= lrg2pass * desi_mask.LRG_2PASS
# Standards; still need to set STD_WD
desi_target |= fstd * desi_mask.STD_FSTAR
desi_target |= fstd_bright * desi_mask.STD_BRIGHT
# BGS, bright and faint
bgs_target = bgs_bright * bgs_mask.BGS_BRIGHT
bgs_target |= bgs_bright * bgs_mask.BGS_BRIGHT_SOUTH
bgs_target |= bgs_faint * bgs_mask.BGS_FAINT
bgs_target |= bgs_faint * bgs_mask.BGS_FAINT_SOUTH
# Nothing for MWS yet; will be GAIA-based.
if isinstance(bgs_target, numbers.Integral):
mws_target = 0
else:
mws_target = np.zeros_like(bgs_target)
# Are any BGS or MWS bit set? Tell desi_target too.
desi_target |= (bgs_target != 0) * desi_mask.BGS_ANY
desi_target |= (mws_target != 0) * desi_mask.MWS_ANY
return desi_target, bgs_target, mws_target
#from astropy.io import fits
from lvmtarget.cuts import apply_cuts
from lvmtarget.io import read_tractor
import fitsio
tractordir = '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/tractor/330'
#tractordir = '/data/legacysurvey/dr3.1/tractor/330/'
for brick in ['3301m002', '3301m007', '3303p000']:
filepath = '{}/tractor-{}.fits'.format(tractordir, brick)
desi_target = apply_cuts(filepath)[0]
yes = np.where(desi_target != 0)[0]
no = np.where(desi_target == 0)[0]
keep = np.concatenate([yes[0:3], no[0:3]])
# data, hdr = fits.getdata(filepath, header=True)
# fits.writeto('t/'+basename(filepath), data[keep], header=hdr)
data, hdr = read_tractor(filepath, header=True)
fitsio.write('t/' + basename(filepath), data[keep], header=hdr)
sweepdir = '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1'
#sweepdir = '/data/legacysurvey/dr2p/sweep/'
for radec in ['310m005-320p000', '320m005-330p000', '330m005-340p000']:
filepath = '{}/sweep-{}.fits'.format(sweepdir, radec)
desi_target = apply_cuts(filepath)[0]
yes = np.where(desi_target != 0)[0]
no = np.where(desi_target == 0)[0]
keep = np.concatenate([yes[0:3], no[0:3]])
# data, hdr = fits.getdata(filepath, header=True)
# fits.writeto('t/'+basename(filepath), data[keep], header=hdr)
data, hdr = read_tractor(filepath, header=True)
fitsio.write('t/' + basename(filepath), data[keep], header=hdr)
def make_bright_star_mask(
bands,
maglim,
numproc=4,
rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1',
infilename=None,
outfilename=None):
"""Make a bright star mask from a structure of bright stars drawn from the sweeps
Parameters
----------
bands : :class:`str`
A magnitude band from the sweeps, e.g., "G", "R", "Z".
Can pass multiple bands as string, e.g. "GRZ", in which case maglim has to be a
list of the same length as the string
maglim : :class:`float`
The upper limit in that magnitude band for which to assemble a list of bright stars.
Can pass a list of magnitude limits, in which case bands has to be a string of the
same length (e.g., "GRZ" for [12.3,12.7,12.6]
numproc : :class:`int`, optional
Number of processes over which to parallelize
rootdirname : :class:`str`, optional, defaults to dr3
Root directory containing either sweeps or tractor files...e.g. for dr3 this might be
/global/project/projectdirs/cosmo/data/legacysurvey/dr3/sweeps/dr3.1
infilename : :class:`str`, optional,
if this exists, then the list of bright stars is read in from the file of this name
if this is not passed, then code defaults to deriving the recarray of bright stars
via a call to collect_bright_stars
outfilename : :class:`str`, optional, defaults to not writing anything to file
(FITS) File name to which to write the output bright star mask
Returns
-------
:class:`recarray`
The bright star mask in the form RA, DEC, TARGETID, IN_RADIUS, NEAR_RADIUS (may also be written to file
if "outfilename" is passed)
The radii are in ARCMINUTES
TARGETID is as calculated in :mod:`lvmtarget.targets.encode_targetid`
Notes
-----
- IN_RADIUS is a smaller radius that corresponds to the IN_BRIGHT_OBJECT bit in data/targetmask.yaml
- NEAR_RADIUS is a radius that corresponds to the NEAR_BRIGHT_OBJECT bit in data/targetmask.yaml
- Currently uses the radius-as-a-function-of-B-mag for Tycho stars from the BOSS mask (in every band) to set
the NEAR_RADIUS:
R = (0.0802B*B - 1.860B + 11.625) (see Eqn. 9 of https://arxiv.org/pdf/1203.6594.pdf)
and half that radius to set the IN_RADIUS.
- It's an open question as to what the correct radii are for DESI observations
"""
#ADM set bands to uppercase if passed as lower case
bands = bands.upper()
#ADM the band names and nobs columns as arrays instead of strings
bandnames = np.array(["FLUX_" + band for band in bands])
nobsnames = np.array(["NOBS_" + band for band in bands])
#ADM force the input maglim to be a list (in case a single value was passed)
if type(maglim) == type(16) or type(maglim) == type(16.):
maglim = [maglim]
if len(bandnames) != len(maglim):
raise IOError(
'bands has to be the same length as maglim and {} does not equal {}'
.format(len(bandnames), len(maglim)))
#ADM change input magnitude(s) to a flux to test against
fluxlim = 10.**((22.5 - np.array(maglim)) / 2.5)
if infilename is not None:
objs = io.read_tractor(infilename)
else:
objs = collect_bright_stars(bands, maglim, numproc, rootdirname,
outfilename)
#ADM write the fluxes and bands as arrays instead of named columns
fluxes = objs[bandnames].view(
objs[bandnames].dtype[0]).reshape(objs[bandnames].shape + (-1, ))
nobs = objs[nobsnames].view(
objs[nobsnames].dtype[0]).reshape(objs[nobsnames].shape + (-1, ))
#ADM set any observations with NOBS = 0 to have small flux so glitches don't end up as bright star masks.
w = np.where(nobs == 0)
if len(w[0]) > 0:
fluxes[w] = 0.
#ADM limit to the passed faint limit
w = np.where(np.any(fluxes > fluxlim, axis=1))
fluxes = fluxes[w]
objs = objs[w]
#ADM grab the (GRZ) magnitudes for observations
#ADM and record only the largest flux (smallest magnitude)
fluxmax = np.max(fluxes, axis=1)
mags = 22.5 - 2.5 * np.log10(fluxmax)
#ADM convert the largest magnitude into radii for "in" and "near" bright objects. This will require
#ADM more consideration to determine the truly correct numbers for DESI
near_radius = (0.0802 * mags * mags - 1.860 * mags + 11.625)
in_radius = 0.5 * (0.0802 * mags * mags - 1.860 * mags + 11.625)
#ADM calculate the TARGETID
targetid = encode_targetid(objid=objs['OBJID'],
brickid=objs['BRICKID'],
release=objs['RELEASE'])
#ADM create an output recarray that is just RA, Dec, TARGETID and the radius
done = objs[['RA', 'DEC']].copy()
done = rfn.append_fields(done, ["TARGETID", "IN_RADIUS", "NEAR_RADIUS"],
[targetid, in_radius, near_radius],
usemask=False,
dtypes=['>i8', '<f8', '<f8'])
if outfilename is not None:
fitsio.write(outfilename, done, clobber=True)
return done
