def test_cuts_basic(self):
#- Cuts work with either data or filenames
desi, bgs, mws = cuts.apply_cuts(self.tractorfiles[0])
desi, bgs, mws = cuts.apply_cuts(self.sweepfiles[0])
data = io.read_tractor(self.tractorfiles[0])
desi, bgs, mws = cuts.apply_cuts(data)
data = io.read_tractor(self.sweepfiles[0])
desi, bgs, mws = cuts.apply_cuts(data)
def test_tractor_columns(self):
tscolumns = io.tscolumns + ['BRICK_PRIMARY',]
tractorfile = io.list_tractorfiles(self.datadir)[0]
data = io.read_tractor(tractorfile)
self.assertEqual(set(data.dtype.names), set(tscolumns))
columns = ['BX', 'BY']
data = io.read_tractor(tractorfile, columns=columns)
self.assertEqual(set(data.dtype.names), set(columns))
data = io.read_tractor(tractorfile, columns=tuple(columns))
self.assertEqual(set(data.dtype.names), set(columns))
def test_readwrite_tractor(self):
tractorfile = io.list_tractorfiles(self.datadir)[0]
sweepfile = io.list_sweepfiles(self.datadir)[0]
data = io.read_tractor(sweepfile)
data = io.read_tractor(tractorfile)
self.assertEqual(len(data), 6) #- test data has 6 objects per file
data, hdr = io.read_tractor(tractorfile, header=True)
self.assertEqual(len(data), 6) #- test data has 6 objects per file
io.write_targets(self.testfile, data, indir=self.datadir)
d2, h2 = fits.getdata(self.testfile, header=True)
self.assertEqual(h2['DEPVER02'], self.datadir)
self.assertEqual(data.dtype.names, d2.dtype.names)
for column in data.dtype.names:
self.assertTrue(np.all(data[column] == d2[column]))
def _select_targets_file(filename):
'''Returns targets in filename that pass the cuts'''
from desitarget import io
objects = io.read_tractor(filename, columns=columns)
keep = apply_cuts(objects)
return io.fix_tractor_dr1_dtype(objects[keep])
def _get_bright_stars(filename):
'''Retrieves bright stars from a sweeps/Tractor file'''
objs = io.read_tractor(filename)
#ADM write the fluxes as an array instead of as named columns
fluxes = objs[bandnames].view(
objs[bandnames].dtype[0]).reshape(objs[bandnames].shape + (-1, ))
#ADM Retain rows for which ANY band is brighter than maglim
w = np.where(np.any(fluxes > fluxlim, axis=1))
if len(w[0]) > 0:
return objs[w]
def test_unextinct_fluxes(self):
targets = io.read_tractor(self.tractorfiles[0])
t1 = cuts.unextinct_fluxes(targets)
self.assertTrue(isinstance(t1, np.ndarray))
t2 = cuts.unextinct_fluxes(Table(targets))
self.assertTrue(isinstance(t2, Table))
for col in ['GFLUX', 'RFLUX', 'ZFLUX', 'W1FLUX', 'W2FLUX', 'WFLUX']:
self.assertIn(col, t1.dtype.names)
self.assertIn(col, t2.dtype.names)
self.assertTrue(np.all(t1[col] == t2[col]))
def test_unextinct_fluxes(self):
targets = io.read_tractor(self.tractorfiles[0])
t1 = cuts.unextinct_fluxes(targets)
self.assertTrue(isinstance(t1, np.ndarray))
t2 = cuts.unextinct_fluxes(Table(targets))
self.assertTrue(isinstance(t2, Table))
for col in ['GFLUX', 'RFLUX', 'ZFLUX', 'W1FLUX', 'W2FLUX']:
self.assertIn(col, t1.dtype.names)
self.assertIn(col, t2.dtype.names)
self.assertTrue(np.all(t1[col] == t2[col]))
def _get_bright_sources(filename):
"""Retrieves bright sources from a sweeps/Tractor file"""
objs = io.read_tractor(filename)
# ADM write the fluxes as an array instead of as named columns.
# ADM Retain rows for which ANY band is brighter than maglim.
ok = np.zeros(objs[bandnames[0]].shape, dtype=bool)
for i, bandname in enumerate(bandnames):
ok |= (objs[bandname] > fluxlim[i])
w = np.where(ok)
if len(w[0]) > 0:
return objs[w]
def test_cuts_basic(self):
"""Test cuts work with either data or filenames
"""
# ADM test for tractor files.
# ADM No QSO cuts for speed. This doesn't affect coverage.
cmx, pshift = cuts.apply_cuts(self.tractorfiles[0],
cmxdir=self.cmxdir,
noqso=True)
data = io.read_tractor(self.tractorfiles[0])
cmx2, pshift2 = cuts.apply_cuts(data, cmxdir=self.cmxdir, noqso=True)
self.assertTrue(np.all(cmx == cmx2))
self.assertTrue(np.all(pshift == pshift2))
# ADM test for sweeps files.
# ADM No QSO cuts for speed. This doesn't affect coverage.
cmx, pshift = cuts.apply_cuts(self.sweepfiles[0],
cmxdir=self.cmxdir,
noqso=True)
data = io.read_tractor(self.sweepfiles[0])
cmx2, pshift2 = cuts.apply_cuts(data, cmxdir=self.cmxdir, noqso=True)
self.assertTrue(np.all(cmx == cmx2))
self.assertTrue(np.all(pshift == pshift2))
def test_fix_dr1(self):
'''test the DR1 TYPE dype fix (make everything S4)'''
# - First, break it
files = io.list_sweepfiles(self.datadir)
objects = io.read_tractor(files[0])
dt = objects.dtype.descr
for i in range(len(dt)):
if dt[i][0] == 'TYPE':
dt[i] = ('TYPE', 'S10')
break
badobjects = objects.astype(np.dtype(dt))
newobjects = io.fix_tractor_dr1_dtype(badobjects)
self.assertEqual(newobjects['TYPE'].dtype, np.dtype('S4'))
def test_fix_dr1(self):
'''test the DR1 TYPE dype fix (make everything S4)'''
#- First, break it
files = io.list_sweepfiles(self.datadir)
objects = io.read_tractor(files[0])
dt = objects.dtype.descr
for i in range(len(dt)):
if dt[i][0] == 'TYPE':
dt[i] = ('TYPE', 'S10')
break
badobjects = objects.astype(np.dtype(dt))
newobjects = io.fix_tractor_dr1_dtype(badobjects)
self.assertEqual(newobjects['TYPE'].dtype, np.dtype('S4'))
def apply_cuts(objects):
"""Apply the cuts we want."""
#zfaint = 22.0
#keep = np.where((np.sum((objects['DECAM_NOBS'][:,[1,2,4]]>=3)*1,axis=1)==3)*
# (np.sum((objects['DECAM_ANYMASK'][:,[1,2,4]]>0)*1,axis=1)==0)*
# (objects['DECAM_FLUX'][:,4]<(10**(-0.4*(zfaint-22.5)))))[0]
if isinstance(objects, (str, unicode)):
from desitarget import io
objects = io.read_tractor(objects)
#- ensure uppercase column names if astropy Table
if isinstance(objects, (Table, Row)):
for col in objects.columns.itervalues():
if not col.name.isupper():
col.name = col.name.upper()
#- undo Milky Way extinction
flux = unextinct_fluxes(objects)
gflux = flux['GFLUX']
rflux = flux['RFLUX']
zflux = flux['ZFLUX']
w1flux = flux['W1FLUX']
wflux = flux['WFLUX']
#- DR1 has targets off the edge of the brick; trim to just this brick
try:
primary = objects['BRICK_PRIMARY']
except (KeyError, ValueError):
if _is_row(objects):
primary = True
else:
primary = np.ones_like(objects, dtype=bool)
blue = isBLUE(primary=primary, gflux=gflux, zflux=zflux, rflux=rflux)
#keep = np.where((blue==1))[0]
#keep = np.where((blue==1)*
# (np.sum((objects['DECAM_ANYMASK'][:,[1,2,4]]>0)*1,axis=1)==0))[0]
keep = np.where(
(blue == 1) * (np.sum(
(objects['DECAM_ANYMASK'][:, [1, 2, 4]] > 0) * 1, axis=1) == 0) *
(np.sum(
(objects['DECAM_FLUX'][:, [1, 2, 4]] > 0) * 1, axis=1) == 3))[0]
return keep
def apply_cuts(objects):
"""Apply the cuts we want."""
#zfaint = 22.0
#keep = np.where((np.sum((objects['DECAM_NOBS'][:,[1,2,4]]>=3)*1,axis=1)==3)*
# (np.sum((objects['DECAM_ANYMASK'][:,[1,2,4]]>0)*1,axis=1)==0)*
# (objects['DECAM_FLUX'][:,4]<(10**(-0.4*(zfaint-22.5)))))[0]
if isinstance(objects, (str, unicode)):
from desitarget import io
objects = io.read_tractor(objects)
#- ensure uppercase column names if astropy Table
if isinstance(objects, (Table, Row)):
for col in objects.columns.itervalues():
if not col.name.isupper():
col.name = col.name.upper()
#- undo Milky Way extinction
flux = unextinct_fluxes(objects)
gflux = flux['GFLUX']
rflux = flux['RFLUX']
zflux = flux['ZFLUX']
w1flux = flux['W1FLUX']
wflux = flux['WFLUX']
#- DR1 has targets off the edge of the brick; trim to just this brick
try:
primary = objects['BRICK_PRIMARY']
except (KeyError, ValueError):
if _is_row(objects):
primary = True
else:
primary = np.ones_like(objects, dtype=bool)
blue = isBLUE(primary=primary, gflux=gflux, zflux=zflux, rflux=rflux)
#keep = np.where((blue==1))[0]
#keep = np.where((blue==1)*
# (np.sum((objects['DECAM_ANYMASK'][:,[1,2,4]]>0)*1,axis=1)==0))[0]
keep = np.where((blue==1)*
(np.sum((objects['DECAM_ANYMASK'][:,[1,2,4]]>0)*1,axis=1)==0)*
(np.sum((objects['DECAM_FLUX'][:,[1,2,4]]>0)*1,axis=1)==3))[0]
return keep
def _select_targets_file(filename):
'''Returns targets in filename that pass the cuts'''
from desitarget import io
objects = io.read_tractor(filename)
desi_target, bgs_target, mws_target = apply_cuts(objects)
#- desi_target includes BGS_ANY and MWS_ANY, so we can filter just
#- on desi_target != 0
keep = (desi_target != 0)
objects = objects[keep]
desi_target = desi_target[keep]
bgs_target = bgs_target[keep]
mws_target = mws_target[keep]
#- Add *_target mask columns
targets = desitarget.targets.finalize(
objects, desi_target, bgs_target, mws_target)
return io.fix_tractor_dr1_dtype(targets)
def _check_input_files(filename):
'''Check for corrupted values in a file'''
from functools import partial
from os.path import getsize
#ADM read in Tractor or sweeps files
objects = io.read_tractor(filename)
#ADM if everything is OK the default meassage will be "OK"
filemessageroot = 'OK'
filemessageend = ''
#ADM columns that shouldn't have zero values
cols = [
'BRICKID',
# 'RA_IVAR', 'DEC_IVAR',
'MW_TRANSMISSION_G',
'MW_TRANSMISSION_R',
'MW_TRANSMISSION_Z',
# 'WISE_FLUX',
# 'WISE_MW_TRANSMISSION','DCHISQ'
]
#ADM for each of these columnes that shouldn't have zero values,
#ADM loop through and look for zero values
for colname in cols:
if np.min(objects[colname]) == 0:
filemessageroot = "WARNING...some values are zero for"
filemessageend += " " + colname
#ADM now, loop through entries in the file and search for 4096-byte
#ADM blocks that are all zeros (a sign of corruption in file-writing)
#ADM Note that fits files are padded by 2880 bytes, so we only want to
#ADM process the file length (in bytes) - 2880
bytestop = getsize(filename) - 2880
with open(filename, 'rb') as f:
for block_number, data in enumerate(
iter(partial(f.read, 4096), b'')):
if not any(data):
if block_number * 4096 < bytestop:
filemessageroot = "WARNING...some values are zero for"
filemessageend += ' 4096-byte-block-#{0}'.format(
block_number)
return [filename, filemessageroot + filemessageend]
def _get_gaia_matches(fnwdir):
'''wrapper on match_gaia_to_primary() given a file name'''
# ADM extract the output file name.
fn = os.path.basename(fnwdir)
outfile = '{}/{}'.format(outdir, fn.replace(".fits", ender))
# ADM read in the objects.
objs, hdr = io.read_tractor(fnwdir, header=True)
# ADM match to Gaia sources.
gaiainfo = match_gaia_to_primary(objs)
log.info('Done with Gaia match for {} primary objects...t = {:.1f}s'
.format(len(objs), time()-start))
# ADM remove the GAIA_RA, GAIA_DEC columns as they aren't
# ADM in the imaging surveys data model.
gaiainfo = pop_gaia_coords(gaiainfo)
# ADM add the Gaia column information to the sweeps array.
for col in gaiainfo.dtype.names:
objs[col] = gaiainfo[col]
fitsio.write(outfile, objs, extname='SWEEP', header=hdr, clobber=True)
return True
def _select_targets_file(filename):
'''Returns targets in filename that pass the cuts'''
objects = io.read_tractor(filename)
cmx_target, priority_shift = apply_cuts(objects, cmxdir=cmxdir)
return _finalize_targets(objects, cmx_target, priority_shift)
def apply_cuts(objects, cmxdir=None):
"""Perform commissioning (cmx) target selection on objects, return target mask arrays
Parameters
----------
objects: numpy structured array with UPPERCASE columns needed for
target selection, OR a string tractor/sweep filename
cmxdir : :class:`str`, optional, defaults to :envvar:`CMX_DIR`
Directory in which to find commmissioning files to which to match, such as the
CALSPEC stars. If not specified, the cmx directory is taken to be the value of
the :envvar:`CMX_DIR` environment variable.
Returns
-------
:class:`~numpy.ndarray`
commissioning target selection bitmask flags for each object
See desitarget.cmx.cmx_targetmask.cmx_mask for the definition of each bit
"""
# -Check if objects is a filename instead of the actual data
if isinstance(objects, str):
objects = io.read_tractor(objects)
# -Ensure uppercase column names if astropy Table
if isinstance(objects, (Table, Row)):
for col in list(objects.columns.values()):
if not col.name.isupper():
col.name = col.name.upper()
# ADM retrieve/check the cmxdir.
cmxdir = _get_cmxdir(cmxdir)
# ADM As we need the column names.
colnames = _get_colnames(objects)
photsys_north, photsys_south, obs_rflux, gflux, rflux, zflux, \
w1flux, w2flux, rfiberflux, objtype, release, gfluxivar, rfluxivar, zfluxivar, \
gnobs, rnobs, znobs, gfracflux, rfracflux, zfracflux, \
gfracmasked, rfracmasked, zfracmasked, \
gfracin, rfracin, zfracin, gallmask, rallmask, zallmask, \
gsnr, rsnr, zsnr, w1snr, w2snr, dchisq, deltaChi2, brightstarinblob = \
_prepare_optical_wise(objects, colnames=colnames)
# ADM in addition, cmx needs ra and dec.
ra, dec = objects["RA"], objects["DEC"]
# ADM Currently only coded for objects with Gaia matches
# ADM (e.g. DR6 or above). Fail for earlier Data Releases.
if np.any(release < 6000):
log.critical('Commissioning cuts only coded for DR6 or above')
raise ValueError
if (np.max(objects['PMRA']) == 0.) & np.any(release < 7000):
d = "/project/projectdirs/desi/target/gaia_dr2_match_dr6"
log.info("Zero objects have a proper motion.")
log.critical(
"Did you mean to send the Gaia-matched sweeps in, e.g., {}?".
format(d))
raise IOError
# Process the Gaia inputs for target selection.
gaia, pmra, pmdec, parallax, parallaxovererror, parallaxerr, gaiagmag, gaiabmag, \
gaiarmag, gaiaaen, gaiadupsource, Grr, gaiaparamssolved, gaiabprpfactor, \
gaiasigma5dmax, galb = _prepare_gaia(objects, colnames=colnames)
# ADM a couple of extra columns; the observed g/z fluxes.
obs_gflux, obs_zflux = objects['FLUX_G'], objects['FLUX_Z']
# ADM initially, every object passes the cuts (is True).
# ADM need to guard against the case of a single row being passed.
# ADM initially every class has a priority shift of zero.
if _is_row(objects):
primary = np.bool_(True)
priority_shift = np.array(0)
else:
primary = np.ones_like(objects, dtype=bool)
priority_shift = np.zeros_like(objects, dtype=int)
# ADM determine if an object passes the default logic for cmx stars.
isgood = passesSTD_logic(gfracflux=gfracflux,
rfracflux=rfracflux,
zfracflux=zfracflux,
objtype=objtype,
gaia=gaia,
pmra=pmra,
pmdec=pmdec,
aen=gaiaaen,
dupsource=gaiadupsource,
paramssolved=gaiaparamssolved,
primary=primary)
# ADM determine if an object is a "dither" star.
std_dither, shift_dither = isSTD_dither(obs_gflux=obs_gflux,
obs_rflux=obs_rflux,
obs_zflux=obs_zflux,
isgood=isgood,
primary=primary)
# ADM set up an initial priority shift.
# ADM determine if an object is a bright test star.
std_test = isSTD_test(obs_gflux=obs_gflux,
obs_rflux=obs_rflux,
obs_zflux=obs_zflux,
isgood=isgood,
primary=primary)
# ADM determine if an object matched a CALSPEC standard.
std_calspec = isSTD_calspec(ra=ra, dec=dec, cmxdir=cmxdir, primary=primary)
# ADM determine if an object is SV0_STD_BRIGHT. Resembles first
# ADM iteration of SV, but locked in cmx_cuts (and could be altered).
sv0_std_bright = isSV0_STD_bright(gflux=gflux,
rflux=rflux,
zflux=zflux,
pmra=pmra,
pmdec=pmdec,
parallax=parallax,
gaiagmag=gaiagmag,
isgood=isgood,
primary=primary)
# ADM determine if an object is SV0_BGS
sv0_bgs = isSV0_BGS(rflux=rflux, objtype=objtype, primary=primary)
# ADM determine if an object is SV0_MWS
sv0_mws = isSV0_MWS(rflux=rflux,
obs_rflux=obs_rflux,
objtype=objtype,
gaiagmag=gaiagmag,
gaiabmag=gaiabmag,
gaiarmag=gaiarmag,
pmra=pmra,
pmdec=pmdec,
parallax=parallax,
parallaxovererror=parallaxovererror,
photbprpexcessfactor=gaiabprpfactor,
astrometricsigma5dmax=gaiasigma5dmax,
galb=galb,
gaia=gaia,
primary=primary)
# ADM Construct the targetflag bits.
cmx_target = std_dither * cmx_mask.STD_GAIA
cmx_target |= std_test * cmx_mask.STD_TEST
cmx_target |= std_calspec * cmx_mask.STD_CALSPEC
cmx_target |= sv0_std_bright * cmx_mask.SV0_STD_BRIGHT
cmx_target |= sv0_bgs * cmx_mask.SV0_BGS
cmx_target |= sv0_mws * cmx_mask.SV0_MWS
# ADM update the priority with any shifts.
# ADM we may need to update this logic if there are other shifts.
priority_shift[std_dither] = shift_dither[std_dither]
return cmx_target, priority_shift
def apply_cuts(objects, qso_selection='randomforest'):
"""Perform target selection on objects, returning target mask arrays
Args:
objects: numpy structured array with UPPERCASE columns needed for
target selection, OR a string tractor/sweep filename
Options:
qso_selection : algorithm to use for QSO selection; valid options
are 'colorcuts' and 'randomforest'
Returns:
(desi_target, bgs_target, mws_target) where each element is
an ndarray of target selection bitmask flags for each object
Bugs:
If objects is a astropy Table with lowercase column names, this
converts them to UPPERCASE in-place, thus modifying the input table.
To avoid this, pass in objects.copy() instead.
See desitarget.targetmask for the definition of each bit
"""
#- Check if objects is a filename instead of the actual data
if isinstance(objects, str):
objects = io.read_tractor(objects)
#- ensure uppercase column names if astropy Table
if isinstance(objects, (Table, Row)):
for col in list(objects.columns.values()):
if not col.name.isupper():
col.name = col.name.upper()
obs_rflux = objects[
'FLUX_R'] # observed r-band flux (used for F standards, below)
#- undo Milky Way extinction
flux = unextinct_fluxes(objects)
gflux = flux['GFLUX']
rflux = flux['RFLUX']
zflux = flux['ZFLUX']
w1flux = flux['W1FLUX']
w2flux = flux['W2FLUX']
objtype = objects['TYPE']
gfluxivar = objects['FLUX_IVAR_G']
gfracflux = objects['FRACFLUX_G'].T # note transpose
rfracflux = objects['FRACFLUX_R'].T # note transpose
zfracflux = objects['FRACFLUX_Z'].T # note transpose
gsnr = objects['FLUX_G'] * np.sqrt(objects['FLUX_IVAR_G'])
rsnr = objects['FLUX_R'] * np.sqrt(objects['FLUX_IVAR_R'])
zsnr = objects['FLUX_Z'] * np.sqrt(objects['FLUX_IVAR_Z'])
w1snr = objects['FLUX_W1'] * np.sqrt(objects['FLUX_IVAR_W1'])
w2snr = objects['FLUX_W2'] * np.sqrt(objects['FLUX_IVAR_W2'])
# Delta chi2 between PSF and SIMP morphologies; note the sign....
dchisq = objects['DCHISQ']
deltaChi2 = dchisq[..., 0] - dchisq[..., 1]
#ADM remove handful of NaN values from DCHISQ values and make them unselectable
w = np.where(deltaChi2 != deltaChi2)
#ADM this is to catch the single-object case for unit tests
if len(w[0]) > 0:
deltaChi2[w] = -1e6
#- DR1 has targets off the edge of the brick; trim to just this brick
try:
primary = objects['BRICK_PRIMARY']
except (KeyError, ValueError):
if _is_row(objects):
primary = True
else:
primary = np.ones_like(objects, dtype=bool)
lrg = isLRG(primary=primary,
gflux=gflux,
rflux=rflux,
zflux=zflux,
w1flux=w1flux,
gflux_ivar=gfluxivar,
rflux_snr=rsnr,
zflux_snr=zsnr,
w1flux_snr=w1snr)
elg = isELG(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux)
bgs_bright = isBGS_bright(primary=primary, rflux=rflux, objtype=objtype)
bgs_faint = isBGS_faint(primary=primary, rflux=rflux, objtype=objtype)
if qso_selection == 'colorcuts':
qso = isQSO_cuts(primary=primary,
zflux=zflux,
rflux=rflux,
gflux=gflux,
w1flux=w1flux,
w2flux=w2flux,
deltaChi2=deltaChi2,
objtype=objtype,
w1snr=w1snr,
w2snr=w2snr)
elif qso_selection == 'randomforest':
qso = isQSO_randomforest(primary=primary,
zflux=zflux,
rflux=rflux,
gflux=gflux,
w1flux=w1flux,
w2flux=w2flux,
deltaChi2=deltaChi2,
objtype=objtype)
else:
raise ValueError(
'Unknown qso_selection {}; valid options are {}'.format(
qso_selection, qso_selection_options))
#ADM Make sure to pass all of the needed columns! At one point we stopped
#ADM passing objtype, which meant no standards were being returned.
fstd = isFSTD(primary=primary,
zflux=zflux,
rflux=rflux,
gflux=gflux,
gfracflux=gfracflux,
rfracflux=rfracflux,
zfracflux=zfracflux,
gsnr=gsnr,
rsnr=rsnr,
zsnr=zsnr,
obs_rflux=obs_rflux,
objtype=objtype)
fstd_bright = isFSTD(primary=primary,
zflux=zflux,
rflux=rflux,
gflux=gflux,
gfracflux=gfracflux,
rfracflux=rfracflux,
zfracflux=zfracflux,
gsnr=gsnr,
rsnr=rsnr,
zsnr=zsnr,
obs_rflux=obs_rflux,
objtype=objtype,
bright=True)
# Construct the targetflag bits; currently our only cuts are DECam based
# (i.e. South). This should really be refactored into a dedicated function.
desi_target = lrg * desi_mask.LRG_SOUTH
desi_target |= elg * desi_mask.ELG_SOUTH
desi_target |= qso * desi_mask.QSO_SOUTH
desi_target |= lrg * desi_mask.LRG
desi_target |= elg * desi_mask.ELG
desi_target |= qso * desi_mask.QSO
# Standards; still need to set STD_WD
desi_target |= fstd * desi_mask.STD_FSTAR
desi_target |= fstd_bright * desi_mask.STD_BRIGHT
# BGS, bright and faint
bgs_target = bgs_bright * bgs_mask.BGS_BRIGHT
bgs_target |= bgs_bright * bgs_mask.BGS_BRIGHT_SOUTH
bgs_target |= bgs_faint * bgs_mask.BGS_FAINT
bgs_target |= bgs_faint * bgs_mask.BGS_FAINT_SOUTH
# Nothing for MWS yet; will be GAIA-based.
if isinstance(bgs_target, numbers.Integral):
mws_target = 0
else:
mws_target = np.zeros_like(bgs_target)
# Are any BGS or MWS bit set? Tell desi_target too.
desi_target |= (bgs_target != 0) * desi_mask.BGS_ANY
desi_target |= (mws_target != 0) * desi_mask.MWS_ANY
return desi_target, bgs_target, mws_target
def apply_cuts(objects):
"""Perform target selection on objects, returning target mask arrays
Args:
objects: numpy structured array with UPPERCASE columns needed for
target selection, OR a string tractor/sweep filename
Returns:
(desi_target, bgs_target, mws_target) where each element is
an ndarray of target selection bitmask flags for each object
Bugs:
If objects is a astropy Table with lowercase column names, this
converts them to UPPERCASE in-place, thus modifying the input table.
To avoid this, pass in objects.copy() instead.
See desitarget.targetmask for the definition of each bit
"""
#- Check if objects is a filename instead of the actual data
if isinstance(objects, (str, unicode)):
from desitarget import io
objects = io.read_tractor(objects)
#- ensure uppercase column names if astropy Table
if isinstance(objects, (Table, Row)):
for col in objects.columns.itervalues():
if not col.name.isupper():
col.name = col.name.upper()
#- undo Milky Way extinction
flux = unextinct_fluxes(objects)
gflux = flux['GFLUX']
rflux = flux['RFLUX']
zflux = flux['ZFLUX']
w1flux = flux['W1FLUX']
w2flux = flux['W2FLUX']
objtype = objects['TYPE']
wise_snr = objects['WISE_FLUX'] * np.sqrt(objects['WISE_FLUX_IVAR'])
#- DR1 has targets off the edge of the brick; trim to just this brick
try:
primary = objects['BRICK_PRIMARY']
except (KeyError, ValueError):
if _is_row(objects):
primary = True
else:
primary = np.ones_like(objects, dtype=bool)
lrg = isLRG(primary=primary, zflux=zflux, rflux=rflux, w1flux=w1flux)
elg = isELG(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux)
bgs = isBGS(primary=primary, rflux=rflux, objtype=objtype)
qso = isQSO(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux,
w1flux=w1flux, w2flux=w2flux, objtype=objtype,
wise_snr=wise_snr)
#----- Standard stars
fstd = isFSTD_colors(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux)
fstd &= psflike(objtype)
fracflux = objects['DECAM_FRACFLUX'].T
signal2noise = objects['DECAM_FLUX'] * np.sqrt(objects['DECAM_FLUX_IVAR'])
with warnings.catch_warnings():
# FIXME: what warnings are we ignoring?
warnings.simplefilter('ignore')
for j in (1,2,4): #- g, r, z
fstd &= fracflux[j] < 0.04
fstd &= signal2noise[..., j] > 10
#- observed flux; no Milky Way extinction
obs_rflux = objects['DECAM_FLUX'][..., 2]
fstd &= obs_rflux < 10**((22.5-16.0)/2.5)
fstd &= obs_rflux > 10**((22.5-19.0)/2.5)
#-----
#- construct the targetflag bits
#- Currently our only cuts are DECam based (i.e. South)
desi_target = lrg * desi_mask.LRG_SOUTH
desi_target |= elg * desi_mask.ELG_SOUTH
desi_target |= qso * desi_mask.QSO_SOUTH
desi_target |= lrg * desi_mask.LRG
desi_target |= elg * desi_mask.ELG
desi_target |= qso * desi_mask.QSO
desi_target |= fstd * desi_mask.STD_FSTAR
bgs_target = bgs * bgs_mask.BGS_BRIGHT
bgs_target |= bgs * bgs_mask.BGS_BRIGHT_SOUTH
#- nothing for MWS yet; will be GAIA-based
if isinstance(bgs_target, int):
mws_target = 0
else:
mws_target = np.zeros_like(bgs_target)
#- Are any BGS or MWS bit set? Tell desi_target too.
desi_target |= (bgs_target != 0) * desi_mask.BGS_ANY
desi_target |= (mws_target != 0) * desi_mask.MWS_ANY
return desi_target, bgs_target, mws_target
def make_bright_star_mask(
bands,
maglim,
numproc=4,
rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1',
infilename=None,
outfilename=None,
verbose=False):
"""Make a bright star mask from a structure of bright stars drawn from the sweeps
Parameters
----------
bands : :class:`str`
A magnitude band from the sweeps, e.g., "G", "R", "Z".
Can pass multiple bands as string, e.g. "GRZ", in which case maglim has to be a
list of the same length as the string
maglim : :class:`float`
The upper limit in that magnitude band for which to assemble a list of bright stars.
Can pass a list of magnitude limits, in which case bands has to be a string of the
same length (e.g., "GRZ" for [12.3,12.7,12.6]
numproc : :class:`int`, optional
Number of processes over which to parallelize
rootdirname : :class:`str`, optional, defaults to dr3
Root directory containing either sweeps or tractor files...e.g. for dr3 this might be
/global/project/projectdirs/cosmo/data/legacysurvey/dr3/sweeps/dr3.1
infilename : :class:`str`, optional,
if this exists, then the list of bright stars is read in from the file of this name
if this is not passed, then code defaults to deriving the recarray of bright stars
via a call to collect_bright_stars
outfilename : :class:`str`, optional, defaults to not writing anything to file
(FITS) File name to which to write the output bright star mask
verbose : :class:`bool`, optional
Send to write progress to screen
Returns
-------
:class:`recarray`
The bright star mask in the form RA, DEC, TARGETID, IN_RADIUS, NEAR_RADIUS (may also be written to file
if "outfilename" is passed)
The radii are in ARCMINUTES
TARGETID is as calculated in :mod:`desitarget.targets.encode_targetid`
Notes
-----
- IN_RADIUS is a smaller radius that corresponds to the IN_BRIGHT_OBJECT bit in data/targetmask.yaml
- NEAR_RADIUS is a radius that corresponds to the NEAR_BRIGHT_OBJECT bit in data/targetmask.yaml
- Currently uses the radius-as-a-function-of-B-mag for Tycho stars from the BOSS mask (in every band) to set
the NEAR_RADIUS:
R = (0.0802B*B - 1.860B + 11.625) (see Eqn. 9 of https://arxiv.org/pdf/1203.6594.pdf)
and half that radius to set the IN_RADIUS.
- It's an open question as to what the correct radii are for DESI observations
"""
#ADM set bands to uppercase if passed as lower case
bands = bands.upper()
#ADM the band names and nobs columns as arrays instead of strings
bandnames = np.array(["FLUX_" + band for band in bands])
nobsnames = np.array(["NOBS_" + band for band in bands])
#ADM force the input maglim to be a list (in case a single value was passed)
if type(maglim) == type(16) or type(maglim) == type(16.):
maglim = [maglim]
if len(bandnames) != len(maglim):
raise IOError(
'bands has to be the same length as maglim and {} does not equal {}'
.format(len(bandnames), len(maglim)))
#ADM change input magnitude(s) to a flux to test against
fluxlim = 10.**((22.5 - np.array(maglim)) / 2.5)
if infilename is not None:
objs = io.read_tractor(infilename)
else:
objs = collect_bright_stars(bands, maglim, numproc, rootdirname,
outfilename, verbose)
#ADM write the fluxes and bands as arrays instead of named columns
fluxes = objs[bandnames].view(
objs[bandnames].dtype[0]).reshape(objs[bandnames].shape + (-1, ))
nobs = objs[nobsnames].view(
objs[nobsnames].dtype[0]).reshape(objs[nobsnames].shape + (-1, ))
#ADM set any observations with NOBS = 0 to have small flux so glitches don't end up as bright star masks.
w = np.where(nobs == 0)
if len(w[0]) > 0:
fluxes[w] = 0.
#ADM limit to the passed faint limit
w = np.where(np.any(fluxes > fluxlim, axis=1))
fluxes = fluxes[w]
objs = objs[w]
#ADM grab the (GRZ) magnitudes for observations
#ADM and record only the largest flux (smallest magnitude)
fluxmax = np.max(fluxes, axis=1)
mags = 22.5 - 2.5 * np.log10(fluxmax)
#ADM convert the largest magnitude into radii for "in" and "near" bright objects. This will require
#ADM more consideration to determine the truly correct numbers for DESI
near_radius = (0.0802 * mags * mags - 1.860 * mags + 11.625)
in_radius = 0.5 * (0.0802 * mags * mags - 1.860 * mags + 11.625)
#ADM calculate the TARGETID
targetid = encode_targetid(objid=objs['OBJID'],
brickid=objs['BRICKID'],
release=objs['RELEASE'])
#ADM create an output recarray that is just RA, Dec, TARGETID and the radius
done = objs[['RA', 'DEC']].copy()
done = rfn.append_fields(done, ["TARGETID", "IN_RADIUS", "NEAR_RADIUS"],
[targetid, in_radius, near_radius],
usemask=False,
dtypes=['>i8', '<f8', '<f8'])
if outfilename is not None:
fitsio.write(outfilename, done, clobber=True)
return done
def apply_cuts(objects, cmxdir=None, noqso=False):
"""Commissioning (cmx) target selection, return target mask arrays.
Parameters
----------
objects: numpy structured array with UPPERCASE columns needed for
target selection, OR a string tractor/sweep filename
cmxdir : :class:`str`, optional, defaults to :envvar:`CMX_DIR`
Directory to find commmissioning files to which to match, such
as the CALSPEC stars. If not specified, the cmx directory is
taken to be the value of :envvar:`CMX_DIR`.
noqso : :class:`boolean`, optional, defaults to ``False``
If passed, do not run the quasar selection. All QSO bits will be
set to zero. Intended use is to speed unit tests.
Returns
-------
:class:`~numpy.ndarray`
commissioning target selection bitmask flags for each object.
See desitarget.cmx.cmx_targetmask.cmx_mask for bit definitions.
"""
# -Check if objects is a filename instead of the actual data
if isinstance(objects, str):
objects = io.read_tractor(objects)
# -Ensure uppercase column names if astropy Table
if isinstance(objects, (Table, Row)):
for col in list(objects.columns.values()):
if not col.name.isupper():
col.name = col.name.upper()
# ADM retrieve/check the cmxdir.
cmxdir = _get_cmxdir(cmxdir)
# ADM As we need the column names.
colnames = _get_colnames(objects)
photsys_north, photsys_south, obs_rflux, gflux, rflux, zflux, \
w1flux, w2flux, rfiberflux, objtype, release, \
gfluxivar, rfluxivar, zfluxivar, \
gnobs, rnobs, znobs, gfracflux, rfracflux, zfracflux, \
gfracmasked, rfracmasked, zfracmasked, \
gfracin, rfracin, zfracin, gallmask, rallmask, zallmask, \
gsnr, rsnr, zsnr, w1snr, w2snr, dchisq, deltaChi2, maskbits = \
_prepare_optical_wise(objects)
# ADM in addition, cmx needs ra and dec.
ra, dec = objects["RA"], objects["DEC"]
# ADM Currently only coded for objects with Gaia matches
# ADM (e.g. DR6 or above). Fail for earlier Data Releases.
if np.any(release < 6000):
log.critical('Commissioning cuts only coded for DR6 or above')
raise ValueError
if (np.max(objects['PMRA']) == 0.) & np.any(release < 7000):
d = "/project/projectdirs/desi/target/gaia_dr2_match_dr6"
log.info("Zero objects have a proper motion.")
log.critical(
"Did you mean to send the Gaia-matched sweeps in, e.g., {}?".
format(d))
raise IOError
# Process the Gaia inputs for target selection.
gaia, pmra, pmdec, parallax, parallaxovererror, parallaxerr, gaiagmag, gaiabmag, \
gaiarmag, gaiaaen, gaiadupsource, Grr, gaiaparamssolved, gaiabprpfactor, \
gaiasigma5dmax, galb = _prepare_gaia(objects, colnames=colnames)
# ADM a couple of extra columns; the observed g/z fluxes.
obs_gflux, obs_zflux = objects['FLUX_G'], objects['FLUX_Z']
# ADM initially, every object passes the cuts (is True).
# ADM need to guard against the case of a single row being passed.
# ADM initially every class has a priority shift of zero.
if _is_row(objects):
primary = np.bool_(True)
priority_shift = np.array(0)
else:
primary = np.ones_like(objects, dtype=bool)
priority_shift = np.zeros_like(objects, dtype=int)
# ADM determine if an object passes the default logic for cmx stars.
isgood = passesSTD_logic(gfracflux=gfracflux,
rfracflux=rfracflux,
zfracflux=zfracflux,
objtype=objtype,
gaia=gaia,
pmra=pmra,
pmdec=pmdec,
aen=gaiaaen,
dupsource=gaiadupsource,
paramssolved=gaiaparamssolved,
primary=primary)
# ADM determine if an object is a "dither" star.
# ADM and priority shift.
std_dither, shift_dither = isSTD_dither(obs_gflux=obs_gflux,
obs_rflux=obs_rflux,
obs_zflux=obs_zflux,
isgood=isgood,
primary=primary)
# ADM determine if an object is a bright test star.
std_test = isSTD_test(obs_gflux=obs_gflux,
obs_rflux=obs_rflux,
obs_zflux=obs_zflux,
isgood=isgood,
primary=primary)
# ADM determine if an object matched a CALSPEC standard.
std_calspec = isSTD_calspec(ra=ra, dec=dec, cmxdir=cmxdir, primary=primary)
# ADM determine if an object is SV0_BGS.
sv0_bgs = isSV0_BGS(rflux=rflux, objtype=objtype, primary=primary)
# ADM determine if an object is SV0_MWS or WD.
sv0_mws, sv0_wd = isSV0_MWS(rflux=rflux,
obs_rflux=obs_rflux,
objtype=objtype,
gaiagmag=gaiagmag,
gaiabmag=gaiabmag,
gaiarmag=gaiarmag,
pmra=pmra,
pmdec=pmdec,
parallax=parallax,
parallaxerr=parallaxerr,
parallaxovererror=parallaxovererror,
photbprpexcessfactor=gaiabprpfactor,
astrometricsigma5dmax=gaiasigma5dmax,
gaiaaen=gaiaaen,
paramssolved=gaiaparamssolved,
galb=galb,
gaia=gaia,
primary=primary)
# ADM determine if an object is SV0_LRG.
sv0_lrg = isSV0_LRG(gflux=gflux,
rflux=rflux,
zflux=zflux,
w1flux=w1flux,
rflux_snr=rsnr,
zflux_snr=zsnr,
w1flux_snr=w1snr,
gflux_ivar=gfluxivar,
primary=primary)
# ADM determine if an object is SV0_ELG.
sv0_elg = isSV0_ELG(primary=primary,
gflux=gflux,
rflux=rflux,
zflux=zflux,
gsnr=gsnr,
rsnr=rsnr,
zsnr=zsnr,
maskbits=maskbits)
# ADM determine if an object is SV0_QSO.
if noqso:
# ADM don't run quasar cuts if requested, for speed.
sv0_qso = ~primary
else:
sv0_qso = isSV0_QSO(primary=primary,
zflux=zflux,
rflux=rflux,
gflux=gflux,
w1flux=w1flux,
w2flux=w2flux,
objtype=objtype,
dchisq=dchisq,
maskbits=maskbits)
# ADM run the SV0 STD target types for both faint and bright.
# ADM Make sure to pass all of the needed columns! At one point we stopped
# ADM passing objtype, which meant no standards were being returned.
sv0_std_classes = []
for bright in [False, True]:
sv0_std_classes.append(
isSV0_STD(primary=primary,
zflux=zflux,
rflux=rflux,
gflux=gflux,
gfracflux=gfracflux,
rfracflux=rfracflux,
zfracflux=zfracflux,
gfracmasked=gfracmasked,
rfracmasked=rfracmasked,
objtype=objtype,
zfracmasked=zfracmasked,
gnobs=gnobs,
rnobs=rnobs,
znobs=znobs,
gfluxivar=gfluxivar,
rfluxivar=rfluxivar,
zfluxivar=zfluxivar,
gaia=gaia,
astrometricexcessnoise=gaiaaen,
paramssolved=gaiaparamssolved,
pmra=pmra,
pmdec=pmdec,
parallax=parallax,
dupsource=gaiadupsource,
gaiagmag=gaiagmag,
gaiabmag=gaiabmag,
gaiarmag=gaiarmag,
bright=bright))
sv0_std_faint, sv0_std_bright = sv0_std_classes
# ADM the nominal main survey cuts for standard stars. These are currently
# ADM identical to the SV0 cuts, so treat accordingly:
std_faint, std_bright = sv0_std_classes
# ADM Construct the target flag bits.
cmx_target = std_dither * cmx_mask.STD_GAIA
cmx_target |= std_test * cmx_mask.STD_TEST
cmx_target |= std_calspec * cmx_mask.STD_CALSPEC
cmx_target |= sv0_std_faint * cmx_mask.SV0_STD_FAINT
cmx_target |= sv0_std_bright * cmx_mask.SV0_STD_BRIGHT
cmx_target |= sv0_bgs * cmx_mask.SV0_BGS
cmx_target |= sv0_mws * cmx_mask.SV0_MWS
cmx_target |= sv0_lrg * cmx_mask.SV0_LRG
cmx_target |= sv0_elg * cmx_mask.SV0_ELG
cmx_target |= sv0_qso * cmx_mask.SV0_QSO
cmx_target |= sv0_wd * cmx_mask.SV0_WD
cmx_target |= std_faint * cmx_mask.STD_FAINT
cmx_target |= std_bright * cmx_mask.STD_BRIGHT
# ADM update the priority with any shifts.
# ADM we may need to update this logic if there are other shifts.
priority_shift[std_dither] = shift_dither[std_dither]
return cmx_target, priority_shift
def apply_sandbox_cuts(objects, FoMthresh=None, MethodELG='XD'):
"""Perform target selection on objects, returning target mask arrays
Args:
objects: numpy structured array with UPPERCASE columns needed for
target selection, OR a string tractor/sweep filename
FoMthresh: If this is passed, then run apply_XD_globalerror and
return the Figure of Merits calculated for the ELGs in a file
"FoM.fits" in the current working directory.
MethodELG: Three methods available for ELGs
XD: Extreme deconvolution
RF_spectro: Random Forest trained with spectro z (VIPERS and DEEP2)
RF_photo: Random Forest trained with photo z (HSC)
Returns:
(desi_target, bgs_target, mws_target) where each element is
an ndarray of target selection bitmask flags for each object
If FoMthresh is passed
where FoM are the Figure of Merit values calculated by apply_XD_globalerror
Bugs:
If objects is a astropy Table with lowercase column names, this
converts them to UPPERCASE in-place, thus modifying the input table.
To avoid this, pass in objects.copy() instead.
See desitarget.targetmask for the definition of each bit
"""
#- Check if objects is a filename instead of the actual data
if isinstance(objects, str):
from desitarget import io
objects = io.read_tractor(objects)
#- ensure uppercase column names if astropy Table
if isinstance(objects, (Table, Row)):
for col in list(objects.columns.values()):
if not col.name.isupper():
col.name = col.name.upper()
#- undo Milky Way extinction
flux = unextinct_fluxes(objects)
gflux = flux['GFLUX']
rflux = flux['RFLUX']
zflux = flux['ZFLUX']
w1flux = flux['W1FLUX']
w2flux = flux['W2FLUX']
objtype = objects['TYPE']
gflux_ivar = objects['FLUX_IVAR_G']
rflux_ivar = objects['FLUX_IVAR_R']
zflux_ivar = objects['FLUX_IVAR_Z']
gflux_snr = objects['FLUX_G'] * np.sqrt(objects['FLUX_IVAR_G'])
rflux_snr = objects['FLUX_R'] * np.sqrt(objects['FLUX_IVAR_R'])
zflux_snr = objects['FLUX_Z'] * np.sqrt(objects['FLUX_IVAR_Z'])
w1flux_snr = objects['FLUX_W1'] * np.sqrt(objects['FLUX_IVAR_W1'])
#- DR1 has targets off the edge of the brick; trim to just this brick
try:
primary = objects['BRICK_PRIMARY']
except (KeyError, ValueError):
if _is_row(objects):
primary = True
else:
primary = np.ones_like(objects, dtype=bool)
lrg = isLRG_2016v3(gflux=gflux,
rflux=rflux,
zflux=zflux,
w1flux=w1flux,
gflux_ivar=gflux_ivar,
rflux_snr=rflux_snr,
zflux_snr=zflux_snr,
w1flux_snr=w1flux_snr,
primary=primary)
if FoMthresh is not None:
if (MethodELG == 'XD'):
elg, FoM = apply_XD_globalerror(
objects,
FoMthresh,
glim=23.8,
rlim=23.4,
zlim=22.4,
gr_ref=0.5,
rz_ref=0.5,
reg_r=1e-4 / (0.025**2 * 0.05),
f_i=[1., 1., 0., 0.25, 0., 0.25, 0.],
gmin=21.,
gmax=24.)
elif (MethodELG == 'RF_photo'):
elg, FoM = isELG_randomforest(pcut=abs(FoMthresh),
primary=primary,
zflux=zflux,
rflux=rflux,
gflux=gflux,
w1flux=w1flux,
w2flux=w2flux,
training='photo')
elif (MethodELG == 'RF_spectro'):
elg, FoM = isELG_randomforest(pcut=abs(FoMthresh),
primary=primary,
zflux=zflux,
rflux=rflux,
gflux=gflux,
w1flux=w1flux,
w2flux=w2flux,
training='spectro')
#- construct the targetflag bits
#- Currently our only cuts are DECam based (i.e. South)
desi_target = lrg * desi_mask.LRG_SOUTH
#desi_target |= elg * desi_mask.ELG_SOUTH
#desi_target |= qso * desi_mask.QSO_SOUTH
desi_target |= lrg * desi_mask.LRG
if FoMthresh is not None:
desi_target |= elg * desi_mask.ELG
#desi_target |= qso * desi_mask.QSO
#desi_target |= fstd * desi_mask.STD_FSTAR
bgs_target = np.zeros_like(desi_target)
#bgs_target = bgs_bright * bgs_mask.BGS_BRIGHT
#bgs_target |= bgs_bright * bgs_mask.BGS_BRIGHT_SOUTH
#bgs_target |= bgs_faint * bgs_mask.BGS_FAINT
#bgs_target |= bgs_faint * bgs_mask.BGS_FAINT_SOUTH
#- nothing for MWS yet; will be GAIA-based
#if isinstance(bgs_target, numbers.Integral):
# mws_target = 0
#else:
# mws_target = np.zeros_like(bgs_target)
mws_target = np.zeros_like(desi_target)
#- Are any BGS or MWS bit set? Tell desi_target too.
desi_target |= (bgs_target != 0) * desi_mask.BGS_ANY
desi_target |= (mws_target != 0) * desi_mask.MWS_ANY
if FoMthresh is not None:
keep = (desi_target != 0)
write_fom_targets(objects[keep], FoM[keep], desi_target[keep],
bgs_target[keep], mws_target[keep])
return desi_target, bgs_target, mws_target
# from desitarget.gaiamatch import find_gaia_files
start = time()
tractordir = '/global/cfs/cdirs/cosmo/data/legacysurvey/dr8/south/tractor/330/'
# tractordir = '/project/projectdirs/cosmo/data/legacysurvey/dr7/tractor/330/'
# tractordir = '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/tractor/330'
# tractordir = '/data/legacysurvey/dr3.1/tractor/330/'
for brick in ['3301m002', '3301m007', '3303p000']:
filepath = '{}/tractor-{}.fits'.format(tractordir, brick)
desi_target, bgs_target, mws_target = apply_cuts(filepath)
# ADM as nobody is testing the MWS in the sandbox, yet, we need to
# ADM ensure we ignore MWS targets for testing the main algorithms.
yes = np.where((desi_target != 0) & (mws_target == 0))[0]
no = np.where(desi_target == 0)[0]
keep = np.concatenate([yes[0:3], no[0:3]])
data, hdr = read_tractor(filepath, header=True)
# ADM the FRACDEV and FRACDEV_IVAR columns can
# ADM contain some NaNs, which break testing.
wnan = np.where(data["FRACDEV"] != data["FRACDEV"])
if len(wnan[0]) > 0:
data["FRACDEV"][wnan] = 0.
wnan = np.where(data["FRACDEV_IVAR"] != data["FRACDEV_IVAR"])
if len(wnan[0]) > 0:
data["FRACDEV_IVAR"][wnan] = 0.
# ADM the "CONTINUE" comment keyword is not yet implemented
# ADM in fitsio, so delete it to prevent fitsio barfing on headers.
hdr.delete("CONTINUE")
fitsio.write('t/' + basename(filepath),
data[keep],
def make_bright_source_mask(
bands,
maglim,
numproc=4,
rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr5/sweep/5.0',
infilename=None,
outfilename=None):
"""Make a mask of bright sources from a structure of bright sources drawn from the sweeps.
Parameters
----------
bands : :class:`str`
A magnitude band from the sweeps, e.g., "G", "R", "Z".
Can pass multiple bands as string, e.g. ``"GRZ"``, in which case maglim has to be a
list of the same length as the string.
maglim : :class:`float`
The upper limit in that magnitude band for which to assemble a list of bright sources.
Can pass a list of magnitude limits, in which case bands has to be a string of the
same length (e.g., ``"GRZ"`` for [12.3,12.7,12.6]).
numproc : :class:`int`, optional
Number of processes over which to parallelize.
rootdirname : :class:`str`, optional, defaults to dr3
Root directory containing either sweeps or tractor files...e.g. for dr5 this might be
``/global/project/projectdirs/cosmo/data/legacysurvey/dr5/sweep/dr5.0``. This is only
used if ``infilename`` is not passed.
infilename : :class:`str`, optional,
if this exists, then the list of bright sources is read in from the file of this name.
if this is not passed, then code defaults to deriving the recarray of bright sources
from ``rootdirname`` via a call to ``collect_bright_sources``.
outfilename : :class:`str`, optional, defaults to not writing anything to file
(FITS) File name to which to write the output bright source mask.
Returns
-------
:class:`recarray`
- The bright source mask in the form ``RA`, ``DEC``, ``TARGETID``, ``IN_RADIUS``,
``NEAR_RADIUS``, ``E1``, ``E2``, ``TYPE``
(may also be written to file if ``outfilename`` is passed).
- ``TARGETID`` is as calculated in :mod:`desitarget.targets.encode_targetid`.
- The radii are in ARCSECONDS (they default to equivalents of half-light radii for ellipses).
- ``E1`` and ``E2`` are the ellipticity components as defined at the bottom of, e.g.:
http://legacysurvey.org/dr5/catalogs/.
- ``TYPE`` is the ``TYPE`` from the sweeps files, see, e.g.:
http://legacysurvey.org/dr5/files/#sweep-catalogs.
Notes
-----
- ``IN_RADIUS`` is a smaller radius that corresponds to the ``IN_BRIGHT_OBJECT`` bit in
``data/targetmask.yaml`` (and is in ARCSECONDS).
- ``NEAR_RADIUS`` is a radius that corresponds to the ``NEAR_BRIGHT_OBJECT`` bit in
``data/targetmask.yaml`` (and is in ARCSECONDS).
- Currently uses the radius-as-a-function-of-B-mag for Tycho stars from the BOSS mask
(in every band) to set the ``NEAR_RADIUS``:
R = (0.0802B*B - 1.860B + 11.625) (see Eqn. 9 of https://arxiv.org/pdf/1203.6594.pdf)
and half that radius to set the ``IN_RADIUS``. We convert this from arcminutes to arcseconds.
- It's an open question as to what the correct radii are for DESI observations.
"""
# ADM set bands to uppercase if passed as lower case.
bands = bands.upper()
# ADM the band names and nobs columns as arrays instead of strings.
bandnames = np.array(["FLUX_" + band for band in bands])
nobsnames = np.array(["NOBS_" + band for band in bands])
# ADM force the input maglim to be a list (in case a single value was passed).
if isinstance(maglim, int) or isinstance(maglim, float):
maglim = [maglim]
if len(bandnames) != len(maglim):
msg = "bands has to be the same length as maglim and {} does not equal {}".format(
len(bandnames), len(maglim))
raise IOError(msg)
# ADM change input magnitude(s) to a flux to test against.
fluxlim = 10.**((22.5 - np.array(maglim)) / 2.5)
if infilename is not None:
objs = io.read_tractor(infilename)
else:
objs = collect_bright_sources(bands, maglim, numproc, rootdirname,
outfilename)
# ADM write the fluxes and bands as arrays instead of named columns
# ADM to circumvent a numpy future warning, this requires two copies of the fluxes array.
fluxcheck = objs[bandnames].view(
objs[bandnames].dtype[0]).reshape(objs[bandnames].shape + (-1, ))
fluxes = fluxcheck.copy()
nobs = objs[nobsnames].view(
objs[nobsnames].dtype[0]).reshape(objs[nobsnames].shape + (-1, ))
# ADM set any observations with NOBS = 0 to have small flux so glitches don't end up as bright object masks.
w = np.where(nobs == 0)
if len(w[0]) > 0:
fluxes[w] = 0.
# ADM limit to the passed faint limit.
w = np.where(np.any(fluxes > fluxlim, axis=1))
fluxes = fluxes[w]
objs = objs[w]
# ADM grab the (GRZ) magnitudes for observations
# ADM and record only the largest flux (smallest magnitude).
fluxmax = np.max(fluxes, axis=1)
mags = 22.5 - 2.5 * np.log10(fluxmax)
# ADM each object's TYPE.
objtype = objs["TYPE"]
# ADM calculate the TARGETID.
targetid = encode_targetid(objid=objs['OBJID'],
brickid=objs['BRICKID'],
release=objs['RELEASE'])
# ADM first set the shape parameters assuming everything is an exponential
# ADM this will correctly assign e1, e2 of 0 to things with zero shape.
in_radius = objs['SHAPEEXP_R']
e1 = objs['SHAPEEXP_E1']
e2 = objs['SHAPEEXP_E2']
# ADM now to account for deVaucouleurs objects, or things that are dominated by
# ADM deVaucouleurs profiles, update objects with a larger "DEV" than "EXP" radius.
wdev = np.where(objs['SHAPEDEV_R'] > objs['SHAPEEXP_R'])
if len(wdev[0]) > 0:
in_radius[wdev] = objs[wdev]['SHAPEDEV_R']
e1[wdev] = objs[wdev]['SHAPEDEV_E1']
e2[wdev] = objs[wdev]['SHAPEDEV_E2']
# ADM finally use the Tycho radius (see the notes above) for PSF or star-like objects.
# ADM More consideration will be needed to derive correct numbers for this for DESI!!!
# ADM this calculation was for "near" Tycho objects and was in arcmin, so we convert
# ADM it to arcsec and multiply it by infac (see the top of the module).
tycho_in_radius = infac * (0.0802 * mags * mags - 1.860 * mags +
11.625) * 60.
wpsf = np.where(_psflike(objtype))
in_radius[wpsf] = tycho_in_radius[wpsf]
# ADM set "near" as a multiple of "in" radius using the factor at the top of the code.
near_radius = in_radius * nearfac
# ADM create an output recarray that is just RA, Dec, TARGETID and the radius.
done = objs[['RA', 'DEC']].copy()
done = rfn.append_fields(
done, ["TARGETID", "IN_RADIUS", "NEAR_RADIUS", "E1", "E2", "TYPE"],
[targetid, in_radius, near_radius, e1, e2, objtype],
usemask=False,
dtypes=['>i8', '>f4', '>f4', '>f4', '>f4', '|S4'])
if outfilename is not None:
fitsio.write(outfilename, done, clobber=True)
return done
