def isStdStar(fibermap, bright=None):
"""
Determines if target(s) are standard stars
Args:
fibermap: table including DESI_TARGET or SV1_DESI_TARGET bit mask(s)
Optional:
bright: if True, only bright time standards; if False, only darktime, otherwise both
Returns bool or array of bool
TODO: move out of scripts/stdstars.py
"""
target_colnames, target_masks, survey = main_cmx_or_sv(fibermap)
desi_target = fibermap[target_colnames[0]] # (SV1_)DESI_TARGET
desi_mask = target_masks[0] # (sv1) desi_mask
yes = (desi_target & desi_mask.STD_WD) != 0
if bright is None:
yes |= (desi_target & desi_mask.mask('STD_WD|STD_FAINT|STD_BRIGHT')) != 0
elif bright:
yes |= (desi_target & desi_mask.mask('STD_WD|STD_BRIGHT')) != 0
else:
yes |= (desi_target & desi_mask.mask('STD_WD|STD_FAINT')) != 0
return yes
def default_target_masks(data):
"""Return the column name and default mask values for the data table.
This identifies the type of target data and returns the defaults for
the program type.
Args:
data (Table): A Table or recarray.
Returns:
(tuple): The survey, column name, science mask, standard mask,
sky mask, suppsky mask, safe mask, and exclude mask for the data.
"""
col = None
filecols, filemasks, filesurvey = main_cmx_or_sv(data)
if filesurvey == "main":
col = "DESI_TARGET"
elif filesurvey == "cmx":
col = filecols[0]
elif filesurvey == "sv1":
col = "SV1_DESI_TARGET"
sciencemask, stdmask, skymask, suppskymask, safemask, excludemask = \
default_survey_target_masks(filesurvey)
return (filesurvey, col, sciencemask, stdmask, skymask, suppskymask,
safemask, excludemask)
def fibermeta2fibermap(fiberassign, meta):
'''
Convert a fiberassign + targeting metadata table into a fibermap Table
A future refactor will standardize the column names of fiber assignment,
target catalogs, and fibermaps, but in the meantime this is needed.
'''
#- Handle DESI_TARGET vs. SV1_DESI_TARGET etc.
target_colnames, target_masks, survey = main_cmx_or_sv(fiberassign)
targetcol = target_colnames[0] #- DESI_TARGET or SV1_DESI_TARGET
desi_mask = target_masks[0] #- desi_mask or sv1_desi_mask
#- Copy column names in common
fibermap = desispec.io.empty_fibermap(len(fiberassign))
for c in ['FIBER', 'TARGETID', 'BRICKNAME']:
fibermap[c] = fiberassign[c]
for c in target_colnames:
fibermap[c] = fiberassign[c]
for band in ['G', 'R', 'Z', 'W1', 'W2']:
key = 'FLUX_' + band
fibermap[key] = meta[key]
#- TODO: FLUX_IVAR_*
#- set OBJTYPE
#- TODO: what about MWS science targets that are also standard stars?
#- Loop over STD options for backwards/forwards compatibility
stdmask = 0
for name in [
'STD', 'STD_FSTAR', 'STD_WD'
'STD_FAINT', 'STD_FAINT_BEST', 'STD_BRIGHT', 'STD_BRIGHT_BEST'
]:
if name in desi_mask.names():
stdmask |= desi_mask[name]
isSTD = (fiberassign[targetcol] & stdmask) != 0
isSKY = (fiberassign[targetcol] & desi_mask.SKY) != 0
isSCI = (~isSTD & ~isSKY)
fibermap['OBJTYPE'][isSKY] = 'SKY'
fibermap['OBJTYPE'][isSCI | isSTD] = 'TGT'
fibermap['LAMBDAREF'] = 5400.0
fibermap['TARGET_RA'] = fiberassign['TARGET_RA']
fibermap['TARGET_DEC'] = fiberassign['TARGET_DEC']
fibermap['FIBER_RA'] = fiberassign['TARGET_RA']
fibermap['FIBER_DEC'] = fiberassign['TARGET_DEC']
fibermap['FIBERASSIGN_X'] = fiberassign['FIBERASSIGN_X']
fibermap['FIBERASSIGN_Y'] = fiberassign['FIBERASSIGN_Y']
fibermap['DELTA_X'] = 0.0
fibermap['DELTA_Y'] = 0.0
#- TODO: POSITIONER -> LOCATION
#- TODO: TARGETCAT (how should we propagate this info into here?)
#- TODO: NaNs in fibermap for unassigned positioners targets
return fibermap
def make_ledger_in_hp(targets, outdirname, nside, pixlist,
obscon="DARK", indirname=None, verbose=True):
"""
Make an initial MTL ledger file for targets in a set of HEALPixels.
Parameters
----------
targets : :class:`~numpy.array`
Targets made by, e.g. `desitarget.cuts.select_targets()`.
outdirname : :class:`str`
Output directory to which to write the MTLs (the file names are
constructed on the fly).
nside : :class:`int`
(NESTED) HEALPixel nside that corresponds to `pixnum`.
pixlist : :class:`list` or `int`
HEALPixels at `nside` at which to write the MTLs.
obscon : :class:`str`, optional, defaults to "DARK"
A string matching ONE obscondition in the desitarget bitmask yaml
file (i.e. in `desitarget.targetmask.obsconditions`), e.g. "GRAY"
Governs how priorities are set when merging targets. Also governs
the sub-directory to which the ledger is written.
indirname : :class:`str`
A directory associated with the targets. Written to the headers
of the output MTL files.
verbose : :class:`bool`, optional, defaults to ``True``
If ``True`` then log target and file information.
Returns
-------
Nothing, but writes the `targets` out to `outdirname` split across
each HEALPixel in `pixlist`.
"""
t0 = time()
# ADM in case an integer was passed.
pixlist = np.atleast_1d(pixlist)
# ADM execute MTL.
mtl = make_mtl(targets, obscon, trimcols=True)
# ADM the HEALPixel within which each target in the MTL lies.
theta, phi = np.radians(90-mtl["DEC"]), np.radians(mtl["RA"])
mtlpix = hp.ang2pix(nside, theta, phi, nest=True)
# ADM write the MTLs.
_, _, survey = main_cmx_or_sv(mtl)
for pix in pixlist:
inpix = mtlpix == pix
ecsv = get_mtl_ledger_format() == "ecsv"
nt, fn = io.write_mtl(
outdirname, mtl[inpix].as_array(), indir=indirname, ecsv=ecsv,
survey=survey, obscon=obscon, nsidefile=nside, hpxlist=pix)
if verbose:
log.info('{} targets written to {}...t={:.1f}s'.format(
nt, fn, time()-t0))
return
def test_cmx_priorities(self):
"""Test that priority calculation can handle commissioning files.
"""
t = self.targets.copy()
z = self.zcat
# ADM restructure the table to look like a commissioning table.
t.rename_column('DESI_TARGET', 'CMX_TARGET')
t.remove_column('BGS_TARGET')
t.remove_column('MWS_TARGET')
# - No targeting bits set is priority=0
self.assertTrue(np.all(calc_priority(t, z, "GRAY|DARK") == 0))
# ADM retrieve the cmx_mask.
colnames, masks, _ = main_cmx_or_sv(t)
cmx_mask = masks[0]
# ADM test handling of unobserved SV0_BGS and SV0_MWS
for name, obscon in [("SV0_BGS", "BRIGHT"), ("SV0_MWS", "POOR")]:
t['CMX_TARGET'] = cmx_mask[name]
self.assertTrue(
np.all(
calc_priority(t, z, obscon) ==
cmx_mask[name].priorities['UNOBS']))
# ADM done is Done, regardless of ZWARN.
for name, obscon in [("SV0_BGS", "BRIGHT"), ("SV0_MWS", "POOR")]:
t['CMX_TARGET'] = cmx_mask[name]
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
# APC: Use NUMOBS_INIT here to avoid hardcoding NOBS corresponding to "done".
numobs_done = t['NUMOBS_INIT'][0]
z['NUMOBS'] = [0, numobs_done, numobs_done]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, obscon, zcat=z)["PRIORITY"]
self.assertEqual(p[0], cmx_mask[name].priorities['UNOBS'])
self.assertEqual(p[1], cmx_mask[name].priorities['DONE'])
self.assertEqual(p[2], cmx_mask[name].priorities['DONE'])
# BGS ZGOOD targets always have lower priority than MWS targets that
# are not DONE.
lowest_bgs_priority_zgood = cmx_mask['SV0_BGS'].priorities[
'MORE_ZGOOD']
lowest_mws_priority_unobs = cmx_mask['SV0_MWS'].priorities['UNOBS']
lowest_mws_priority_zwarn = cmx_mask['SV0_MWS'].priorities[
'MORE_ZWARN']
lowest_mws_priority_zgood = cmx_mask['SV0_MWS'].priorities[
'MORE_ZGOOD']
lowest_mws_priority = min(lowest_mws_priority_unobs,
lowest_mws_priority_zwarn,
lowest_mws_priority_zgood)
self.assertLess(lowest_bgs_priority_zgood, lowest_mws_priority)
def test_isStdStar(self):
"""test isStdStar works for cmx, main, and sv1 fibermaps"""
from desispec.fluxcalibration import isStdStar
from desitarget.targetmask import desi_mask, mws_mask
from desitarget.sv1.sv1_targetmask import desi_mask as sv1_desi_mask
from desitarget.sv1.sv1_targetmask import mws_mask as sv1_mws_mask
from desitarget.cmx.cmx_targetmask import cmx_mask
from desitarget.targets import main_cmx_or_sv
from astropy.table import Table
#- CMX
fm = Table()
fm['CMX_TARGET'] = np.zeros(10, dtype=int)
fm['CMX_TARGET'][0:2] = cmx_mask.STD_FAINT
fm['CMX_TARGET'][2:4] = cmx_mask.SV0_STD_FAINT
self.assertEqual(main_cmx_or_sv(fm)[2], 'cmx')
self.assertEqual(np.count_nonzero(isStdStar(fm)), 4)
#- SV1
fm = Table()
fm['SV1_DESI_TARGET'] = np.zeros(10, dtype=int)
fm['SV1_MWS_TARGET'] = np.zeros(10, dtype=int)
fm['SV1_DESI_TARGET'][0:2] = sv1_desi_mask.STD_FAINT
fm['SV1_MWS_TARGET'][2:4] = sv1_mws_mask.GAIA_STD_FAINT
self.assertEqual(main_cmx_or_sv(fm)[2], 'sv1')
self.assertEqual(np.count_nonzero(isStdStar(fm)), 4)
#- Main
fm = Table()
fm['DESI_TARGET'] = np.zeros(10, dtype=int)
fm['MWS_TARGET'] = np.zeros(10, dtype=int)
fm['DESI_TARGET'][0:2] = desi_mask.STD_FAINT
fm['DESI_TARGET'][2:4] = desi_mask.STD_BRIGHT
fm['DESI_TARGET'][4:6] |= desi_mask.MWS_ANY
fm['MWS_TARGET'][4:6] = sv1_mws_mask.GAIA_STD_FAINT
self.assertEqual(main_cmx_or_sv(fm)[2], 'main')
self.assertEqual(np.count_nonzero(isStdStar(fm)), 6)
self.assertEqual(np.count_nonzero(isStdStar(fm, bright=False)), 4)
self.assertEqual(np.count_nonzero(isStdStar(fm, bright=True)), 2)
def select_stdstars(data, fibermap, snrcut=10, verbose=False):
"""Select spectra based on S/N and targeting bit.
"""
from astropy.table import Table
from desitarget.targets import main_cmx_or_sv
night, expid = data['NIGHT'][0], data['EXPID'][0]
out_fibermap = fibermap.copy()
# Pre-select standards
istd = np.where(isStdStar(out_fibermap))[0]
target_colnames, target_masks, survey = main_cmx_or_sv(out_fibermap)
target_col, target_mask = target_colnames[0], target_masks[
0] # CMX_TARGET, CMX_MASK for commissioning
#istd = np.where((out_fibermap[target_col] & target_mask.mask('STD_BRIGHT') != 0))[0]
# For each standard, apply a minimum S/N cut (in any camera) to
# subselect the fibers that were reasonably centered on the fiber.
reject = np.ones(len(istd), dtype=bool)
#if len(istd) == 0:
# pdb.set_trace()
if len(istd) > 0:
for ii, fiber in enumerate(out_fibermap['FIBER'][istd]):
wspec = np.where(
(data['EXPID'] == expid) * (data['FIBER'] == fiber))[0]
if len(wspec) > 0:
isnr = np.where(data['MEDIAN_CALIB_SNR'][wspec] > snrcut)[0]
if len(isnr) > 0:
reject[ii] = False
if np.sum(~reject) > 0:
istd_keep = istd[~reject]
else:
istd_keep = []
if np.sum(reject) > 0:
istd_reject = istd[reject]
else:
istd_reject = []
# Hack! Set the targeting bit for the failed standards to zero!
if len(istd_reject) > 0:
out_fibermap[target_col][istd_reject] = 0
if verbose:
print('EXPID={}, NSTD={}/{}'.format(expid, len(istd_keep), len(istd)))
return out_fibermap
def test_priorities(self):
"""Test that priorities are set correctly for both the main survey and SV.
"""
# ADM loop through once for SV and once for the main survey.
for prefix in ["", "SV1_"]:
t = self.targets.copy()
z = self.zcat.copy()
main_names = ['DESI_TARGET', 'BGS_TARGET', 'MWS_TARGET']
for name in main_names:
t.rename_column(name, prefix + name)
# ADM retrieve the mask and column names for this survey flavor.
colnames, masks, _ = main_cmx_or_sv(t)
desi_target, bgs_target, mws_target = colnames
desi_mask, bgs_mask, mws_mask = masks
# - No targeting bits set is priority=0
self.assertTrue(np.all(calc_priority(t, z, "BRIGHT") == 0))
# ADM test QSO > LRG > ELG for main survey and SV.
t[desi_target] = desi_mask.ELG
self.assertTrue(
np.all(
calc_priority(t, z, "GRAY|DARK") ==
desi_mask.ELG.priorities['UNOBS']))
t[desi_target] |= desi_mask.LRG
self.assertTrue(
np.all(
calc_priority(t, z, "GRAY|DARK") ==
desi_mask.LRG.priorities['UNOBS']))
t[desi_target] |= desi_mask.QSO
self.assertTrue(
np.all(
calc_priority(t, z, "GRAY|DARK") ==
desi_mask.QSO.priorities['UNOBS']))
# - different states -> different priorities
# - Done is Done, regardless of ZWARN.
t[desi_target] = desi_mask.ELG
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 1, 1]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, "GRAY|DARK", zcat=z)["PRIORITY"]
self.assertEqual(p[0], desi_mask.ELG.priorities['UNOBS'])
self.assertEqual(p[1], desi_mask.ELG.priorities['DONE'])
self.assertEqual(p[2], desi_mask.ELG.priorities['DONE'])
# ADM In BRIGHT conditions BGS FAINT targets are
# ADM never DONE, only MORE_ZGOOD.
t[desi_target] = desi_mask.BGS_ANY
t[bgs_target] = bgs_mask.BGS_FAINT
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 1, 1]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, "BRIGHT", zcat=z)["PRIORITY"]
self.assertEqual(p[0], bgs_mask.BGS_FAINT.priorities['UNOBS'])
self.assertEqual(p[1], bgs_mask.BGS_FAINT.priorities['MORE_ZWARN'])
self.assertEqual(p[2], bgs_mask.BGS_FAINT.priorities['MORE_ZGOOD'])
# BGS_FAINT: {UNOBS: 2000, MORE_ZWARN: 2000, MORE_ZGOOD: 1000, DONE: 2, OBS: 1, DONOTOBSERVE: 0}
# ADM but in DARK conditions, BGS_FAINT should behave as
# ADM for other target classes.
z = self.zcat.copy()
z['NUMOBS'] = [0, 1, 1]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, "DARK|GRAY", zcat=z)["PRIORITY"]
self.assertEqual(p[0], bgs_mask.BGS_FAINT.priorities['UNOBS'])
self.assertEqual(p[1], bgs_mask.BGS_FAINT.priorities['DONE'])
self.assertEqual(p[2], bgs_mask.BGS_FAINT.priorities['DONE'])
# ADM In BRIGHT conditions BGS BRIGHT targets are
# ADM never DONE, only MORE_ZGOOD.
t[desi_target] = desi_mask.BGS_ANY
t[bgs_target] = bgs_mask.BGS_BRIGHT
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 1, 1]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, "BRIGHT", zcat=z)["PRIORITY"]
self.assertEqual(p[0], bgs_mask.BGS_BRIGHT.priorities['UNOBS'])
self.assertEqual(p[1],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZWARN'])
self.assertEqual(p[2],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZGOOD'])
# BGS_BRIGHT: {UNOBS: 2100, MORE_ZWARN: 2100, MORE_ZGOOD: 1000, DONE: 2, OBS: 1, DONOTOBSERVE: 0}
# ADM In BRIGHT conditions BGS targets are
# ADM NEVER done even after 100 observations
t[desi_target] = desi_mask.BGS_ANY
t[bgs_target] = bgs_mask.BGS_BRIGHT
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 100, 100]
z['ZWARN'] = [1, 1, 0]
p = calc_priority(t, z, "BRIGHT")
self.assertEqual(p[0], bgs_mask.BGS_BRIGHT.priorities['UNOBS'])
self.assertEqual(p[1],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZWARN'])
self.assertEqual(p[2],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZGOOD'])
# BGS ZGOOD targets always have lower priority than MWS targets that
# are not DONE. Exempting the MWS "BACKUP" targets.
# ADM first discard N/S informational bits from bitmask as these
# ADM should never trump the other bits.
bgs_names = [
name for name in bgs_mask.names()
if 'NORTH' not in name and 'SOUTH' not in name
]
mws_names = [
name for name in mws_mask.names() if 'NORTH' not in name
and 'SOUTH' not in name and 'BACKUP' not in name
]
lowest_mws_priority_unobs = [
mws_mask[n].priorities['UNOBS'] for n in mws_names
]
lowest_bgs_priority_zgood = np.min(
[bgs_mask[n].priorities['MORE_ZGOOD'] for n in bgs_names])
# ADM MORE_ZGOOD and MORE_ZWARN are only meaningful if a
# ADM target class requests more than 1 observation (except
# ADM for BGS, which has a numobs=infinity exception)
lowest_mws_priority_zwarn = [
mws_mask[n].priorities['MORE_ZWARN'] for n in mws_names
if mws_mask[n].numobs > 1
]
lowest_mws_priority_zgood = [
mws_mask[n].priorities['MORE_ZGOOD'] for n in mws_names
if mws_mask[n].numobs > 1
]
lowest_mws_priority = np.min(
np.concatenate([
lowest_mws_priority_unobs, lowest_mws_priority_zwarn,
lowest_mws_priority_zgood
]))
self.assertLess(lowest_bgs_priority_zgood, lowest_mws_priority)
def select_fibers_to_fit(fibermap,
sns,
zbest_path=None,
minsn=None,
objtypes=None,
expid_range=None,
fit_targetid=None,
zbest_select=False):
""" Identify fibers to fit
Parameters
----------
fibermap: Table
Fibermap table object
sns: dict of numpy arrays
Array of S/Ns
minsn: float
Threshold S/N
objtypes: list of regular expressions
of DESI_TARGETs ['MWS_ANY,'STD_*'] or None
expid_range: list
The range of EXPID to consider
fit_targetid: list of ints
Fit only specific TARGETIDs
Returns
-------
ret: bool numpy array
Array with True for selected spectra
"""
zbest_maxvel = 1500 # maximum velocity to consider a star
zbest_type = 'STAR'
try:
import desitarget.targets as DT
except ImportError:
DT = None
subset = np.ones(len(fibermap), dtype=bool)
# Always apply EXPID range
if expid_range is not None:
mine, maxe = expid_range
if mine is None:
mine = -1
if maxe is None:
maxe = np.inf
if "EXPID" in fibermap.columns.names:
subset = subset & (fibermap["EXPID"] > mine) & (fibermap['EXPID'] <=
maxe)
# ONLY select good fiberstatus ones
if 'FIBERSTATUS' in fibermap.columns.names:
subset = subset & (fibermap['FIBERSTATUS'] == 0)
elif 'COADD_FIBERSTATUS' in fibermap.columns.names:
subset = subset & (fibermap['COADD_FIBERSTATUS'] == 0)
# Exclude skys but include anything else
subset = subset & (fibermap['OBJTYPE'] != 'SKY')
# Always apply TARGETID selection if provided
if fit_targetid is not None:
subset = subset & np.in1d(fibermap['TARGETID'], fit_targetid)
# Always apply SN cut if provided
if minsn is not None:
maxsn = np.max(np.array(list(sns.values())), axis=0)
with warnings.catch_warnings():
warnings.simplefilter('ignore')
subset = subset & (maxsn > minsn)
# compute the subset based on TARGET types
fibermapT = atpy.Table(fibermap)
types_subset = np.ones(len(fibermap), dtype=bool)
selecting_by_type = False
if DT is not None and objtypes is not None:
selecting_by_type = True
re_types = [re.compile(_) for _ in objtypes]
for i, currow in enumerate(fibermapT):
col_list, mask_list, _ = DT.main_cmx_or_sv(currow, scnd=True)
# collist will be column list like
# DESI_TARGET, BGS_TARGET, MWS_TARGET
# extract the DESI_TARGET part
colname = col_list[0]
mask = mask_list[0]
# all the possible types here
objtypnames = list(mask.names())
objs = []
# check which names match our regular expression
for curo in objtypnames:
for r in re_types:
if r.match(curo) is not None:
objs.append(curo)
# obtain integer values for each object type that matched
# our RE and bitwise OR them
bitmask = functools.reduce(operator.or_,
[mask.mask(_) for _ in objs])
# check if the given row has any hits in the bitmask
types_subset[i] = (currow[colname] & bitmask) > 0
# select objects based on redrock velocity or type
if zbest_select:
if zbest_path is None:
warnings.warn(
'zbest selection requested, but the zbest file not found')
if selecting_by_type:
zbest_subset = np.zeros(len(fibermap), dtype=bool)
else:
# I fit everything
zbest_subset = np.ones(len(fibermap), dtype=bool)
else:
zb = atpy.Table().read(zbest_path, format='fits', hdu='REDSHIFTS')
assert (len(zb) == len(subset))
zbest_subset = (((zb['SPECTYPE'] == zbest_type) |
(np.abs(zb['Z']) < zbest_maxvel / 3e5)))
else:
zbest_subset = np.zeros(len(fibermap), dtype=bool)
# We select either based on type or zbest
subset = subset & (zbest_subset | types_subset)
return subset
def main(args):
log = get_logger()
cmd = [
'desi_compute_fluxcalibration',
]
for key, value in args.__dict__.items():
if value is not None:
cmd += ['--' + key, str(value)]
cmd = ' '.join(cmd)
log.info(cmd)
log.info("read frame")
# read frame
frame = read_frame(args.infile)
# Set fibermask flagged spectra to have 0 flux and variance
frame = get_fiberbitmasked_frame(frame,
bitmask='flux',
ivar_framemask=True)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel = read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models(
args.models)
ok = np.ones(len(model_metadata), dtype=bool)
if args.chi2cut > 0:
log.info("apply cut CHI2DOF<{}".format(args.chi2cut))
good = (model_metadata["CHI2DOF"] < args.chi2cut)
bad = ~good
ok &= good
if np.any(bad):
log.info(" discard {} stars with CHI2DOF= {}".format(
np.sum(bad), list(model_metadata["CHI2DOF"][bad])))
legacy_filters = ('G-R', 'R-Z')
gaia_filters = ('GAIA-BP-RP', 'GAIA-G-RP')
model_column_list = model_metadata.columns.names
if args.color is None:
if 'MODEL_G-R' in model_column_list:
color = 'G-R'
elif 'MODEL_GAIA-BP-RP' in model_column_list:
log.info('Using Gaia filters')
color = 'GAIA-BP-RP'
else:
log.error(
"Can't find either G-R or BP-RP color in the model file.")
sys.exit(15)
else:
if args.color not in legacy_filters and args.color not in gaia_filters:
log.error(
'Color name {} is not allowed, must be one of {} {}'.format(
args.color, legacy_filters, gaia_filters))
sys.exit(14)
color = args.color
if color not in model_column_list:
# This should't happen
log.error(
'The color {} was not computed in the models'.format(color))
sys.exit(16)
if args.delta_color_cut > 0:
log.info("apply cut |delta color|<{}".format(args.delta_color_cut))
good = (np.abs(model_metadata["MODEL_" + color] -
model_metadata["DATA_" + color]) < args.delta_color_cut)
bad = ok & (~good)
ok &= good
if np.any(bad):
vals = model_metadata["MODEL_" +
color][bad] - model_metadata["DATA_" +
color][bad]
log.info(" discard {} stars with dcolor= {}".format(
np.sum(bad), list(vals)))
if args.min_color is not None:
log.info("apply cut DATA_{}>{}".format(color, args.min_color))
good = (model_metadata["DATA_{}".format(color)] > args.min_color)
bad = ok & (~good)
ok &= good
if np.any(bad):
vals = model_metadata["DATA_{}".format(color)][bad]
log.info(" discard {} stars with {}= {}".format(
np.sum(bad), color, list(vals)))
if args.chi2cut_nsig > 0:
# automatically reject stars that ar chi2 outliers
mchi2 = np.median(model_metadata["CHI2DOF"])
rmschi2 = np.std(model_metadata["CHI2DOF"])
maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2
log.info("apply cut CHI2DOF<{} based on chi2cut_nsig={}".format(
maxchi2, args.chi2cut_nsig))
good = (model_metadata["CHI2DOF"] <= maxchi2)
bad = ok & (~good)
ok &= good
if np.any(bad):
log.info(" discard {} stars with CHI2DOF={}".format(
np.sum(bad), list(model_metadata["CHI2DOF"][bad])))
ok = np.where(ok)[0]
if ok.size == 0:
log.error("selection cuts discarded all stars")
sys.exit(12)
nstars = model_flux.shape[0]
nbad = nstars - ok.size
if nbad > 0:
log.warning("discarding %d star(s) out of %d because of cuts" %
(nbad, nstars))
model_flux = model_flux[ok]
model_fibers = model_fibers[ok]
model_metadata = model_metadata[:][ok]
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
## check whether star fibers from args.models are consistent with fibers from fibermap
## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
fibermap_std_indices = np.where(isStdStar(fibermap))[0]
if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)):
target_colnames, target_masks, survey = main_cmx_or_sv(fibermap)
colname = target_colnames[0]
for i in model_fibers % 500:
log.error(
"inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap"
.format(i, fibermap["OBJTYPE"][i], colname,
fibermap[colname][i]))
sys.exit(12)
# Make sure the fibers of interest aren't entirely masked.
if np.sum(
np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) ==
frame.nwave) == len(model_fibers):
log.warning('All standard-star spectra are masked!')
return
fluxcalib = compute_flux_calibration(
frame,
model_wave,
model_flux,
model_fibers % 500,
highest_throughput_nstars=args.highest_throughput,
exposure_seeing_fwhm=args.seeing_fwhm)
# QA
if (args.qafile is not None):
from desispec.io import write_qa_frame
from desispec.io.qa import load_qa_frame
from desispec.qa import qa_plots
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame_meta=frame.meta,
flavor=frame.meta['FLAVOR'])
# Run
#import pdb; pdb.set_trace()
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)
# record inputs
frame.meta['IN_FRAME'] = shorten_filename(args.infile)
frame.meta['IN_SKY'] = shorten_filename(args.sky)
frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat)
frame.meta['STDMODEL'] = shorten_filename(args.models)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s" % args.outfile)
def make_mtl(targets, obscon, zcat=None, trim=False, scnd=None):
"""Adds NUMOBS, PRIORITY, and OBSCONDITIONS columns to a targets table.
Parameters
----------
targets : :class:`~numpy.array` or `~astropy.table.Table`
A numpy rec array or astropy Table with at least the columns
``TARGETID``, ``DESI_TARGET``, ``NUMOBS_INIT``, ``PRIORITY_INIT``.
or the corresponding columns for SV or commissioning.
obscon : :class:`str`
A combination of strings that are in the desitarget bitmask yaml
file (specifically in `desitarget.targetmask.obsconditions`), e.g.
"DARK|GRAY". Governs the behavior of how priorities are set based
on "obsconditions" in the desitarget bitmask yaml file.
zcat : :class:`~astropy.table.Table`, optional
Redshift catalog table with columns ``TARGETID``, ``NUMOBS``, ``Z``,
``ZWARN``.
trim : :class:`bool`, optional
If ``True`` (default), don't include targets that don't need
any more observations. If ``False``, include every input target.
scnd : :class:`~numpy.array`, `~astropy.table.Table`, optional
A set of secondary targets associated with the `targets`. As with
the `target` must include at least ``TARGETID``, ``NUMOBS_INIT``,
``PRIORITY_INIT`` or the corresponding SV columns.
The secondary targets will be padded to have the same columns
as the targets, and concatenated with them.
Returns
-------
:class:`~astropy.table.Table`
MTL Table with targets columns plus:
* NUMOBS_MORE - number of additional observations requested
* PRIORITY - target priority (larger number = higher priority)
* OBSCONDITIONS - replaces old GRAYLAYER
"""
start = time()
# ADM set up the default logger.
from desiutil.log import get_logger
log = get_logger()
# ADM if secondaries were passed, concatenate them with the targets.
if scnd is not None:
nrows = len(scnd)
log.info(
'Pad {} primary targets with {} secondaries...t={:.1f}s'.format(
len(targets), nrows,
time() - start))
padit = np.zeros(nrows, dtype=targets.dtype)
sharedcols = set(targets.dtype.names).intersection(
set(scnd.dtype.names))
for col in sharedcols:
padit[col] = scnd[col]
targets = np.concatenate([targets, padit])
# APC Propagate a flag on which targets came from scnd
is_scnd = np.repeat(False, len(targets))
is_scnd[-nrows:] = True
log.info('Done with padding...t={:.1f}s'.format(time() - start))
# ADM determine whether the input targets are main survey, cmx or SV.
colnames, masks, survey = main_cmx_or_sv(targets)
# ADM set the first column to be the "desitarget" column
desi_target, desi_mask = colnames[0], masks[0]
# Trim targets from zcat that aren't in original targets table
if zcat is not None:
ok = np.in1d(zcat['TARGETID'], targets['TARGETID'])
num_extra = np.count_nonzero(~ok)
if num_extra > 0:
log.warning("Ignoring {} zcat entries that aren't "
"in the input target list".format(num_extra))
zcat = zcat[ok]
n = len(targets)
# ADM if the input target columns were incorrectly called NUMOBS or PRIORITY
# ADM rename them to NUMOBS_INIT or PRIORITY_INIT.
# ADM Note that the syntax is slightly different for a Table.
for name in ['NUMOBS', 'PRIORITY']:
if isinstance(targets, Table):
try:
targets.rename_column(name, name + '_INIT')
except KeyError:
pass
else:
targets.dtype.names = [
name + '_INIT' if col == name else col
for col in targets.dtype.names
]
# ADM if a redshift catalog was passed, order it to match the input targets
# ADM catalog on 'TARGETID'.
if zcat is not None:
# ADM there might be a quicker way to do this?
# ADM set up a dictionary of the indexes of each target id.
d = dict(tuple(zip(targets["TARGETID"], np.arange(n))))
# ADM loop through the zcat and look-up the index in the dictionary.
zmatcher = np.array([d[tid] for tid in zcat["TARGETID"]])
ztargets = zcat
if ztargets.masked:
unobs = ztargets['NUMOBS'].mask
ztargets['NUMOBS'][unobs] = 0
unobsz = ztargets['Z'].mask
ztargets['Z'][unobsz] = -1
unobszw = ztargets['ZWARN'].mask
ztargets['ZWARN'][unobszw] = -1
else:
ztargets = Table()
ztargets['TARGETID'] = targets['TARGETID']
ztargets['NUMOBS'] = np.zeros(n, dtype=np.int32)
ztargets['Z'] = -1 * np.ones(n, dtype=np.float32)
ztargets['ZWARN'] = -1 * np.ones(n, dtype=np.int32)
# ADM if zcat wasn't passed, there is a one-to-one correspondence
# ADM between the targets and the zcat.
zmatcher = np.arange(n)
# ADM extract just the targets that match the input zcat.
targets_zmatcher = targets[zmatcher]
# ADM use passed value of NUMOBS_INIT instead of calling the memory-heavy calc_numobs.
# ztargets['NUMOBS_MORE'] = np.maximum(0, calc_numobs(ztargets) - ztargets['NUMOBS'])
ztargets['NUMOBS_MORE'] = np.maximum(
0, targets_zmatcher['NUMOBS_INIT'] - ztargets['NUMOBS'])
# ADM need a minor hack to ensure BGS targets are observed once
# ADM (and only once) every time during the BRIGHT survey, regardless
# ADM of how often they've previously been observed. I've turned this
# ADM off for commissioning. Not sure if we'll keep it in general.
if survey != 'cmx':
# ADM only if we're considering bright survey conditions.
if (obsconditions.mask(obscon) & obsconditions.mask("BRIGHT")) != 0:
ii = targets_zmatcher[desi_target] & desi_mask.BGS_ANY > 0
ztargets['NUMOBS_MORE'][ii] = 1
if survey == 'main':
# If the object is confirmed to be a tracer QSO, then don't request more observations
if (obsconditions.mask(obscon) & obsconditions.mask("DARK")) != 0:
if zcat is not None:
ii = ztargets['SPECTYPE'] == 'QSO'
ii &= (ztargets['ZWARN'] == 0)
ii &= (ztargets['Z'] < 2.1)
ii &= (ztargets['NUMOBS'] > 0)
ztargets['NUMOBS_MORE'][ii] = 0
# ADM assign priorities, note that only things in the zcat can have changed priorities.
# ADM anything else will be assigned PRIORITY_INIT, below.
priority = calc_priority(targets_zmatcher, ztargets, obscon)
# If priority went to 0==DONOTOBSERVE or 1==OBS or 2==DONE, then NUMOBS_MORE should also be 0.
# ## mtl['NUMOBS_MORE'] = ztargets['NUMOBS_MORE']
ii = (priority <= 2)
log.info(
'{:d} of {:d} targets have priority zero, setting N_obs=0.'.format(
np.sum(ii), n))
ztargets['NUMOBS_MORE'][ii] = 0
# - Set the OBSCONDITIONS mask for each target bit.
obsconmask = set_obsconditions(targets)
# APC obsconmask will now be incorrect for secondary-only targets. Fix this
# APC using the mask on secondary targets.
if scnd is not None:
obsconmask[is_scnd] = set_obsconditions(targets[is_scnd], scnd=True)
# ADM set up the output mtl table.
mtl = Table(targets)
mtl.meta['EXTNAME'] = 'MTL'
# ADM any target that wasn't matched to the ZCAT should retain its
# ADM original (INIT) value of PRIORITY and NUMOBS.
mtl['NUMOBS_MORE'] = mtl['NUMOBS_INIT']
mtl['PRIORITY'] = mtl['PRIORITY_INIT']
# ADM now populate the new mtl columns with the updated information.
mtl['OBSCONDITIONS'] = obsconmask
mtl['PRIORITY'][zmatcher] = priority
mtl['NUMOBS_MORE'][zmatcher] = ztargets['NUMOBS_MORE']
# Filter out any targets marked as done.
if trim:
notdone = mtl['NUMOBS_MORE'] > 0
log.info('{:d} of {:d} targets are done, trimming these'.format(
len(mtl) - np.sum(notdone), len(mtl)))
mtl = mtl[notdone]
# Filtering can reset the fill_value, which is just wrong wrong wrong
# See https://github.com/astropy/astropy/issues/4707
# and https://github.com/astropy/astropy/issues/4708
mtl['NUMOBS_MORE'].fill_value = -1
log.info('Done...t={:.1f}s'.format(time() - start))
return mtl
def parse_assign(optlist=None):
"""Parse assignment options.
This parses either sys.argv or a list of strings passed in. If passing
an option list, you can create that more easily using the
:func:`option_list` function.
Args:
optlist (list, optional): Optional list of arguments to parse instead
of using sys.argv.
Returns:
(namespace): an ArgumentParser namespace.
"""
log = Logger.get()
parser = argparse.ArgumentParser()
parser.add_argument("--targets", type=str, required=True, nargs="+",
help="Input file with targets of any type. This "
"argument can be specified multiple times (for "
"example if standards / skies / science targets are "
"in different files). By default, the "
"'--mask_column' (default DESI_TARGET)"
"column and bitfield values defined in desitarget "
"are used to determine the type of each target. "
"Each filename may be optionally followed by comma "
"and then one of the strings 'science', 'standard', "
"'sky' or 'safe' to force all targets in that file "
"to be treated as a fixed target type.")
parser.add_argument("--sky", type=str, required=False, nargs="+",
help="Input file with sky or 'bad sky' targets. "
"This option exists in order to treat main-survey"
" sky target files as valid for other survey types."
" If you are running a main survey assignment, you"
" can just pass the sky file to the --targets list.")
parser.add_argument("--gfafile", type=str, required=False, default=None,
help="Optional GFA targets FITS file")
parser.add_argument("--footprint", type=str, required=False, default=None,
help="Optional FITS file defining the footprint. If"
" not specified, the default footprint from desimodel"
" is used.")
parser.add_argument("--tiles", type=str, required=False, default=None,
help="Optional text file containing a subset of the"
" tile IDs to use in the footprint, one ID per line."
" Default uses all tiles in the footprint.")
parser.add_argument("--rundate", type=str, required=False, default=None,
help="Optional date to simulate for this run of "
"fiber assignment, used to load the correct "
"focalplane properties and state from desimodel. "
"Default uses the current date. Format is "
"YYYY-MM-DDTHH:mm:ss+-zz:zz.")
parser.add_argument("--obsdate", type=str, required=False, default="2022-07-01",
help="Plan field rotations for this date (YEARMMDD, "
"or ISO 8601 YEAR-MM-DD with or without time).")
parser.add_argument("--ha", type=float, required=False, default=0.,
help="Design for the given Hour Angle in degrees.")
parser.add_argument("--fieldrot", type=float, required=False, default=None,
help="Override obsdate and use this field rotation "
"for all tiles (degrees counter clockwise in CS5)")
parser.add_argument("--dir", type=str, required=False, default=None,
help="Output directory.")
parser.add_argument("--prefix", type=str, required=False,
default="fba-",
help="Prefix of each file (before the <tile>.fits).")
parser.add_argument("--split", required=False, default=False,
action="store_true",
help="Split output into tile prefix directories.")
parser.add_argument("--standards_per_petal", type=int, required=False,
default=10, help="Required number of standards per"
" petal")
parser.add_argument("--sky_per_petal", type=int, required=False,
default=40, help="Required number of sky targets per"
" petal")
parser.add_argument("--sky_per_slitblock", type=int, required=False,
default=1, help="Required number of sky targets per"
" fiber slitblock")
parser.add_argument("--margin-pos", type=float, required=False, default=0.,
help="Add margin (in mm) around positioner keep-out polygons")
parser.add_argument("--margin-petal", type=float, required=False, default=0.,
help="Add margin (in mm) around petal-boundary keep-out polygons")
parser.add_argument("--margin-gfa", type=float, required=False, default=0.,
help="Add margin (in mm) around GFA keep-out polygons")
parser.add_argument("--write_all_targets", required=False, default=False,
action="store_true",
help="When writing target properties, write data "
"for all available targets, not just those which are "
"assigned. This is convenient, but increases the "
"write time and the file size.")
parser.add_argument("--overwrite", required=False, default=False,
action="store_true",
help="Overwrite any pre-existing output files")
parser.add_argument("--mask_column", required=False, default=None,
help="FITS column to use for applying target "
"masks")
parser.add_argument("--sciencemask", required=False,
default=None,
help="Default DESI_TARGET mask to use for science "
"targets")
parser.add_argument("--stdmask", required=False,
default=None,
help="Default DESI_TARGET mask to use for stdstar "
"targets")
parser.add_argument("--skymask", required=False,
default=None,
help="Default DESI_TARGET mask to use for sky targets")
parser.add_argument("--safemask", required=False,
default=None,
help="Default DESI_TARGET mask to use for safe "
"backup targets")
parser.add_argument("--excludemask", required=False,
default=None,
help="Default DESI_TARGET mask to exclude from "
"any assignments")
parser.add_argument("--by_tile", required=False, default=False,
action="store_true",
help="Do assignment one tile at a time. This disables"
" redistribution.")
parser.add_argument("--no_redistribute", required=False, default=False,
action="store_true",
help="Disable redistribution of science targets.")
parser.add_argument("--no_zero_obsremain", required=False, default=False,
action="store_true",
help="Disable oversubscription of science targets with leftover fibers.")
args = None
if optlist is None:
args = parser.parse_args()
else:
args = parser.parse_args(optlist)
if args.sky is None:
args.sky = list()
# If any of the masks are strings, determine the survey type from
# the first target file to know which bitmask to use
if isinstance(args.sciencemask, str) or \
isinstance(args.stdmask, str) or \
isinstance(args.skymask, str) or \
isinstance(args.safemask, str) or \
isinstance(args.excludemask, str):
import fitsio
from desitarget.targets import main_cmx_or_sv
data = fitsio.read(args.targets[0], 1, rows=[0,1])
filecols, filemasks, filesurvey = main_cmx_or_sv(data)
desi_mask = filemasks[0]
# convert str bit names -> int bit mask
if isinstance(args.sciencemask, str):
try:
args.sciencemask = int(args.sciencemask)
except ValueError:
args.sciencemask = desi_mask.mask(args.sciencemask.replace(",","|"))
if isinstance(args.stdmask, str):
try:
args.stdmask = int(args.stdmask)
except ValueError:
args.stdmask = desi_mask.mask(args.stdmask.replace(",", "|"))
if isinstance(args.skymask, str):
try:
args.skymask = int(args.skymask)
except ValueError:
args.skymask = desi_mask.mask(args.skymask.replace(",", "|"))
if isinstance(args.safemask, str):
try:
args.safemask = int(args.safemask)
except ValueError:
args.safemask = desi_mask.mask(args.safemask.replace(",", "|"))
if isinstance(args.excludemask, str):
try:
args.excludemask = int(args.excludemask)
except ValueError:
args.excludemask = desi_mask.mask(args.excludemask.replace(",","|"))
# convert YEARMMDD to YEAR-MM-DD to be ISO 8601 compatible
if re.match('\d{8}', args.obsdate):
year = args.obsdate[0:4]
mm = args.obsdate[4:6]
dd = args.obsdate[6:8]
#- Note: ISO8601 does not require time portion
args.obsdate = '{}-{}-{}'.format(year, mm, dd)
# Set output directory
if args.dir is None:
if args.rundate is None:
raise RuntimeError(
"You must specify the output directory or the rundate")
args.dir = "out_fiberassign_{}".format(args.rundate)
# Set up margins dict
args.margins = {}
if args.margin_pos != 0:
args.margins['theta'] = args.margin_pos
args.margins['phi'] = args.margin_pos
if args.margin_petal != 0:
## The petal polygon is listed in *clockwise* order, so we have to give it a negative margin!
args.margins['petal'] = -args.margin_petal
if args.margin_gfa != 0:
args.margins['gfa'] = args.margin_gfa
return args
def get_mock_spectra(fiberassign, mockdir=None, nside=64, obscon=None):
'''
Args:
fiberassign: table loaded from fiberassign tile file
Options:
mockdir (str): base directory under which files are found
nside (int): healpix nside for file directory grouping
obscon (str): (observing conditions) None/dark/bright extra dir level
Returns (flux, wave, meta) tuple
'''
nspec = len(fiberassign)
flux = None
meta = None
wave = None
objmeta = None
target_colnames, target_masks, survey = main_cmx_or_sv(fiberassign)
targetcol = target_colnames[0] #- DESI_TARGET or SV1_DESI_TARGET
desi_mask = target_masks[0] #- desi_mask or sv1_desi_mask
issky = (fiberassign[targetcol] & desi_mask.SKY) != 0
skyids = fiberassign['TARGETID'][issky]
#- check several ways in which an unassigned fiber might appear
unassigned = np.isnan(fiberassign['TARGET_RA'])
unassigned |= np.isnan(fiberassign['TARGET_DEC'])
unassigned |= (fiberassign['TARGETID'] < 0)
## TODO: check desi_mask.NO_TARGET once that bit exists
for truthfile, targetids in zip(*targets2truthfiles(
fiberassign[~unassigned], basedir=mockdir, nside=nside,
obscon=obscon)):
#- Sky fibers aren't in the truth files
ok = ~np.in1d(targetids, skyids)
tmpflux, tmpwave, tmpmeta, tmpobjmeta = read_mock_spectra(
truthfile, targetids[ok])
if flux is None:
nwave = tmpflux.shape[1]
flux = np.zeros((nspec, nwave))
meta = np.zeros(nspec, dtype=tmpmeta.dtype)
meta['TARGETID'] = -1
wave = tmpwave.astype('f8')
objmeta = dict()
for key in tmpobjmeta.keys():
objmeta[key] = list()
ii = np.in1d(fiberassign['TARGETID'], tmpmeta['TARGETID'])
flux[ii] = tmpflux
meta[ii] = tmpmeta
assert np.all(wave == tmpwave)
for key in tmpobjmeta.keys():
if key not in objmeta:
objmeta[key] = list()
objmeta[key].append(tmpobjmeta[key])
#- Stack the per-objtype meta tables
for key in objmeta.keys():
objmeta[key] = astropy.table.Table(np.hstack(objmeta[key]))
#- Set meta['TARGETID'] for sky fibers
#- TODO: other things to set?
meta['TARGETID'][issky] = skyids
meta['TARGETID'][unassigned] = fiberassign['TARGETID'][unassigned]
assert np.all(fiberassign['TARGETID'] == meta['TARGETID'])
return flux, wave, astropy.table.Table(meta), objmeta
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
primtargs : :class:`~numpy.ndarray`
An array of primary targets.
scndout : :class`~numpy.ndarray`
Name of a sub-directory to which to write the information in
`desitarget.secondary.outdatamodel` with `TARGETID` and (the
highest) `PRIORITY_INIT` updated with matching primary info.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
pix : :class:`list`, optional, defaults to `None`
Limit secondary targets to (NESTED) HEALpixels that touch
pix at the supplied `nside`, as a speed-up.
nside : :class:`int`, optional, defaults to `None`
The (NESTED) HEALPixel nside to be used with `pixlist`.
Returns
-------
:class:`~numpy.ndarray`
The array of primary targets, with the `SCND_TARGET` bit
populated for matches to secondary targets
"""
# ADM add a SCND_TARGET column to the primary targets.
dt = primtargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(primtargs), dtype=dt)
for col in primtargs.dtype.names:
targs[col] = primtargs[col]
# ADM check if this is an SV or main survey file.
cols, mx, surv = main_cmx_or_sv(targs, scnd=True)
log.info('running on the {} survey...'.format(surv))
if surv != 'main':
scxdir = os.path.join(scxdir, surv)
# ADM read in non-OVERRIDE secondary targets.
scxtargs = read_files(scxdir, mx[3])
scxtargs = scxtargs[~scxtargs["OVERRIDE"]]
# ADM match primary targets to non-OVERRIDE secondary targets.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if nside is not None and pix is not None:
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(primtargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(primtargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(scndout, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# ADM update the secondary targets with the primary information.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM the maximum priority will be used to break ties in the
# ADM unlikely event that a secondary matches two primaries.
hipri = np.maximum(targs["PRIORITY_INIT_DARK"],
targs["PRIORITY_INIT_BRIGHT"])
scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs]
# ADM write the secondary targets that have updated TARGETIDs.
ii = scxtargs["TARGETID"] != -1
nmatches = np.sum(ii)
log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format(
nmatches, scndout,
time() - start))
if nmatches > 0:
hdr = fitsio.FITSHDR()
hdr["SURVEY"] = surv
fitsio.write(scndout,
scxtargs[ii],
extname='SCND_TARG',
header=hdr,
clobber=True)
log.info('Done...t={:.1f}s'.format(time() - start))
return targs
def finalize_secondary(scxtargs,
scnd_mask,
survey='main',
sep=1.,
darkbright=False):
"""Assign secondary targets a realistic TARGETID, finalize columns.
Parameters
----------
scxtargs : :class:`~numpy.ndarray`
An array of secondary targets, must contain the columns `RA`,
`DEC` and `TARGETID`. `TARGETID` should be -1 for objects
that lack a `TARGETID`.
scnd_mask : :class:`desiutil.bitmask.BitMask`
A mask corresponding to a set of secondary targets, e.g, could
be ``from desitarget.targetmask import scnd_mask`` for the
main survey mask.
survey : :class:`str`, optional, defaults to "main"
string indicating whether we are working in the context of the
Main Survey (`main`) or SV (e.g. `sv1`, `sv2` etc.). Used to
set the `RELEASE` number in the `TARGETID` (see Notes).
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match secondary targets to
themselves in ARCSECONDS.
darkbright : :class:`bool`, optional, defaults to ``False``
If sent, then split `NUMOBS_INIT` and `PRIORITY_INIT` into
`NUMOBS_INIT_DARK`, `NUMOBS_INIT_BRIGHT`, `PRIORITY_INIT_DARK`
and `PRIORITY_INIT_BRIGHT` and calculate values appropriate
to "BRIGHT" and "DARK|GRAY" observing conditions.
Returns
-------
:class:`~numpy.ndarray`
The array of secondary targets, with the `TARGETID` bit updated
to be unique and reasonable and the `SCND_TARGET` column renamed
based on the flavor of `scnd_mask`.
Notes
-----
- Secondaries without `OVERRIDE` are also matched to themselves
Such matches are given the same `TARGETID` (that of the primary
if they match a primary) and the bitwise or of `SCND_TARGET` and
`OBSCONDITIONS` bits across matches. The highest `PRIORITY_INIT`
is retained, and others are set to -1. Only secondaries with
priorities that are not -1 are written to the main file. If
multiple matching secondary targets have the same (highest)
priority, the first one encountered retains its `PRIORITY_INIT`
- The secondary `TARGETID` is designed to be reproducible. It
combines `BRICKID` based on location, `OBJID` based on the
order of the targets in the secondary file (`SCND_ORDER`) and
`RELEASE` from the secondary bit number (`SCND_TARGET`) and the
input `survey`. `RELEASE` is set to ((X-1)*100)+np.log2(scnd_bit)
with X from the `survey` string survey=svX and scnd_bit from
`SCND_TARGET`. For the main survey (survey="main") X-1 is 5.
"""
# ADM assign new TARGETIDs to targets without a primary match.
nomatch = scxtargs["TARGETID"] == -1
# ADM get the BRICKIDs for each source.
brxid = bricks.brickid(scxtargs["RA"], scxtargs["DEC"])
# ADM ensure unique secondary bits for different iterations of SV
# ADM and the Main Survey.
if survey == 'main':
Xm1 = 5
elif survey[0:2] == 'sv':
# ADM the re.search just extracts the numbers in the string.
Xm1 = int(re.search(r'\d+', survey).group()) - 1
# ADM we've allowed a max of up to sv5 (!). Fail if surpassed.
if Xm1 >= 5:
msg = "Only coded for up to 'sv5', not {}!!!".format(survey)
log.critical(msg)
raise ValueError(msg)
else:
msg = "allowed surveys: 'main', 'svX', not {}!!!".format(survey)
log.critical(msg)
raise ValueError(msg)
# ADM the RELEASE for each source is the `SCND_TARGET` bit NUMBER.
release = (Xm1 * 100) + np.log2(scxtargs["SCND_TARGET_INIT"]).astype('int')
# ADM build the OBJIDs based on the values of SCND_ORDER.
t0 = time()
log.info("Begin assigning OBJIDs to bricks...")
# ADM So as not to overwhelm the bit-limits for OBJID
# ADM rank by SCND_ORDER for each brick and bit combination.
# ADM First, create a unique ID based on brxid and release.
scnd_order = scxtargs["SCND_ORDER"]
sorter = (1000 * brxid) + release
# ADM sort the unique IDs and split based on where they change.
argsort = np.argsort(sorter)
w = np.where(np.diff(sorter[argsort]))[0]
soperbrxbit = np.split(scnd_order[argsort], w + 1)
# ADM loop through each (brxid, release) and sort on scnd_order.
# ADM double argsort returns the ascending ranked order of the entry
# ADM (whereas a single argsort returns the indexes for ordering).
sortperbrxbit = [np.argsort(np.argsort(so)) for so in soperbrxbit]
# ADM finally unroll the (brxid, release) combinations...
sortedobjid = np.array(list(itertools.chain.from_iterable(sortperbrxbit)))
# ADM ...and reorder based on the initial argsort.
objid = np.zeros_like(sortedobjid) - 1
objid[argsort] = sortedobjid
log.info("Assigned OBJIDs to bricks in {:.1f}s".format(time() - t0))
# ADM check that the objid array was entirely populated.
assert np.all(objid != -1)
# ADM assemble the TARGETID, SCND objects have RELEASE==0.
targetid = encode_targetid(objid=objid, brickid=brxid, release=release)
# ADM a check that the generated TARGETIDs are unique.
if len(set(targetid)) != len(targetid):
msg = "duplicate TARGETIDs generated for secondary targets!!!"
log.critical(msg)
raise ValueError(msg)
# ADM assign the unique TARGETIDs to the secondary objects.
scxtargs["TARGETID"][nomatch] = targetid[nomatch]
log.debug("Assigned {} targetids to unmatched secondaries".format(
len(targetid[nomatch])))
# ADM match secondaries to themselves, to ensure duplicates
# ADM share a TARGETID. Don't match special (OVERRIDE) targets
# ADM or sources that have already been matched to a primary.
w = np.where(~scxtargs["OVERRIDE"] & nomatch)[0]
if len(w) > 0:
log.info("Matching secondary targets to themselves...t={:.1f}s".format(
time() - t0))
# ADM use astropy for the matching. At NERSC, astropy matches
# ADM ~20M objects to themselves in about 10 minutes.
c = SkyCoord(scxtargs["RA"][w] * u.deg, scxtargs["DEC"][w] * u.deg)
m1, m2, _, _ = c.search_around_sky(c, sep * u.arcsec)
log.info("Done with matching...t={:.1f}s".format(time() - t0))
# ADM restrict only to unique matches (and exclude self-matches).
uniq = m1 > m2
m1, m2 = m1[uniq], m2[uniq]
# ADM set same TARGETID for any matches. m2 must come first, here.
scxtargs["TARGETID"][w[m2]] = scxtargs["TARGETID"][w[m1]]
# ADM Ensure secondary targets with matching TARGETIDs have all the
# ADM relevant SCND_TARGET bits set. By definition, targets with
# ADM OVERRIDE set never have matching TARGETIDs.
wnoov = np.where(~scxtargs["OVERRIDE"])[0]
if len(wnoov) > 0:
for _, inds in duplicates(scxtargs["TARGETID"][wnoov]):
scnd_targ = 0
for ind in inds:
scnd_targ |= scxtargs["SCND_TARGET"][wnoov[ind]]
scxtargs["SCND_TARGET"][wnoov[inds]] = scnd_targ
log.info("Done checking SCND_TARGET...t={:.1f}s".format(time() - t0))
# ADM change the data model depending on whether the mask
# ADM is an SVX (X = 1, 2, etc.) mask or not. Nothing will
# ADM change if the mask has no preamble.
prepend = scnd_mask._name[:-9].upper()
scxtargs = rfn.rename_fields(scxtargs,
{'SCND_TARGET': prepend + 'SCND_TARGET'})
# APC same thing for DESI_TARGET
scxtargs = rfn.rename_fields(scxtargs,
{'DESI_TARGET': prepend + 'DESI_TARGET'})
# APC Remove duplicate targetids from secondary-only targets
alldups = []
for _, dups in duplicates(scxtargs['TARGETID']):
# Retain the duplicate with highest priority, breaking ties
# on lowest index in list of duplicates
dups = np.delete(dups, np.argmax(scxtargs['PRIORITY_INIT'][dups]))
alldups.append(dups)
alldups = np.hstack(alldups)
log.debug(
"Flagging {} duplicate secondary targetids with PRIORITY_INIT=-1".
format(len(alldups)))
# ADM and remove the INIT fields in prep for a dark/bright split.
scxtargs = rfn.drop_fields(scxtargs, ["PRIORITY_INIT", "NUMOBS_INIT"])
# ADM set initial priorities, numobs and obsconditions for both
# ADM BRIGHT and DARK|GRAY conditions, if requested.
nscx = len(scxtargs)
nodata = np.zeros(nscx, dtype='int') - 1
if darkbright:
ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"]
else:
ender, obscon = [""], ["DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18"]
cols, vals, forms = [], [], []
for edr, oc in zip(ender, obscon):
cols += ["{}_INIT{}".format(pn, edr) for pn in ["PRIORITY", "NUMOBS"]]
vals += [nodata, nodata]
forms += ['>i8', '>i8']
# ADM write the output array.
newdt = [dt for dt in zip(cols, forms)]
done = np.array(np.zeros(nscx), dtype=scxtargs.dtype.descr + newdt)
for col in scxtargs.dtype.names:
done[col] = scxtargs[col]
for col, val in zip(cols, vals):
done[col] = val
# ADM add the actual PRIORITY/NUMOBS values.
for edr, oc in zip(ender, obscon):
pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr
done[pc], done[nc] = initial_priority_numobs(done,
obscon=oc,
scnd=True)
# APC Flagged duplicates are removed in io.write_secondary
done[pc][alldups] = -1
# APC add secondary flag in DESI_TARGET
cols, mx, surv = main_cmx_or_sv(done, scnd=True)
done[cols[0]] = mx[0]['SCND_ANY']
# ADM set the OBSCONDITIONS.
done["OBSCONDITIONS"] = set_obsconditions(done, scnd=True)
return done
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
primtargs : :class:`~numpy.ndarray`
An array of primary targets.
scndout : :class`~numpy.ndarray`
Name of a sub-directory to which to write the information in
`desitarget.secondary.outdatamodel` with `TARGETID` and (the
highest) `PRIORITY_INIT` updated with matching primary info.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
pix : :class:`list`, optional, defaults to `None`
Limit secondary targets to (NESTED) HEALpixels that touch
pix at the supplied `nside`, as a speed-up.
nside : :class:`int`, optional, defaults to `None`
The (NESTED) HEALPixel nside to be used with `pixlist`.
Returns
-------
:class:`~numpy.ndarray`
The array of primary targets, with the `SCND_TARGET` bit
populated for matches to secondary targets
"""
# ADM add a SCND_TARGET column to the primary targets.
dt = primtargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(primtargs), dtype=dt)
for col in primtargs.dtype.names:
targs[col] = primtargs[col]
# ADM check if this is an SV or main survey file.
cols, mx, surv = main_cmx_or_sv(targs, scnd=True)
log.info('running on the {} survey...'.format(surv))
if surv != 'main':
scxdir = os.path.join(scxdir, surv)
# ADM read in non-OVERRIDE secondary targets.
scxtargs = read_files(scxdir, mx[3])
scxtargs = scxtargs[~scxtargs["OVERRIDE"]]
# ADM match primary targets to non-OVERRIDE secondary targets.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if nside is not None and pix is not None:
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(primtargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(primtargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(scndout, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
desi_mask = desimasks[0]
targs[desicols[0]][umtargs] |= desi_mask.SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# APC Secondary target bits only affect PRIORITY, NUMOBS and
# APC obsconditions for specific DESI_TARGET bits
# APC See https://github.com/desihub/desitarget/pull/530
# APC Only consider primary targets with secondary bits set
scnd_update = (targs[desicols[0]] & desi_mask['SCND_ANY']) != 0
if np.any(scnd_update):
# APC Allow changes to primaries if the DESI_TARGET bitmask has
# APC only the following bits set, in any combination.
log.info(
'Testing if secondary targets can update {} matched primaries'.
format(scnd_update.sum()))
update_from_scnd_bits = desi_mask['SCND_ANY'] | desi_mask[
'MWS_ANY'] | desi_mask['STD_BRIGHT'] | desi_mask[
'STD_FAINT'] | desi_mask['STD_WD']
scnd_update &= ((targs[desicols[0]] & ~update_from_scnd_bits) == 0)
log.info(
'Setting new priority, numobs and obsconditions from secondary for {} matched primaries'
.format(scnd_update.sum()))
# APC Primary and secondary obsconditions are or'd
scnd_obscon = set_obsconditions(targs[scnd_update], scnd=True)
targs['OBSCONDITIONS'][scnd_update] &= scnd_obscon
# APC bit of a hack here
# APC Check for _BRIGHT, _DARK split in column names
darkbright = 'NUMOBS_INIT_DARK' in targs.dtype.names
if darkbright:
ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"]
else:
ender, obscon = [""], [
"DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18"
]
# APC secondaries can increase priority and numobs
for edr, oc in zip(ender, obscon):
pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr
scnd_priority, scnd_numobs = initial_priority_numobs(
targs[scnd_update], obscon=oc, scnd=True)
targs[nc][scnd_update] = np.maximum(targs[nc][scnd_update],
scnd_numobs)
targs[pc][scnd_update] = np.maximum(targs[pc][scnd_update],
scnd_priority)
# ADM update the secondary targets with the primary information.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM the maximum priority will be used to break ties in the
# ADM unlikely event that a secondary matches two primaries.
hipri = np.maximum(targs["PRIORITY_INIT_DARK"],
targs["PRIORITY_INIT_BRIGHT"])
scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs]
# ADM write the secondary targets that have updated TARGETIDs.
ii = scxtargs["TARGETID"] != -1
nmatches = np.sum(ii)
log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format(
nmatches, scndout,
time() - start))
if nmatches > 0:
hdr = fitsio.FITSHDR()
hdr["SURVEY"] = surv
fitsio.write(scndout,
scxtargs[ii],
extname='SCND_TARG',
header=hdr,
clobber=True)
log.info('Done...t={:.1f}s'.format(time() - start))
return targs
def main(args):
log = get_logger()
cmd = [
'desi_compute_fluxcalibration',
]
for key, value in args.__dict__.items():
if value is not None:
cmd += ['--' + key, str(value)]
cmd = ' '.join(cmd)
log.info(cmd)
log.info("read frame")
# read frame
frame = read_frame(args.infile)
# Set fibermask flagged spectra to have 0 flux and variance
frame = get_fiberbitmasked_frame(frame,
bitmask='flux',
ivar_framemask=True)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel = read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models(
args.models)
ok = np.ones(len(model_metadata), dtype=bool)
if args.chi2cut > 0:
log.info("Apply cut CHI2DOF<{}".format(args.chi2cut))
ok &= (model_metadata["CHI2DOF"] < args.chi2cut)
if args.delta_color_cut > 0:
log.info("Apply cut |delta color|<{}".format(args.delta_color_cut))
ok &= (np.abs(model_metadata["MODEL_G-R"] - model_metadata["DATA_G-R"])
< args.delta_color_cut)
if args.min_color is not None:
log.info("Apply cut DATA_G-R>{}".format(args.min_color))
ok &= (model_metadata["DATA_G-R"] > args.min_color)
if args.chi2cut_nsig > 0:
# automatically reject stars that ar chi2 outliers
mchi2 = np.median(model_metadata["CHI2DOF"])
rmschi2 = np.std(model_metadata["CHI2DOF"])
maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2
log.info("Apply cut CHI2DOF<{} based on chi2cut_nsig={}".format(
maxchi2, args.chi2cut_nsig))
ok &= (model_metadata["CHI2DOF"] <= maxchi2)
ok = np.where(ok)[0]
if ok.size == 0:
log.error("cuts discarded all stars")
sys.exit(12)
nstars = model_flux.shape[0]
nbad = nstars - ok.size
if nbad > 0:
log.warning("discarding %d star(s) out of %d because of cuts" %
(nbad, nstars))
model_flux = model_flux[ok]
model_fibers = model_fibers[ok]
model_metadata = model_metadata[:][ok]
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
## check whether star fibers from args.models are consistent with fibers from fibermap
## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
fibermap_std_indices = np.where(isStdStar(fibermap))[0]
if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)):
target_colnames, target_masks, survey = main_cmx_or_sv(fibermap)
colname = target_colnames[0]
for i in model_fibers % 500:
log.error(
"inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap"
.format(i, fibermap["OBJTYPE"][i], colname,
fibermap[colname][i]))
sys.exit(12)
# Make sure the fibers of interest aren't entirely masked.
if np.sum(
np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) ==
frame.nwave) == len(model_fibers):
log.warning('All standard-star spectra are masked!')
return
fluxcalib = compute_flux_calibration(
frame,
model_wave,
model_flux,
model_fibers % 500,
highest_throughput_nstars=args.highest_throughput)
# QA
if (args.qafile is not None):
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame_meta=frame.meta,
flavor=frame.meta['FLAVOR'])
# Run
#import pdb; pdb.set_trace()
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s" % args.outfile)
def get_source_types(fibermap):
'''
Return a list of specsim source types based upon fibermap['DESI_TARGET']
Args:
fibermap: fibermap Table including DESI_TARGET column
Returns array of source_types 'sky', 'elg', 'lrg', 'qso', 'star'
Unassigned fibers fibermap['TARGETID'] == -1 will be treated as 'sky'
If fibermap.meta['FLAVOR'] = 'arc' or 'flat', returned source types will
match that flavor, though specsim doesn't use those as source_types
TODO: specsim/desimodel doesn't have a fiber input loss model for BGS yet,
so BGS targets get source_type = 'lrg' (!)
'''
from desiutil.log import get_logger
log = get_logger()
if ('DESI_TARGET' not in fibermap.dtype.names) and \
('SV1_DESI_TARGET' not in fibermap.dtype.names):
log.warning(
"(SV1_)DESI_TARGET not in fibermap table; using source_type='star' for everything"
)
return np.array([
'star',
] * len(fibermap))
target_colnames, target_masks, survey = main_cmx_or_sv(fibermap)
targetcol = target_colnames[0] #- DESI_TARGET or SV1_DESI_TARGET
tm = target_masks[0] #- desi_mask or sv1_desi_mask
source_type = np.zeros(len(fibermap), dtype='U4')
assert np.all(source_type == '')
if 'TARGETID' in fibermap.dtype.names:
unassigned = fibermap['TARGETID'] == -1
source_type[unassigned] = 'sky'
source_type[(fibermap['OBJTYPE'] == 'FLT')] = 'FLAT'
source_type[(fibermap['OBJTYPE'] == 'ARC')] = 'ARC'
source_type[(fibermap[targetcol] & tm.SKY) != 0] = 'sky'
source_type[(fibermap[targetcol] & tm.ELG) != 0] = 'elg'
source_type[(fibermap[targetcol] & tm.LRG) != 0] = 'lrg'
source_type[(fibermap[targetcol] & tm.QSO) != 0] = 'qso'
source_type[(fibermap[targetcol] & tm.BGS_ANY) != 0] = 'bgs'
starmask = 0
for name in [
'STD', 'STD_FSTAR', 'STD_WD', 'MWS_ANY', 'STD_FAINT',
'STD_FAINT_BEST', 'STD_BRIGHT', 'STD_BRIGHT_BEST'
]:
if name in desitarget.targetmask.desi_mask.names():
starmask |= desitarget.targetmask.desi_mask[name]
source_type[(fibermap[targetcol] & starmask) != 0] = 'star'
#- Simulate unassigned fibers as sky
## TODO: when fiberassign and desitarget are updated, use
## desitarget.targetmask.desi_mask.NO_TARGET to identify these
source_type[fibermap['TARGETID'] < 0] = 'sky'
assert not np.any(source_type == '')
for name in sorted(np.unique(source_type)):
n = np.count_nonzero(source_type == name)
log.debug('{} {} targets'.format(name, n))
return source_type
def make_mtl(targets, obscon, zcat=None, scnd=None,
trim=False, trimcols=False, trimtozcat=False):
"""Adds fiberassign and zcat columns to a targets table.
Parameters
----------
targets : :class:`~numpy.array` or `~astropy.table.Table`
A numpy rec array or astropy Table with at least the columns
``TARGETID``, ``DESI_TARGET``, ``NUMOBS_INIT``, ``PRIORITY_INIT``.
or the corresponding columns for SV or commissioning.
obscon : :class:`str`
A combination of strings that are in the desitarget bitmask yaml
file (specifically in `desitarget.targetmask.obsconditions`), e.g.
"DARK|GRAY". Governs the behavior of how priorities are set based
on "obsconditions" in the desitarget bitmask yaml file.
zcat : :class:`~astropy.table.Table`, optional
Redshift catalog table with columns ``TARGETID``, ``NUMOBS``, ``Z``,
``ZWARN``.
scnd : :class:`~numpy.array`, `~astropy.table.Table`, optional
A set of secondary targets associated with the `targets`. As with
the `target` must include at least ``TARGETID``, ``NUMOBS_INIT``,
``PRIORITY_INIT`` or the corresponding SV columns.
The secondary targets will be padded to have the same columns
as the targets, and concatenated with them.
trim : :class:`bool`, optional
If ``True`` (default), don't include targets that don't need
any more observations. If ``False``, include every input target.
trimcols : :class:`bool`, optional, defaults to ``False``
Only pass through columns in `targets` that are actually needed
for fiberassign (see `desitarget.mtl.mtldatamodel`).
trimtozcat : :class:`bool`, optional, defaults to ``False``
Only return targets that have been UPDATED (i.e. the targets with
a match in `zcat`). Returns all targets if `zcat` is ``None``.
Returns
-------
:class:`~astropy.table.Table`
MTL Table with targets columns plus:
* NUMOBS_MORE - number of additional observations requested
* PRIORITY - target priority (larger number = higher priority)
* TARGET_STATE - the observing state that corresponds to PRIORITY
* OBSCONDITIONS - replaces old GRAYLAYER
* TIMESTAMP - time that (this) make_mtl() function was run
* VERSION - version of desitarget used to run make_mtl()
"""
start = time()
# ADM set up the default logger.
from desiutil.log import get_logger
log = get_logger()
# ADM if trimcols was passed, reduce input target columns to minimal.
if trimcols:
mtldm = switch_main_cmx_or_sv(mtldatamodel, targets)
cullcols = list(set(targets.dtype.names) - set(mtldm.dtype.names))
if isinstance(targets, Table):
targets.remove_columns(cullcols)
else:
targets = rfn.drop_fields(targets, cullcols)
# ADM determine whether the input targets are main survey, cmx or SV.
colnames, masks, survey = main_cmx_or_sv(targets, scnd=True)
# ADM set the first column to be the "desitarget" column
desi_target, desi_mask = colnames[0], masks[0]
scnd_target = colnames[-1]
# ADM if secondaries were passed, concatenate them with the targets.
if scnd is not None:
nrows = len(scnd)
log.info('Pad {} primary targets with {} secondaries...t={:.1f}s'.format(
len(targets), nrows, time()-start))
padit = np.zeros(nrows, dtype=targets.dtype)
sharedcols = set(targets.dtype.names).intersection(set(scnd.dtype.names))
for col in sharedcols:
padit[col] = scnd[col]
targets = np.concatenate([targets, padit])
# APC Propagate a flag on which targets came from scnd
is_scnd = np.repeat(False, len(targets))
is_scnd[-nrows:] = True
log.info('Done with padding...t={:.1f}s'.format(time()-start))
# Trim targets from zcat that aren't in original targets table.
if zcat is not None:
ok = np.in1d(zcat['TARGETID'], targets['TARGETID'])
num_extra = np.count_nonzero(~ok)
if num_extra > 0:
log.warning("Ignoring {} zcat entries that aren't "
"in the input target list".format(num_extra))
zcat = zcat[ok]
n = len(targets)
# ADM if a redshift catalog was passed, order it to match the input targets
# ADM catalog on 'TARGETID'.
if zcat is not None:
# ADM there might be a quicker way to do this?
# ADM set up a dictionary of the indexes of each target id.
d = dict(tuple(zip(targets["TARGETID"], np.arange(n))))
# ADM loop through the zcat and look-up the index in the dictionary.
zmatcher = np.array([d[tid] for tid in zcat["TARGETID"]])
ztargets = zcat
if ztargets.masked:
unobs = ztargets['NUMOBS'].mask
ztargets['NUMOBS'][unobs] = 0
unobsz = ztargets['Z'].mask
ztargets['Z'][unobsz] = -1
unobszw = ztargets['ZWARN'].mask
ztargets['ZWARN'][unobszw] = -1
else:
ztargets = Table()
ztargets['TARGETID'] = targets['TARGETID']
ztargets['NUMOBS'] = np.zeros(n, dtype=np.int32)
ztargets['Z'] = -1 * np.ones(n, dtype=np.float32)
ztargets['ZWARN'] = -1 * np.ones(n, dtype=np.int32)
# ADM if zcat wasn't passed, there is a one-to-one correspondence
# ADM between the targets and the zcat.
zmatcher = np.arange(n)
# ADM extract just the targets that match the input zcat.
targets_zmatcher = targets[zmatcher]
# ADM update the number of observations for the targets.
ztargets['NUMOBS_MORE'] = calc_numobs_more(targets_zmatcher, ztargets, obscon)
# ADM assign priorities. Only things in the zcat can have changed
# ADM priorities. Anything else is assigned PRIORITY_INIT, below.
priority, target_state = calc_priority(
targets_zmatcher, ztargets, obscon, state=True)
# If priority went to 0==DONOTOBSERVE or 1==OBS or 2==DONE, then
# NUMOBS_MORE should also be 0.
# ## mtl['NUMOBS_MORE'] = ztargets['NUMOBS_MORE']
ii = (priority <= 2)
log.info('{:d} of {:d} targets have priority zero, setting N_obs=0.'.format(
np.sum(ii), n))
ztargets['NUMOBS_MORE'][ii] = 0
# - Set the OBSCONDITIONS mask for each target bit.
obsconmask = set_obsconditions(targets)
# APC obsconmask will now be incorrect for secondary-only targets. Fix this
# APC using the mask on secondary targets.
if scnd is not None:
obsconmask[is_scnd] = set_obsconditions(targets[is_scnd], scnd=True)
# ADM set up the output mtl table.
mtl = Table(targets)
mtl.meta['EXTNAME'] = 'MTL'
# ADM add a placeholder for the secondary bit-mask, if it isn't there.
if scnd_target not in mtl.dtype.names:
mtl[scnd_target] = np.zeros(len(mtl),
dtype=mtldatamodel["SCND_TARGET"].dtype)
# ADM initialize columns to avoid zero-length/missing/format errors.
zcols = ["NUMOBS_MORE", "NUMOBS", "Z", "ZWARN"]
for col in zcols + ["TARGET_STATE", "TIMESTAMP", "VERSION"]:
mtl[col] = np.empty(len(mtl), dtype=mtldatamodel[col].dtype)
# ADM any target that wasn't matched to the ZCAT should retain its
# ADM original (INIT) value of PRIORITY and NUMOBS.
mtl['NUMOBS_MORE'] = mtl['NUMOBS_INIT']
mtl['PRIORITY'] = mtl['PRIORITY_INIT']
mtl['TARGET_STATE'] = "UNOBS"
# ADM add the time and version of the desitarget code that was run.
utc = datetime.utcnow().isoformat(timespec='seconds')
mtl["TIMESTAMP"] = utc
mtl["VERSION"] = dt_version
# ADM now populate the new mtl columns with the updated information.
mtl['OBSCONDITIONS'] = obsconmask
mtl['PRIORITY'][zmatcher] = priority
mtl['TARGET_STATE'][zmatcher] = target_state
for col in zcols:
mtl[col][zmatcher] = ztargets[col]
# Filter out any targets marked as done.
if trim:
notdone = mtl['NUMOBS_MORE'] > 0
log.info('{:d} of {:d} targets are done, trimming these'.format(
len(mtl) - np.sum(notdone), len(mtl))
)
mtl = mtl[notdone]
# Filtering can reset the fill_value, which is just wrong wrong wrong
# See https://github.com/astropy/astropy/issues/4707
# and https://github.com/astropy/astropy/issues/4708
mtl['NUMOBS_MORE'].fill_value = -1
# ADM assert the data model is complete.
# ADM turning this off for now, useful for testing.
# mtltypes = [mtl[i].dtype.type for i in mtl.dtype.names]
# mtldmtypes = [mtldm[i].dtype.type for i in mtl.dtype.names]
# assert set(mtl.dtype.names) == set(mtldm.dtype.names)
# assert mtltypes == mtldmtypes
log.info('Done...t={:.1f}s'.format(time()-start))
if trimtozcat:
return mtl[zmatcher]
return mtl
def test_priorities(self):
"""Test that priorities are set correctly for both the main survey and SV.
"""
# ADM loop through once for SV and once for the main survey.
for prefix in ["", "SV1_"]:
t = self.targets.copy()
z = self.zcat.copy()
main_names = ['DESI_TARGET', 'BGS_TARGET', 'MWS_TARGET']
for name in main_names:
t.rename_column(name, prefix + name)
# ADM retrieve the mask and column names for this survey flavor.
colnames, masks, _ = main_cmx_or_sv(t)
desi_target, bgs_target, mws_target = colnames
desi_mask, bgs_mask, mws_mask = masks
# - No targeting bits set is priority=0
self.assertTrue(np.all(calc_priority(t, z) == 0))
# - test QSO > (LRG_1PASS | LRG_2PASS) > ELG
t[desi_target] = desi_mask.ELG
self.assertTrue(
np.all(
calc_priority(t, z) == desi_mask.ELG.priorities['UNOBS']))
t[desi_target] |= desi_mask.LRG_1PASS
self.assertTrue(
np.all(
calc_priority(t, z) == desi_mask.LRG.priorities['UNOBS']))
t[desi_target] |= desi_mask.LRG_2PASS
self.assertTrue(
np.all(
calc_priority(t, z) == desi_mask.LRG.priorities['UNOBS']))
t[desi_target] |= desi_mask.QSO
self.assertTrue(
np.all(
calc_priority(t, z) == desi_mask.QSO.priorities['UNOBS']))
# - different states -> different priorities
# - Done is Done, regardless of ZWARN.
t[desi_target] = desi_mask.ELG
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 1, 1]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, z)["PRIORITY"]
self.assertEqual(p[0], desi_mask.ELG.priorities['UNOBS'])
self.assertEqual(p[1], desi_mask.ELG.priorities['DONE'])
self.assertEqual(p[2], desi_mask.ELG.priorities['DONE'])
# - BGS FAINT targets are never DONE, only MORE_ZGOOD.
t[desi_target] = desi_mask.BGS_ANY
t[bgs_target] = bgs_mask.BGS_FAINT
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 1, 1]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, z)["PRIORITY"]
self.assertEqual(p[0], bgs_mask.BGS_FAINT.priorities['UNOBS'])
self.assertEqual(p[1], bgs_mask.BGS_FAINT.priorities['MORE_ZWARN'])
self.assertEqual(p[2], bgs_mask.BGS_FAINT.priorities['MORE_ZGOOD'])
# BGS_FAINT: {UNOBS: 2000, MORE_ZWARN: 2000, MORE_ZGOOD: 1000, DONE: 2, OBS: 1, DONOTOBSERVE: 0}
# - BGS BRIGHT targets are never DONE, only MORE_ZGOOD.
t[desi_target] = desi_mask.BGS_ANY
t[bgs_target] = bgs_mask.BGS_BRIGHT
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 1, 1]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, z)["PRIORITY"]
self.assertEqual(p[0], bgs_mask.BGS_BRIGHT.priorities['UNOBS'])
self.assertEqual(p[1],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZWARN'])
self.assertEqual(p[2],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZGOOD'])
# BGS_BRIGHT: {UNOBS: 2100, MORE_ZWARN: 2100, MORE_ZGOOD: 1000, DONE: 2, OBS: 1, DONOTOBSERVE: 0}
# BGS targets are NEVER done even after 100 observations
t[desi_target] = desi_mask.BGS_ANY
t[bgs_target] = bgs_mask.BGS_BRIGHT
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 100, 100]
z['ZWARN'] = [1, 1, 0]
p = calc_priority(t, z)
self.assertEqual(p[0], bgs_mask.BGS_BRIGHT.priorities['UNOBS'])
self.assertEqual(p[1],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZWARN'])
self.assertEqual(p[2],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZGOOD'])
# BGS ZGOOD targets always have lower priority than MWS targets that
# are not DONE.
# ADM first discard N/S informational bits from bitmask as these
# ADM should never trump the other bits.
bgs_names = [
name for name in bgs_mask.names()
if 'NORTH' not in name and 'SOUTH' not in name
]
mws_names = [
name for name in mws_mask.names()
if 'NORTH' not in name and 'SOUTH' not in name
]
lowest_bgs_priority_zgood = min(
[bgs_mask[n].priorities['MORE_ZGOOD'] for n in bgs_names])
lowest_mws_priority_unobs = min(
[mws_mask[n].priorities['UNOBS'] for n in mws_names])
lowest_mws_priority_zwarn = min(
[mws_mask[n].priorities['MORE_ZWARN'] for n in mws_names])
lowest_mws_priority_zgood = min(
[mws_mask[n].priorities['MORE_ZGOOD'] for n in mws_names])
lowest_mws_priority = min(lowest_mws_priority_unobs,
lowest_mws_priority_zwarn,
lowest_mws_priority_zgood)
self.assertLess(lowest_bgs_priority_zgood, lowest_mws_priority)
def match_secondary(infile, scxtargs, sep=1., scxdir=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
infile : :class:`str`
The full path to a file containing primary targets.
scxtargs : :class:`~numpy.ndarray`
An array of secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets. If passed,
this is written to the output primary file header as `SCNDDIR`.
Returns
-------
:class:`~numpy.ndarray`
The array of secondary targets, with the `TARGETID` bit
updated with any matching primary targets from `infile`.
Notes
-----
- The primary target `infiles` are written back to their
original path with `.fits` changed to `-wscnd.fits` and the
`SCND_TARGET` bit populated for matching targets.
"""
# ADM just the file name for logging.
fn = os.path.basename(infile)
# ADM read in the primary targets.
log.info('Reading primary targets file {}...t={:.1f}s'.format(
infile,
time() - start))
intargs, hdr = fitsio.read(infile, extension="TARGETS", header=True)
# ADM fail if file's already been matched to secondary targets.
if "SCNDDIR" in hdr:
msg = "{} already matched to secondary targets".format(fn) \
+ " (did you mean to remove {}?)!!!".format(fn)
log.critical(msg)
raise ValueError(msg)
# ADM add the SCNDDIR to the primary targets file header.
hdr["SCNDDIR"] = scxdir
# ADM add a SCND_TARGET column to the primary targets.
dt = intargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(intargs), dtype=dt)
for col in intargs.dtype.names:
targs[col] = intargs[col]
# ADM match to all secondary targets for non-custom primary files.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if 'FILEHPX' in hdr:
nside, pix = hdr['FILENSID'], hdr['FILEHPX']
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(intargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(intargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(fn, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# ADM update the secondary targets with the primary TARGETID.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM form the output primary file name and write the file.
base, ext = os.path.splitext(infile)
outfile = "{}{}{}".format(base, '-wscnd', ext)
log.info('Writing updated primary targets to {}...t={:.1f}s'.format(
outfile,
time() - start))
fitsio.write(outfile, targs, extname='TARGETS', header=hdr, clobber=True)
log.info('Done for {}...t={:.1f}s'.format(fn, time() - start))
return scxtargs
def make_mtl(targets, zcat=None, trim=False):
"""Adds NUMOBS, PRIORITY, and OBSCONDITIONS columns to a targets table.
Parameters
----------
targets : :class:`~numpy.array` or `~astropy.table.Table`
A numpy rec array or astropy Table with at least the columns
``TARGETID``, ``DESI_TARGET``, ``NUMOBS_INIT``, ``PRIORITY_INIT``.
or the corresponding columns for SV or commissioning.
zcat : :class:`~astropy.table.Table`, optional
Redshift catalog table with columns ``TARGETID``, ``NUMOBS``, ``Z``,
``ZWARN``.
trim : :class:`bool`, optional
If ``True`` (default), don't include targets that don't need
any more observations. If ``False``, include every input target.
Returns
-------
:class:`~astropy.table.Table`
MTL Table with targets columns plus:
* NUMOBS_MORE - number of additional observations requested
* PRIORITY - target priority (larger number = higher priority)
* OBSCONDITIONS - replaces old GRAYLAYER
"""
# ADM set up the default logger.
from desiutil.log import get_logger
log = get_logger()
# ADM determine whether the input targets are main survey, cmx or SV.
colnames, masks, survey = main_cmx_or_sv(targets)
# ADM set the first column to be the "desitarget" column
desi_target, desi_mask = colnames[0], masks[0]
# Trim targets from zcat that aren't in original targets table
if zcat is not None:
ok = np.in1d(zcat['TARGETID'], targets['TARGETID'])
num_extra = np.count_nonzero(~ok)
if num_extra > 0:
log.warning("Ignoring {} zcat entries that aren't "
"in the input target list".format(num_extra))
zcat = zcat[ok]
n = len(targets)
# ADM if the input target columns were incorrectly called NUMOBS or PRIORITY
# ADM rename them to NUMOBS_INIT or PRIORITY_INIT.
# ADM Note that the syntax is slightly different for a Table.
for name in ['NUMOBS', 'PRIORITY']:
if isinstance(targets, Table):
try:
targets.rename_column(name, name + '_INIT')
except KeyError:
pass
else:
targets.dtype.names = [
name + '_INIT' if col == name else col
for col in targets.dtype.names
]
# ADM if a redshift catalog was passed, order it to match the input targets
# ADM catalog on 'TARGETID'.
if zcat is not None:
# ADM there might be a quicker way to do this?
# ADM set up a dictionary of the indexes of each target id.
d = dict(tuple(zip(targets["TARGETID"], np.arange(n))))
# ADM loop through the zcat and look-up the index in the dictionary.
zmatcher = np.array([d[tid] for tid in zcat["TARGETID"]])
ztargets = zcat
if ztargets.masked:
unobs = ztargets['NUMOBS'].mask
ztargets['NUMOBS'][unobs] = 0
unobsz = ztargets['Z'].mask
ztargets['Z'][unobsz] = -1
unobszw = ztargets['ZWARN'].mask
ztargets['ZWARN'][unobszw] = -1
else:
ztargets = Table()
ztargets['TARGETID'] = targets['TARGETID']
ztargets['NUMOBS'] = np.zeros(n, dtype=np.int32)
ztargets['Z'] = -1 * np.ones(n, dtype=np.float32)
ztargets['ZWARN'] = -1 * np.ones(n, dtype=np.int32)
# ADM if zcat wasn't passed, there is a one-to-one correspondence
# ADM between the targets and the zcat.
zmatcher = np.arange(n)
# ADM extract just the targets that match the input zcat.
targets_zmatcher = targets[zmatcher]
# ADM use passed value of NUMOBS_INIT instead of calling the memory-heavy calc_numobs.
# ztargets['NUMOBS_MORE'] = np.maximum(0, calc_numobs(ztargets) - ztargets['NUMOBS'])
ztargets['NUMOBS_MORE'] = np.maximum(
0, targets_zmatcher['NUMOBS_INIT'] - ztargets['NUMOBS'])
# ADM we need a minor hack to ensure that BGS targets are observed once (and only once)
# ADM every time, regardless of how many times they've previously been observed.
# ADM I've turned this off for commissioning. Not sure if we'll keep it in general.
if survey != 'cmx':
ii = targets_zmatcher[desi_target] & desi_mask.BGS_ANY > 0
ztargets['NUMOBS_MORE'][ii] = 1
# ADM assign priorities, note that only things in the zcat can have changed priorities.
# ADM anything else will be assigned PRIORITY_INIT, below.
priority = calc_priority(targets_zmatcher, ztargets)
# If priority went to 0==DONOTOBSERVE or 1==OBS or 2==DONE, then NUMOBS_MORE should also be 0.
# ## mtl['NUMOBS_MORE'] = ztargets['NUMOBS_MORE']
ii = (priority <= 2)
log.info(
'{:d} of {:d} targets have priority zero, setting N_obs=0.'.format(
np.sum(ii), n))
ztargets['NUMOBS_MORE'][ii] = 0
# - Set the OBSCONDITIONS mask for each target bit.
obscon = set_obsconditions(targets)
# ADM set up the output mtl table.
mtl = Table(targets)
mtl.meta['EXTNAME'] = 'MTL'
# ADM any target that wasn't matched to the ZCAT should retain its
# ADM original (INIT) value of PRIORITY and NUMOBS.
mtl['NUMOBS_MORE'] = mtl['NUMOBS_INIT']
mtl['PRIORITY'] = mtl['PRIORITY_INIT']
# ADM now populate the new mtl columns with the updated information.
mtl['OBSCONDITIONS'] = obscon
mtl['PRIORITY'][zmatcher] = priority
mtl['NUMOBS_MORE'][zmatcher] = ztargets['NUMOBS_MORE']
# Filter out any targets marked as done.
if trim:
notdone = mtl['NUMOBS_MORE'] > 0
log.info('{:d} of {:d} targets are done, trimming these'.format(
len(mtl) - np.sum(notdone), len(mtl)))
mtl = mtl[notdone]
# Filtering can reset the fill_value, which is just wrong wrong wrong
# See https://github.com/astropy/astropy/issues/4707
# and https://github.com/astropy/astropy/issues/4708
mtl['NUMOBS_MORE'].fill_value = -1
return mtl
def run(self, indir):
'''TODO: document'''
log = desiutil.log.get_logger()
results = list()
infiles = glob.glob(os.path.join(indir, 'qcframe-*.fits'))
if len(infiles) == 0:
log.error("no qcframe in {}".format(indir))
return None
# find number of spectros
spectros = []
for filename in infiles:
hdr = fitsio.read_header(filename)
s = int(hdr['CAMERA'][1])
spectros.append(s)
spectros = np.unique(spectros)
for spectro in spectros:
infiles = glob.glob(
os.path.join(indir, 'qcframe-?{}-*.fits'.format(spectro)))
qframes = {}
fmap = None
for infile in infiles:
qframe = read_qframe(infile)
cam = qframe.meta["CAMERA"][0].upper()
qframes[cam] = qframe
if fmap is None:
fmap = qframe.fibermap
night = int(qframe.meta['NIGHT'])
expid = int(qframe.meta['EXPID'])
# need to decode fibermap
columns, masks, survey = targets.main_cmx_or_sv(fmap)
desi_target = fmap[columns[0]] # (SV1_)DESI_TARGET
desi_mask = masks[0] # (sv1) desi_mask
stars = (
desi_target
& desi_mask.mask('STD_WD|STD_FAINT|STD_BRIGHT')) != 0
qsos = (desi_target & desi_mask.QSO) != 0
# sw = np.zeros(self.rwave.size)
# swf = np.zeros(self.rwave.size)
#- we need a ridiculously large array to cover the r filter, but precache
#- what wavelength ranges matter for each band
iiband = dict()
for c in ["B", "R", "Z"]:
if c in qframes:
qframe = qframes[c]
iiband[c] = (qframe.wave[0][0] < self.rwave)
iiband[c] &= (self.rwave < qframe.wave[0][-1])
#- for each fiber, generate list of arguments to pass to get_dico
#- get_fiber_data extracts data for only *one* fiber from qframes, fmap, which have data for all fibers
#- this reduces the parallel processing overhead
argslist = [(self, iiband,
get_fiber_data(qframes, fmap, f, fiber, night, expid,
spectro, stars, qsos))
for f, fiber in enumerate(fmap["FIBER"])]
ncpu = get_ncpu(None)
if ncpu > 1:
pool = mp.Pool(ncpu)
results = pool.starmap(get_dico, argslist)
else:
for args in argslist:
results.append(get_dico(**args))
if len(results) == 0:
return None
return Table(results, names=results[0].keys())
