def load_target_table(tgs,
tgdata,
survey=None,
typeforce=None,
typecol=None,
sciencemask=None,
stdmask=None,
skymask=None,
suppskymask=None,
safemask=None,
excludemask=None):
"""Append targets from a table.
Use the table data to append targets to the input Targets object.
A subset of the columns in the file will be stored in each Target added
to the Targets object. Each target is classified into one or more of the
4 types used internally in assignment (science, standard, sky, safe).
This classification is controlled by applying bitmasks to the specified
data column. Alternatively, all targets in the file can be forced to one
type.
Args:
tgs (Targets): The targets object on which to append this data.
tgdata (Table): A table or recarray with the target properties.
survey (str): The survey type. If None, query from columns.
typeforce (int): If specified, it must equal one of the TARGET_TYPE_*
values. All targets read from the file will be assigned this type.
typecol (str): Optional column to use for bitmask matching (default
uses the result of main_cmx_or_sv from desitarget).
sciencemask (int): Bitmask for classifying targets as science.
stdmask (int): Bitmask for classifying targets as a standard.
skymask (int): Bitmask for classifying targets as sky.
suppskymask (int): Bitmask for classifying targets as suppsky.
safemask (int): Bitmask for classifying targets as a safe location.
excludemask (int): Bitmask for excluding targets.
Returns:
None
"""
log = Logger.get()
if "TARGETID" not in tgdata.dtype.names:
msg = "TARGETID column is required"
log.error(msg)
raise RuntimeError(msg)
if tgdata.dtype["TARGETID"].char != "l":
msg = "TARGETID column should be int64"
log.error(msg)
raise RuntimeError(msg)
if "PRIORITY" in tgdata.dtype.names:
if tgdata.dtype["PRIORITY"].char not in ["i", "l"]:
msg = "PRIORITY column should be an integer type"
log.error(msg)
raise RuntimeError(msg)
if "SUBPRIORITY" not in tgdata.dtype.names:
msg = "SUBPRIORITY column is required"
log.error(msg)
raise RuntimeError(msg)
if tgdata.dtype["SUBPRIORITY"].char != "d":
msg = "SUBPRIORITY column should be float64"
log.error(msg)
raise RuntimeError(msg)
if "NUMOBS_MORE" in tgdata.dtype.names:
if tgdata.dtype["NUMOBS_MORE"].char not in ["i", "l"]:
msg = "NUMOBS_MORE column should be an integer type"
log.error(msg)
raise RuntimeError(msg)
if "NUMOBS_INIT" in tgdata.dtype.names:
if tgdata.dtype["NUMOBS_INIT"].char not in ["i", "l"]:
msg = "NUMOBS_INIT column should be an integer type"
log.error(msg)
raise RuntimeError(msg)
if "OBSCONDITIONS" not in tgdata.dtype.names:
msg = "OBSCONDITIONS column is required"
log.error(msg)
raise RuntimeError(msg)
if tgdata.dtype["OBSCONDITIONS"].char not in ["i", "l"]:
msg = "OBSCONDITIONS column should be an integer type"
log.error(msg)
raise RuntimeError(msg)
# Are we loading raw output? If so, we require the survey key to get
# the default masks.
fsurvey = None
fcol = None
fsciencemask = None
fstdmask = None
fskymask = None
fsuppskymask = None
fsafemask = None
fexcludemask = None
if typecol == "FA_TYPE":
if survey is None:
msg = "When loading raw fiberassign tables, the survey must be \
specified"
log.error(msg)
raise RuntimeError(msg)
fsciencemask, fstdmask, fskymask, fsuppskymask, fsafemask, \
fexcludemask = default_survey_target_masks(survey)
else:
fsurvey, fcol, fsciencemask, fstdmask, fskymask, fsuppskymask, \
fsafemask, fexcludemask = default_target_masks(tgdata)
if fcol is None:
# File could not be identified. In this case, the user must
# completely specify the bitmask and column to use.
if typeforce is None:
if (typecol is None) or (sciencemask is None) \
or (stdmask is None) or (skymask is None) \
or (suppskymask is None) or (safemask is None) \
or (excludemask is None):
msg = "Unknown survey type. To use this table, \
specify the column name and every bitmask."
log.error(msg)
raise RuntimeError(msg)
if survey is None:
survey = fsurvey
if typecol is None:
typecol = fcol
if sciencemask is None:
sciencemask = fsciencemask
if stdmask is None:
stdmask = fstdmask
if skymask is None:
skymask = fskymask
if suppskymask is None:
suppskymask = fsuppskymask
if safemask is None:
safemask = fsafemask
if excludemask is None:
excludemask = fexcludemask
log.debug("Target table using survey '{}', column {}:".format(
survey, typecol))
if survey == "main":
log.debug(" sciencemask {}".format("|".join(
desi_mask.names(sciencemask))))
log.debug(" stdmask {}".format("|".join(
desi_mask.names(stdmask))))
log.debug(" skymask {}".format("|".join(
desi_mask.names(skymask))))
log.debug(" suppskymask {}".format("|".join(
desi_mask.names(suppskymask))))
log.debug(" safemask {}".format("|".join(
desi_mask.names(safemask))))
log.debug(" excludemask {}".format("|".join(
desi_mask.names(excludemask))))
elif survey == "cmx":
log.debug(" sciencemask {}".format("|".join(
cmx_mask.names(sciencemask))))
log.debug(" stdmask {}".format("|".join(cmx_mask.names(stdmask))))
log.debug(" skymask {}".format("|".join(cmx_mask.names(skymask))))
log.debug(" suppskymask {}".format("|".join(
cmx_mask.names(suppskymask))))
log.debug(" safemask {}".format("|".join(
cmx_mask.names(safemask))))
log.debug(" excludemask {}".format("|".join(
cmx_mask.names(excludemask))))
elif survey == "sv1":
log.debug(" sciencemask {}".format("|".join(
sv1_mask.names(sciencemask))))
log.debug(" stdmask {}".format("|".join(sv1_mask.names(stdmask))))
log.debug(" skymask {}".format("|".join(sv1_mask.names(skymask))))
log.debug(" suppskymask {}".format("|".join(
sv1_mask.names(suppskymask))))
log.debug(" safemask {}".format("|".join(
sv1_mask.names(safemask))))
log.debug(" excludemask {}".format("|".join(
sv1_mask.names(excludemask))))
else:
raise RuntimeError("unknown survey type, should never get here!")
append_target_table(tgs, tgdata, survey, typeforce, typecol, sciencemask,
stdmask, skymask, suppskymask, safemask, excludemask)
return
def specviewer_selection(spectra,
log=None,
mask=None,
mask_type=None,
gmag_cut=None,
rmag_cut=None,
chi2cut=None,
zbest=None,
snr_cut=None):
'''
Simple sub-selection on spectra based on meta-data.
Implemented cuts based on : target mask ; photo mag (g, r) ; chi2 from fit ; SNR (in spectra.scores, BRZ)
'''
# Target mask selection
if mask is not None:
assert mask_type in ['SV1_DESI_TARGET', 'DESI_TARGET', 'CMX_TARGET']
if mask_type == 'SV1_DESI_TARGET':
assert (mask in sv1_desi_mask.names())
w, = np.where(
(spectra.fibermap['SV1_DESI_TARGET'] & sv1_desi_mask[mask]))
elif mask_type == 'DESI_TARGET':
assert (mask in desi_mask.names())
w, = np.where((spectra.fibermap['DESI_TARGET'] & desi_mask[mask]))
elif mask_type == 'CMX_TARGET':
assert (mask in cmx_mask.names())
w, = np.where((spectra.fibermap['CMX_TARGET'] & cmx_mask[mask]))
if len(w) == 0:
if log is not None: log.info(" * No spectra with mask " + mask)
return 0
else:
targetids = spectra.fibermap['TARGETID'][w]
spectra = myspecselect.myspecselect(spectra, targets=targetids)
# Photometry selection
if gmag_cut is not None:
assert len(gmag_cut) == 2 # Require range [gmin, gmax]
gmag = np.zeros(spectra.num_spectra())
w, = np.where((spectra.fibermap['FLUX_G'] > 0)
& (spectra.fibermap['MW_TRANSMISSION_G'] > 0))
gmag[w] = -2.5 * np.log10(
spectra.fibermap['FLUX_G'][w] /
spectra.fibermap['MW_TRANSMISSION_G'][w]) + 22.5
w, = np.where((gmag > gmag_cut[0]) & (gmag < gmag_cut[1]))
if len(w) == 0:
if log is not None:
log.info(" * No spectra with g_mag in requested range")
return 0
else:
targetids = spectra.fibermap['TARGETID'][w]
spectra = myspecselect.myspecselect(spectra, targets=targetids)
if rmag_cut is not None:
assert len(rmag_cut) == 2 # Require range [rmin, rmax]
rmag = np.zeros(spectra.num_spectra())
w, = np.where((spectra.fibermap['FLUX_R'] > 0)
& (spectra.fibermap['MW_TRANSMISSION_R'] > 0))
rmag[w] = -2.5 * np.log10(
spectra.fibermap['FLUX_R'][w] /
spectra.fibermap['MW_TRANSMISSION_R'][w]) + 22.5
w, = np.where((rmag > rmag_cut[0]) & (rmag < rmag_cut[1]))
if len(w) == 0:
if log is not None:
log.info(" * No spectra with r_mag in requested range")
return 0
else:
targetids = spectra.fibermap['TARGETID'][w]
spectra = myspecselect.myspecselect(spectra, targets=targetids)
# SNR selection ## TODO check it !! May not work ...
if snr_cut is not None:
assert ((len(snr_cut) == 2) and (spectra.scores is not None))
for band in ['B', 'R', 'Z']:
w, = np.where(
(spectra.scores['MEDIAN_CALIB_SNR_' + band] > snr_cut[0])
& (spectra.scores['MEDIAN_CALIB_SNR_' + band] < snr_cut[1]))
if len(w) == 0:
if log is not None:
log.info(" * No spectra with MEDIAN_CALIB_SNR_" + band +
" in requested range")
return 0
else:
targetids = spectra.fibermap['TARGETID'][w]
spectra = myspecselect.myspecselect(spectra, targets=targetids)
# Chi2 selection
if chi2cut is not None:
assert len(chi2cut) == 2 # Require range [chi2min, chi2max]
assert (zbest is not None)
thezb, kk = match_zcat_to_spectra(zbest, spectra)
w, = np.where((thezb['DELTACHI2'] > chi2cut[0])
& (thezb['DELTACHI2'] < chi2cut[1]))
if len(w) == 0:
if log is not None:
log.info(
" * No target in this pixel with DeltaChi2 in requested range"
)
return 0
else:
targetids = spectra.fibermap['TARGETID'][w]
spectra = myspecselect.myspecselect(spectra, targets=targetids)
return spectra
def specviewer_selection(spectra,
log=None,
mask=None,
mask_type=None,
gmag_cut=None,
rmag_cut=None,
chi2cut=None,
zbest=None,
snr_cut=None,
with_dirty_mask_merge=False,
remove_scores=False):
'''
Simple sub-selection on spectra based on meta-data.
Implemented cuts based on : target mask ; photo mag (g, r) ; chi2 from fit ; SNR (in spectra.scores, BRZ)
- if chi2cut : a catalog zbest must be provided, with entries matching exactly those of spectra
'''
# SNR selection
if snr_cut is not None:
assert ((len(snr_cut) == 2) and (spectra.scores is not None))
for band in ['B', 'R', 'Z']:
w, = np.where(
(spectra.scores['MEDIAN_CALIB_SNR_' + band] > snr_cut[0])
& (spectra.scores['MEDIAN_CALIB_SNR_' + band] < snr_cut[1]))
if len(w) == 0:
if log is not None:
log.info(" * No spectra with MEDIAN_CALIB_SNR_" + band +
" in requested range")
return 0
else:
targetids = spectra.fibermap['TARGETID'][w]
spectra = myspecselect.myspecselect(
spectra, targets=targetids, remove_scores=remove_scores)
# Target mask selection
if mask is not None:
assert mask_type in ['SV1_DESI_TARGET', 'DESI_TARGET', 'CMX_TARGET']
if mask_type == 'SV1_DESI_TARGET':
assert (mask in sv1_desi_mask.names())
w, = np.where(
(spectra.fibermap['SV1_DESI_TARGET'] & sv1_desi_mask[mask]))
elif mask_type == 'DESI_TARGET':
assert (mask in desi_mask.names())
w, = np.where((spectra.fibermap['DESI_TARGET'] & desi_mask[mask]))
elif mask_type == 'CMX_TARGET':
assert (mask in cmx_mask.names())
mask2 = None
if with_dirty_mask_merge: # Self-explanatory... only for fast VI of minisv
if mask in ['SV0_QSO', 'SV0_ELG', 'SV0_LRG']:
mask2 = mask.replace('SV0', 'MINI_SV')
if mask == 'SV0_BGS': mask2 = 'MINI_SV_BGS_BRIGHT'
if mask in ['SV0_STD_FAINT', 'SV0_STD_BRIGHT']:
mask2 = mask.replace('SV0_', '')
if mask2 is None:
w, = np.where(
(spectra.fibermap['CMX_TARGET'] & cmx_mask[mask]))
else:
w, = np.where((spectra.fibermap['CMX_TARGET'] & cmx_mask[mask])
| (spectra.fibermap['CMX_TARGET']
& cmx_mask[mask2]))
if len(w) == 0:
if log is not None: log.info(" * No spectra with mask " + mask)
return 0
else:
targetids = spectra.fibermap['TARGETID'][w]
spectra = myspecselect.myspecselect(spectra,
targets=targetids,
remove_scores=remove_scores)
# Photometry selection
if gmag_cut is not None:
assert len(gmag_cut) == 2 # Require range [gmin, gmax]
gmag = np.zeros(spectra.num_spectra())
w, = np.where((spectra.fibermap['FLUX_G'] > 0)
& (spectra.fibermap['MW_TRANSMISSION_G'] > 0))
gmag[w] = -2.5 * np.log10(
spectra.fibermap['FLUX_G'][w] /
spectra.fibermap['MW_TRANSMISSION_G'][w]) + 22.5
w, = np.where((gmag > gmag_cut[0]) & (gmag < gmag_cut[1]))
if len(w) == 0:
if log is not None:
log.info(" * No spectra with g_mag in requested range")
return 0
else:
targetids = spectra.fibermap['TARGETID'][w]
spectra = myspecselect.myspecselect(spectra, targets=targetids)
if rmag_cut is not None:
assert len(rmag_cut) == 2 # Require range [rmin, rmax]
rmag = np.zeros(spectra.num_spectra())
w, = np.where((spectra.fibermap['FLUX_R'] > 0)
& (spectra.fibermap['MW_TRANSMISSION_R'] > 0))
rmag[w] = -2.5 * np.log10(
spectra.fibermap['FLUX_R'][w] /
spectra.fibermap['MW_TRANSMISSION_R'][w]) + 22.5
w, = np.where((rmag > rmag_cut[0]) & (rmag < rmag_cut[1]))
if len(w) == 0:
if log is not None:
log.info(" * No spectra with r_mag in requested range")
return 0
else:
targetids = spectra.fibermap['TARGETID'][w]
spectra = myspecselect.myspecselect(spectra,
targets=targetids,
remove_scores=remove_scores)
# Chi2 selection
if chi2cut is not None:
assert len(chi2cut) == 2 # Require range [chi2min, chi2max]
if np.any(zbest['TARGETID'] != spectra.fibermap['TARGETID']):
raise RunTimeError(
'specviewer_selection : zbest and spectra do not match (different targetids)'
)
w, = np.where((zbest['DELTACHI2'] > chi2cut[0])
& (zbest['DELTACHI2'] < chi2cut[1]))
if len(w) == 0:
if log is not None:
log.info(
" * No target in this pixel with DeltaChi2 in requested range"
)
return 0
else:
targetids = spectra.fibermap['TARGETID'][w]
spectra = myspecselect.myspecselect(spectra,
targets=targetids,
remove_scores=remove_scores)
return spectra
p.add_argument('--tile', type=int, default=0,
help='Tile ID for processing.')
p.add_argument('--date', default=date.today().strftime('%Y%m%d'),
help='Date of observation [YYYYMMDD]')
p.add_argument('--tfmodel', default=None,
help='TensorFlow model HDF5 definition')
args = p.parse_args()
# Access redux folder.
redux='/global/project/projectdirs/desi/spectro/redux/daily/tiles'
prefix_in='/'.join([redux, '{:05d}'.format(args.tile), args.date])
if not os.path.isdir(prefix_in):
raise SystemExit('{} does not exist.'.format(prefix_in))
# Set up BGS target bit selection.
cmx_bgs_bits = '|'.join([_ for _ in cmx_mask.names() if 'BGS' in _])
# List zbest and coadd files.
zbfiles = sorted(glob('{}/zbest*.fits'.format(prefix_in)))
cafiles = sorted(glob('{}/coadd*.fits'.format(prefix_in)))
if args.tfmodel is not None:
classifier = keras.models.load_model(args.tfmodel)
# Loop through zbest and coadd files for each petal.
# Extract the fibermaps, ZBEST tables, and spectra.
# Keep only BGS targets passing basic event selection.
allzbest = None
allfmap = None
allwave = None
allflux = None