def make_mtl(targets, zcat=None, trim=False):
"""Adds NUMOBS, PRIORITY, and OBSCONDITIONS columns to a targets table.
Parameters
----------
targets : :class:`~numpy.array` or `~astropy.table.Table`
A numpy rec array or astropy Table with at least the columns
``TARGETID``, ``DESI_TARGET``, ``NUMOBS_INIT``, ``PRIORITY_INIT``.
or the corresponding columns for SV or commissioning.
zcat : :class:`~astropy.table.Table`, optional
Redshift catalog table with columns ``TARGETID``, ``NUMOBS``, ``Z``,
``ZWARN``.
trim : :class:`bool`, optional
If ``True`` (default), don't include targets that don't need
any more observations. If ``False``, include every input target.
Returns
-------
:class:`~astropy.table.Table`
MTL Table with targets columns plus:
* NUMOBS_MORE - number of additional observations requested
* PRIORITY - target priority (larger number = higher priority)
* OBSCONDITIONS - replaces old GRAYLAYER
"""
# ADM set up the default logger.
from desiutil.log import get_logger
log = get_logger()
# ADM determine whether the input targets are main survey, cmx or SV.
colnames, masks, survey = main_cmx_or_sv(targets)
# ADM set the first column to be the "desitarget" column
desi_target, desi_mask = colnames[0], masks[0]
# Trim targets from zcat that aren't in original targets table
if zcat is not None:
ok = np.in1d(zcat['TARGETID'], targets['TARGETID'])
num_extra = np.count_nonzero(~ok)
if num_extra > 0:
log.warning("Ignoring {} zcat entries that aren't "
"in the input target list".format(num_extra))
zcat = zcat[ok]
n = len(targets)
# ADM if the input target columns were incorrectly called NUMOBS or PRIORITY
# ADM rename them to NUMOBS_INIT or PRIORITY_INIT.
# ADM Note that the syntax is slightly different for a Table.
for name in ['NUMOBS', 'PRIORITY']:
if isinstance(targets, Table):
try:
targets.rename_column(name, name + '_INIT')
except KeyError:
pass
else:
targets.dtype.names = [
name + '_INIT' if col == name else col
for col in targets.dtype.names
]
# ADM if a redshift catalog was passed, order it to match the input targets
# ADM catalog on 'TARGETID'.
if zcat is not None:
# ADM there might be a quicker way to do this?
# ADM set up a dictionary of the indexes of each target id.
d = dict(tuple(zip(targets["TARGETID"], np.arange(n))))
# ADM loop through the zcat and look-up the index in the dictionary.
zmatcher = np.array([d[tid] for tid in zcat["TARGETID"]])
ztargets = zcat
if ztargets.masked:
unobs = ztargets['NUMOBS'].mask
ztargets['NUMOBS'][unobs] = 0
unobsz = ztargets['Z'].mask
ztargets['Z'][unobsz] = -1
unobszw = ztargets['ZWARN'].mask
ztargets['ZWARN'][unobszw] = -1
else:
ztargets = Table()
ztargets['TARGETID'] = targets['TARGETID']
ztargets['NUMOBS'] = np.zeros(n, dtype=np.int32)
ztargets['Z'] = -1 * np.ones(n, dtype=np.float32)
ztargets['ZWARN'] = -1 * np.ones(n, dtype=np.int32)
# ADM if zcat wasn't passed, there is a one-to-one correspondence
# ADM between the targets and the zcat.
zmatcher = np.arange(n)
# ADM extract just the targets that match the input zcat.
targets_zmatcher = targets[zmatcher]
# ADM use passed value of NUMOBS_INIT instead of calling the memory-heavy calc_numobs.
# ztargets['NUMOBS_MORE'] = np.maximum(0, calc_numobs(ztargets) - ztargets['NUMOBS'])
ztargets['NUMOBS_MORE'] = np.maximum(
0, targets_zmatcher['NUMOBS_INIT'] - ztargets['NUMOBS'])
# ADM we need a minor hack to ensure that BGS targets are observed once (and only once)
# ADM every time, regardless of how many times they've previously been observed.
# ADM I've turned this off for commissioning. Not sure if we'll keep it in general.
if survey != 'cmx':
ii = targets_zmatcher[desi_target] & desi_mask.BGS_ANY > 0
ztargets['NUMOBS_MORE'][ii] = 1
# ADM assign priorities, note that only things in the zcat can have changed priorities.
# ADM anything else will be assigned PRIORITY_INIT, below.
priority = calc_priority(targets_zmatcher, ztargets)
# If priority went to 0==DONOTOBSERVE or 1==OBS or 2==DONE, then NUMOBS_MORE should also be 0.
# ## mtl['NUMOBS_MORE'] = ztargets['NUMOBS_MORE']
ii = (priority <= 2)
log.info(
'{:d} of {:d} targets have priority zero, setting N_obs=0.'.format(
np.sum(ii), n))
ztargets['NUMOBS_MORE'][ii] = 0
# - Set the OBSCONDITIONS mask for each target bit.
obscon = set_obsconditions(targets)
# ADM set up the output mtl table.
mtl = Table(targets)
mtl.meta['EXTNAME'] = 'MTL'
# ADM any target that wasn't matched to the ZCAT should retain its
# ADM original (INIT) value of PRIORITY and NUMOBS.
mtl['NUMOBS_MORE'] = mtl['NUMOBS_INIT']
mtl['PRIORITY'] = mtl['PRIORITY_INIT']
# ADM now populate the new mtl columns with the updated information.
mtl['OBSCONDITIONS'] = obscon
mtl['PRIORITY'][zmatcher] = priority
mtl['NUMOBS_MORE'][zmatcher] = ztargets['NUMOBS_MORE']
# Filter out any targets marked as done.
if trim:
notdone = mtl['NUMOBS_MORE'] > 0
log.info('{:d} of {:d} targets are done, trimming these'.format(
len(mtl) - np.sum(notdone), len(mtl)))
mtl = mtl[notdone]
# Filtering can reset the fill_value, which is just wrong wrong wrong
# See https://github.com/astropy/astropy/issues/4707
# and https://github.com/astropy/astropy/issues/4708
mtl['NUMOBS_MORE'].fill_value = -1
return mtl
def make_mtl(targets, obscon, zcat=None, trim=False, scnd=None):
"""Adds NUMOBS, PRIORITY, and OBSCONDITIONS columns to a targets table.
Parameters
----------
targets : :class:`~numpy.array` or `~astropy.table.Table`
A numpy rec array or astropy Table with at least the columns
``TARGETID``, ``DESI_TARGET``, ``NUMOBS_INIT``, ``PRIORITY_INIT``.
or the corresponding columns for SV or commissioning.
obscon : :class:`str`
A combination of strings that are in the desitarget bitmask yaml
file (specifically in `desitarget.targetmask.obsconditions`), e.g.
"DARK|GRAY". Governs the behavior of how priorities are set based
on "obsconditions" in the desitarget bitmask yaml file.
zcat : :class:`~astropy.table.Table`, optional
Redshift catalog table with columns ``TARGETID``, ``NUMOBS``, ``Z``,
``ZWARN``.
trim : :class:`bool`, optional
If ``True`` (default), don't include targets that don't need
any more observations. If ``False``, include every input target.
scnd : :class:`~numpy.array`, `~astropy.table.Table`, optional
A set of secondary targets associated with the `targets`. As with
the `target` must include at least ``TARGETID``, ``NUMOBS_INIT``,
``PRIORITY_INIT`` or the corresponding SV columns.
The secondary targets will be padded to have the same columns
as the targets, and concatenated with them.
Returns
-------
:class:`~astropy.table.Table`
MTL Table with targets columns plus:
* NUMOBS_MORE - number of additional observations requested
* PRIORITY - target priority (larger number = higher priority)
* OBSCONDITIONS - replaces old GRAYLAYER
"""
start = time()
# ADM set up the default logger.
from desiutil.log import get_logger
log = get_logger()
# ADM if secondaries were passed, concatenate them with the targets.
if scnd is not None:
nrows = len(scnd)
log.info(
'Pad {} primary targets with {} secondaries...t={:.1f}s'.format(
len(targets), nrows,
time() - start))
padit = np.zeros(nrows, dtype=targets.dtype)
sharedcols = set(targets.dtype.names).intersection(
set(scnd.dtype.names))
for col in sharedcols:
padit[col] = scnd[col]
targets = np.concatenate([targets, padit])
# APC Propagate a flag on which targets came from scnd
is_scnd = np.repeat(False, len(targets))
is_scnd[-nrows:] = True
log.info('Done with padding...t={:.1f}s'.format(time() - start))
# ADM determine whether the input targets are main survey, cmx or SV.
colnames, masks, survey = main_cmx_or_sv(targets)
# ADM set the first column to be the "desitarget" column
desi_target, desi_mask = colnames[0], masks[0]
# Trim targets from zcat that aren't in original targets table
if zcat is not None:
ok = np.in1d(zcat['TARGETID'], targets['TARGETID'])
num_extra = np.count_nonzero(~ok)
if num_extra > 0:
log.warning("Ignoring {} zcat entries that aren't "
"in the input target list".format(num_extra))
zcat = zcat[ok]
n = len(targets)
# ADM if the input target columns were incorrectly called NUMOBS or PRIORITY
# ADM rename them to NUMOBS_INIT or PRIORITY_INIT.
# ADM Note that the syntax is slightly different for a Table.
for name in ['NUMOBS', 'PRIORITY']:
if isinstance(targets, Table):
try:
targets.rename_column(name, name + '_INIT')
except KeyError:
pass
else:
targets.dtype.names = [
name + '_INIT' if col == name else col
for col in targets.dtype.names
]
# ADM if a redshift catalog was passed, order it to match the input targets
# ADM catalog on 'TARGETID'.
if zcat is not None:
# ADM there might be a quicker way to do this?
# ADM set up a dictionary of the indexes of each target id.
d = dict(tuple(zip(targets["TARGETID"], np.arange(n))))
# ADM loop through the zcat and look-up the index in the dictionary.
zmatcher = np.array([d[tid] for tid in zcat["TARGETID"]])
ztargets = zcat
if ztargets.masked:
unobs = ztargets['NUMOBS'].mask
ztargets['NUMOBS'][unobs] = 0
unobsz = ztargets['Z'].mask
ztargets['Z'][unobsz] = -1
unobszw = ztargets['ZWARN'].mask
ztargets['ZWARN'][unobszw] = -1
else:
ztargets = Table()
ztargets['TARGETID'] = targets['TARGETID']
ztargets['NUMOBS'] = np.zeros(n, dtype=np.int32)
ztargets['Z'] = -1 * np.ones(n, dtype=np.float32)
ztargets['ZWARN'] = -1 * np.ones(n, dtype=np.int32)
# ADM if zcat wasn't passed, there is a one-to-one correspondence
# ADM between the targets and the zcat.
zmatcher = np.arange(n)
# ADM extract just the targets that match the input zcat.
targets_zmatcher = targets[zmatcher]
# ADM use passed value of NUMOBS_INIT instead of calling the memory-heavy calc_numobs.
# ztargets['NUMOBS_MORE'] = np.maximum(0, calc_numobs(ztargets) - ztargets['NUMOBS'])
ztargets['NUMOBS_MORE'] = np.maximum(
0, targets_zmatcher['NUMOBS_INIT'] - ztargets['NUMOBS'])
# ADM need a minor hack to ensure BGS targets are observed once
# ADM (and only once) every time during the BRIGHT survey, regardless
# ADM of how often they've previously been observed. I've turned this
# ADM off for commissioning. Not sure if we'll keep it in general.
if survey != 'cmx':
# ADM only if we're considering bright survey conditions.
if (obsconditions.mask(obscon) & obsconditions.mask("BRIGHT")) != 0:
ii = targets_zmatcher[desi_target] & desi_mask.BGS_ANY > 0
ztargets['NUMOBS_MORE'][ii] = 1
if survey == 'main':
# If the object is confirmed to be a tracer QSO, then don't request more observations
if (obsconditions.mask(obscon) & obsconditions.mask("DARK")) != 0:
if zcat is not None:
ii = ztargets['SPECTYPE'] == 'QSO'
ii &= (ztargets['ZWARN'] == 0)
ii &= (ztargets['Z'] < 2.1)
ii &= (ztargets['NUMOBS'] > 0)
ztargets['NUMOBS_MORE'][ii] = 0
# ADM assign priorities, note that only things in the zcat can have changed priorities.
# ADM anything else will be assigned PRIORITY_INIT, below.
priority = calc_priority(targets_zmatcher, ztargets, obscon)
# If priority went to 0==DONOTOBSERVE or 1==OBS or 2==DONE, then NUMOBS_MORE should also be 0.
# ## mtl['NUMOBS_MORE'] = ztargets['NUMOBS_MORE']
ii = (priority <= 2)
log.info(
'{:d} of {:d} targets have priority zero, setting N_obs=0.'.format(
np.sum(ii), n))
ztargets['NUMOBS_MORE'][ii] = 0
# - Set the OBSCONDITIONS mask for each target bit.
obsconmask = set_obsconditions(targets)
# APC obsconmask will now be incorrect for secondary-only targets. Fix this
# APC using the mask on secondary targets.
if scnd is not None:
obsconmask[is_scnd] = set_obsconditions(targets[is_scnd], scnd=True)
# ADM set up the output mtl table.
mtl = Table(targets)
mtl.meta['EXTNAME'] = 'MTL'
# ADM any target that wasn't matched to the ZCAT should retain its
# ADM original (INIT) value of PRIORITY and NUMOBS.
mtl['NUMOBS_MORE'] = mtl['NUMOBS_INIT']
mtl['PRIORITY'] = mtl['PRIORITY_INIT']
# ADM now populate the new mtl columns with the updated information.
mtl['OBSCONDITIONS'] = obsconmask
mtl['PRIORITY'][zmatcher] = priority
mtl['NUMOBS_MORE'][zmatcher] = ztargets['NUMOBS_MORE']
# Filter out any targets marked as done.
if trim:
notdone = mtl['NUMOBS_MORE'] > 0
log.info('{:d} of {:d} targets are done, trimming these'.format(
len(mtl) - np.sum(notdone), len(mtl)))
mtl = mtl[notdone]
# Filtering can reset the fill_value, which is just wrong wrong wrong
# See https://github.com/astropy/astropy/issues/4707
# and https://github.com/astropy/astropy/issues/4708
mtl['NUMOBS_MORE'].fill_value = -1
log.info('Done...t={:.1f}s'.format(time() - start))
return mtl
def finalize_secondary(scxtargs,
scnd_mask,
survey='main',
sep=1.,
darkbright=False):
"""Assign secondary targets a realistic TARGETID, finalize columns.
Parameters
----------
scxtargs : :class:`~numpy.ndarray`
An array of secondary targets, must contain the columns `RA`,
`DEC` and `TARGETID`. `TARGETID` should be -1 for objects
that lack a `TARGETID`.
scnd_mask : :class:`desiutil.bitmask.BitMask`
A mask corresponding to a set of secondary targets, e.g, could
be ``from desitarget.targetmask import scnd_mask`` for the
main survey mask.
survey : :class:`str`, optional, defaults to "main"
string indicating whether we are working in the context of the
Main Survey (`main`) or SV (e.g. `sv1`, `sv2` etc.). Used to
set the `RELEASE` number in the `TARGETID` (see Notes).
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match secondary targets to
themselves in ARCSECONDS.
darkbright : :class:`bool`, optional, defaults to ``False``
If sent, then split `NUMOBS_INIT` and `PRIORITY_INIT` into
`NUMOBS_INIT_DARK`, `NUMOBS_INIT_BRIGHT`, `PRIORITY_INIT_DARK`
and `PRIORITY_INIT_BRIGHT` and calculate values appropriate
to "BRIGHT" and "DARK|GRAY" observing conditions.
Returns
-------
:class:`~numpy.ndarray`
The array of secondary targets, with the `TARGETID` bit updated
to be unique and reasonable and the `SCND_TARGET` column renamed
based on the flavor of `scnd_mask`.
Notes
-----
- Secondaries without `OVERRIDE` are also matched to themselves
Such matches are given the same `TARGETID` (that of the primary
if they match a primary) and the bitwise or of `SCND_TARGET` and
`OBSCONDITIONS` bits across matches. The highest `PRIORITY_INIT`
is retained, and others are set to -1. Only secondaries with
priorities that are not -1 are written to the main file. If
multiple matching secondary targets have the same (highest)
priority, the first one encountered retains its `PRIORITY_INIT`
- The secondary `TARGETID` is designed to be reproducible. It
combines `BRICKID` based on location, `OBJID` based on the
order of the targets in the secondary file (`SCND_ORDER`) and
`RELEASE` from the secondary bit number (`SCND_TARGET`) and the
input `survey`. `RELEASE` is set to ((X-1)*100)+np.log2(scnd_bit)
with X from the `survey` string survey=svX and scnd_bit from
`SCND_TARGET`. For the main survey (survey="main") X-1 is 5.
"""
# ADM assign new TARGETIDs to targets without a primary match.
nomatch = scxtargs["TARGETID"] == -1
# ADM get the BRICKIDs for each source.
brxid = bricks.brickid(scxtargs["RA"], scxtargs["DEC"])
# ADM ensure unique secondary bits for different iterations of SV
# ADM and the Main Survey.
if survey == 'main':
Xm1 = 5
elif survey[0:2] == 'sv':
# ADM the re.search just extracts the numbers in the string.
Xm1 = int(re.search(r'\d+', survey).group()) - 1
# ADM we've allowed a max of up to sv5 (!). Fail if surpassed.
if Xm1 >= 5:
msg = "Only coded for up to 'sv5', not {}!!!".format(survey)
log.critical(msg)
raise ValueError(msg)
else:
msg = "allowed surveys: 'main', 'svX', not {}!!!".format(survey)
log.critical(msg)
raise ValueError(msg)
# ADM the RELEASE for each source is the `SCND_TARGET` bit NUMBER.
release = (Xm1 * 100) + np.log2(scxtargs["SCND_TARGET_INIT"]).astype('int')
# ADM build the OBJIDs based on the values of SCND_ORDER.
t0 = time()
log.info("Begin assigning OBJIDs to bricks...")
# ADM So as not to overwhelm the bit-limits for OBJID
# ADM rank by SCND_ORDER for each brick and bit combination.
# ADM First, create a unique ID based on brxid and release.
scnd_order = scxtargs["SCND_ORDER"]
sorter = (1000 * brxid) + release
# ADM sort the unique IDs and split based on where they change.
argsort = np.argsort(sorter)
w = np.where(np.diff(sorter[argsort]))[0]
soperbrxbit = np.split(scnd_order[argsort], w + 1)
# ADM loop through each (brxid, release) and sort on scnd_order.
# ADM double argsort returns the ascending ranked order of the entry
# ADM (whereas a single argsort returns the indexes for ordering).
sortperbrxbit = [np.argsort(np.argsort(so)) for so in soperbrxbit]
# ADM finally unroll the (brxid, release) combinations...
sortedobjid = np.array(list(itertools.chain.from_iterable(sortperbrxbit)))
# ADM ...and reorder based on the initial argsort.
objid = np.zeros_like(sortedobjid) - 1
objid[argsort] = sortedobjid
log.info("Assigned OBJIDs to bricks in {:.1f}s".format(time() - t0))
# ADM check that the objid array was entirely populated.
assert np.all(objid != -1)
# ADM assemble the TARGETID, SCND objects have RELEASE==0.
targetid = encode_targetid(objid=objid, brickid=brxid, release=release)
# ADM a check that the generated TARGETIDs are unique.
if len(set(targetid)) != len(targetid):
msg = "duplicate TARGETIDs generated for secondary targets!!!"
log.critical(msg)
raise ValueError(msg)
# ADM assign the unique TARGETIDs to the secondary objects.
scxtargs["TARGETID"][nomatch] = targetid[nomatch]
log.debug("Assigned {} targetids to unmatched secondaries".format(
len(targetid[nomatch])))
# ADM match secondaries to themselves, to ensure duplicates
# ADM share a TARGETID. Don't match special (OVERRIDE) targets
# ADM or sources that have already been matched to a primary.
w = np.where(~scxtargs["OVERRIDE"] & nomatch)[0]
if len(w) > 0:
log.info("Matching secondary targets to themselves...t={:.1f}s".format(
time() - t0))
# ADM use astropy for the matching. At NERSC, astropy matches
# ADM ~20M objects to themselves in about 10 minutes.
c = SkyCoord(scxtargs["RA"][w] * u.deg, scxtargs["DEC"][w] * u.deg)
m1, m2, _, _ = c.search_around_sky(c, sep * u.arcsec)
log.info("Done with matching...t={:.1f}s".format(time() - t0))
# ADM restrict only to unique matches (and exclude self-matches).
uniq = m1 > m2
m1, m2 = m1[uniq], m2[uniq]
# ADM set same TARGETID for any matches. m2 must come first, here.
scxtargs["TARGETID"][w[m2]] = scxtargs["TARGETID"][w[m1]]
# ADM Ensure secondary targets with matching TARGETIDs have all the
# ADM relevant SCND_TARGET bits set. By definition, targets with
# ADM OVERRIDE set never have matching TARGETIDs.
wnoov = np.where(~scxtargs["OVERRIDE"])[0]
if len(wnoov) > 0:
for _, inds in duplicates(scxtargs["TARGETID"][wnoov]):
scnd_targ = 0
for ind in inds:
scnd_targ |= scxtargs["SCND_TARGET"][wnoov[ind]]
scxtargs["SCND_TARGET"][wnoov[inds]] = scnd_targ
log.info("Done checking SCND_TARGET...t={:.1f}s".format(time() - t0))
# ADM change the data model depending on whether the mask
# ADM is an SVX (X = 1, 2, etc.) mask or not. Nothing will
# ADM change if the mask has no preamble.
prepend = scnd_mask._name[:-9].upper()
scxtargs = rfn.rename_fields(scxtargs,
{'SCND_TARGET': prepend + 'SCND_TARGET'})
# APC same thing for DESI_TARGET
scxtargs = rfn.rename_fields(scxtargs,
{'DESI_TARGET': prepend + 'DESI_TARGET'})
# APC Remove duplicate targetids from secondary-only targets
alldups = []
for _, dups in duplicates(scxtargs['TARGETID']):
# Retain the duplicate with highest priority, breaking ties
# on lowest index in list of duplicates
dups = np.delete(dups, np.argmax(scxtargs['PRIORITY_INIT'][dups]))
alldups.append(dups)
alldups = np.hstack(alldups)
log.debug(
"Flagging {} duplicate secondary targetids with PRIORITY_INIT=-1".
format(len(alldups)))
# ADM and remove the INIT fields in prep for a dark/bright split.
scxtargs = rfn.drop_fields(scxtargs, ["PRIORITY_INIT", "NUMOBS_INIT"])
# ADM set initial priorities, numobs and obsconditions for both
# ADM BRIGHT and DARK|GRAY conditions, if requested.
nscx = len(scxtargs)
nodata = np.zeros(nscx, dtype='int') - 1
if darkbright:
ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"]
else:
ender, obscon = [""], ["DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18"]
cols, vals, forms = [], [], []
for edr, oc in zip(ender, obscon):
cols += ["{}_INIT{}".format(pn, edr) for pn in ["PRIORITY", "NUMOBS"]]
vals += [nodata, nodata]
forms += ['>i8', '>i8']
# ADM write the output array.
newdt = [dt for dt in zip(cols, forms)]
done = np.array(np.zeros(nscx), dtype=scxtargs.dtype.descr + newdt)
for col in scxtargs.dtype.names:
done[col] = scxtargs[col]
for col, val in zip(cols, vals):
done[col] = val
# ADM add the actual PRIORITY/NUMOBS values.
for edr, oc in zip(ender, obscon):
pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr
done[pc], done[nc] = initial_priority_numobs(done,
obscon=oc,
scnd=True)
# APC Flagged duplicates are removed in io.write_secondary
done[pc][alldups] = -1
# APC add secondary flag in DESI_TARGET
cols, mx, surv = main_cmx_or_sv(done, scnd=True)
done[cols[0]] = mx[0]['SCND_ANY']
# ADM set the OBSCONDITIONS.
done["OBSCONDITIONS"] = set_obsconditions(done, scnd=True)
return done
def make_mtl(targets, obscon, zcat=None, scnd=None,
trim=False, trimcols=False, trimtozcat=False):
"""Adds fiberassign and zcat columns to a targets table.
Parameters
----------
targets : :class:`~numpy.array` or `~astropy.table.Table`
A numpy rec array or astropy Table with at least the columns
``TARGETID``, ``DESI_TARGET``, ``NUMOBS_INIT``, ``PRIORITY_INIT``.
or the corresponding columns for SV or commissioning.
obscon : :class:`str`
A combination of strings that are in the desitarget bitmask yaml
file (specifically in `desitarget.targetmask.obsconditions`), e.g.
"DARK|GRAY". Governs the behavior of how priorities are set based
on "obsconditions" in the desitarget bitmask yaml file.
zcat : :class:`~astropy.table.Table`, optional
Redshift catalog table with columns ``TARGETID``, ``NUMOBS``, ``Z``,
``ZWARN``.
scnd : :class:`~numpy.array`, `~astropy.table.Table`, optional
A set of secondary targets associated with the `targets`. As with
the `target` must include at least ``TARGETID``, ``NUMOBS_INIT``,
``PRIORITY_INIT`` or the corresponding SV columns.
The secondary targets will be padded to have the same columns
as the targets, and concatenated with them.
trim : :class:`bool`, optional
If ``True`` (default), don't include targets that don't need
any more observations. If ``False``, include every input target.
trimcols : :class:`bool`, optional, defaults to ``False``
Only pass through columns in `targets` that are actually needed
for fiberassign (see `desitarget.mtl.mtldatamodel`).
trimtozcat : :class:`bool`, optional, defaults to ``False``
Only return targets that have been UPDATED (i.e. the targets with
a match in `zcat`). Returns all targets if `zcat` is ``None``.
Returns
-------
:class:`~astropy.table.Table`
MTL Table with targets columns plus:
* NUMOBS_MORE - number of additional observations requested
* PRIORITY - target priority (larger number = higher priority)
* TARGET_STATE - the observing state that corresponds to PRIORITY
* OBSCONDITIONS - replaces old GRAYLAYER
* TIMESTAMP - time that (this) make_mtl() function was run
* VERSION - version of desitarget used to run make_mtl()
"""
start = time()
# ADM set up the default logger.
from desiutil.log import get_logger
log = get_logger()
# ADM if trimcols was passed, reduce input target columns to minimal.
if trimcols:
mtldm = switch_main_cmx_or_sv(mtldatamodel, targets)
cullcols = list(set(targets.dtype.names) - set(mtldm.dtype.names))
if isinstance(targets, Table):
targets.remove_columns(cullcols)
else:
targets = rfn.drop_fields(targets, cullcols)
# ADM determine whether the input targets are main survey, cmx or SV.
colnames, masks, survey = main_cmx_or_sv(targets, scnd=True)
# ADM set the first column to be the "desitarget" column
desi_target, desi_mask = colnames[0], masks[0]
scnd_target = colnames[-1]
# ADM if secondaries were passed, concatenate them with the targets.
if scnd is not None:
nrows = len(scnd)
log.info('Pad {} primary targets with {} secondaries...t={:.1f}s'.format(
len(targets), nrows, time()-start))
padit = np.zeros(nrows, dtype=targets.dtype)
sharedcols = set(targets.dtype.names).intersection(set(scnd.dtype.names))
for col in sharedcols:
padit[col] = scnd[col]
targets = np.concatenate([targets, padit])
# APC Propagate a flag on which targets came from scnd
is_scnd = np.repeat(False, len(targets))
is_scnd[-nrows:] = True
log.info('Done with padding...t={:.1f}s'.format(time()-start))
# Trim targets from zcat that aren't in original targets table.
if zcat is not None:
ok = np.in1d(zcat['TARGETID'], targets['TARGETID'])
num_extra = np.count_nonzero(~ok)
if num_extra > 0:
log.warning("Ignoring {} zcat entries that aren't "
"in the input target list".format(num_extra))
zcat = zcat[ok]
n = len(targets)
# ADM if a redshift catalog was passed, order it to match the input targets
# ADM catalog on 'TARGETID'.
if zcat is not None:
# ADM there might be a quicker way to do this?
# ADM set up a dictionary of the indexes of each target id.
d = dict(tuple(zip(targets["TARGETID"], np.arange(n))))
# ADM loop through the zcat and look-up the index in the dictionary.
zmatcher = np.array([d[tid] for tid in zcat["TARGETID"]])
ztargets = zcat
if ztargets.masked:
unobs = ztargets['NUMOBS'].mask
ztargets['NUMOBS'][unobs] = 0
unobsz = ztargets['Z'].mask
ztargets['Z'][unobsz] = -1
unobszw = ztargets['ZWARN'].mask
ztargets['ZWARN'][unobszw] = -1
else:
ztargets = Table()
ztargets['TARGETID'] = targets['TARGETID']
ztargets['NUMOBS'] = np.zeros(n, dtype=np.int32)
ztargets['Z'] = -1 * np.ones(n, dtype=np.float32)
ztargets['ZWARN'] = -1 * np.ones(n, dtype=np.int32)
# ADM if zcat wasn't passed, there is a one-to-one correspondence
# ADM between the targets and the zcat.
zmatcher = np.arange(n)
# ADM extract just the targets that match the input zcat.
targets_zmatcher = targets[zmatcher]
# ADM update the number of observations for the targets.
ztargets['NUMOBS_MORE'] = calc_numobs_more(targets_zmatcher, ztargets, obscon)
# ADM assign priorities. Only things in the zcat can have changed
# ADM priorities. Anything else is assigned PRIORITY_INIT, below.
priority, target_state = calc_priority(
targets_zmatcher, ztargets, obscon, state=True)
# If priority went to 0==DONOTOBSERVE or 1==OBS or 2==DONE, then
# NUMOBS_MORE should also be 0.
# ## mtl['NUMOBS_MORE'] = ztargets['NUMOBS_MORE']
ii = (priority <= 2)
log.info('{:d} of {:d} targets have priority zero, setting N_obs=0.'.format(
np.sum(ii), n))
ztargets['NUMOBS_MORE'][ii] = 0
# - Set the OBSCONDITIONS mask for each target bit.
obsconmask = set_obsconditions(targets)
# APC obsconmask will now be incorrect for secondary-only targets. Fix this
# APC using the mask on secondary targets.
if scnd is not None:
obsconmask[is_scnd] = set_obsconditions(targets[is_scnd], scnd=True)
# ADM set up the output mtl table.
mtl = Table(targets)
mtl.meta['EXTNAME'] = 'MTL'
# ADM add a placeholder for the secondary bit-mask, if it isn't there.
if scnd_target not in mtl.dtype.names:
mtl[scnd_target] = np.zeros(len(mtl),
dtype=mtldatamodel["SCND_TARGET"].dtype)
# ADM initialize columns to avoid zero-length/missing/format errors.
zcols = ["NUMOBS_MORE", "NUMOBS", "Z", "ZWARN"]
for col in zcols + ["TARGET_STATE", "TIMESTAMP", "VERSION"]:
mtl[col] = np.empty(len(mtl), dtype=mtldatamodel[col].dtype)
# ADM any target that wasn't matched to the ZCAT should retain its
# ADM original (INIT) value of PRIORITY and NUMOBS.
mtl['NUMOBS_MORE'] = mtl['NUMOBS_INIT']
mtl['PRIORITY'] = mtl['PRIORITY_INIT']
mtl['TARGET_STATE'] = "UNOBS"
# ADM add the time and version of the desitarget code that was run.
utc = datetime.utcnow().isoformat(timespec='seconds')
mtl["TIMESTAMP"] = utc
mtl["VERSION"] = dt_version
# ADM now populate the new mtl columns with the updated information.
mtl['OBSCONDITIONS'] = obsconmask
mtl['PRIORITY'][zmatcher] = priority
mtl['TARGET_STATE'][zmatcher] = target_state
for col in zcols:
mtl[col][zmatcher] = ztargets[col]
# Filter out any targets marked as done.
if trim:
notdone = mtl['NUMOBS_MORE'] > 0
log.info('{:d} of {:d} targets are done, trimming these'.format(
len(mtl) - np.sum(notdone), len(mtl))
)
mtl = mtl[notdone]
# Filtering can reset the fill_value, which is just wrong wrong wrong
# See https://github.com/astropy/astropy/issues/4707
# and https://github.com/astropy/astropy/issues/4708
mtl['NUMOBS_MORE'].fill_value = -1
# ADM assert the data model is complete.
# ADM turning this off for now, useful for testing.
# mtltypes = [mtl[i].dtype.type for i in mtl.dtype.names]
# mtldmtypes = [mtldm[i].dtype.type for i in mtl.dtype.names]
# assert set(mtl.dtype.names) == set(mtldm.dtype.names)
# assert mtltypes == mtldmtypes
log.info('Done...t={:.1f}s'.format(time()-start))
if trimtozcat:
return mtl[zmatcher]
return mtl
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
primtargs : :class:`~numpy.ndarray`
An array of primary targets.
scndout : :class`~numpy.ndarray`
Name of a sub-directory to which to write the information in
`desitarget.secondary.outdatamodel` with `TARGETID` and (the
highest) `PRIORITY_INIT` updated with matching primary info.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
pix : :class:`list`, optional, defaults to `None`
Limit secondary targets to (NESTED) HEALpixels that touch
pix at the supplied `nside`, as a speed-up.
nside : :class:`int`, optional, defaults to `None`
The (NESTED) HEALPixel nside to be used with `pixlist`.
Returns
-------
:class:`~numpy.ndarray`
The array of primary targets, with the `SCND_TARGET` bit
populated for matches to secondary targets
"""
# ADM add a SCND_TARGET column to the primary targets.
dt = primtargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(primtargs), dtype=dt)
for col in primtargs.dtype.names:
targs[col] = primtargs[col]
# ADM check if this is an SV or main survey file.
cols, mx, surv = main_cmx_or_sv(targs, scnd=True)
log.info('running on the {} survey...'.format(surv))
if surv != 'main':
scxdir = os.path.join(scxdir, surv)
# ADM read in non-OVERRIDE secondary targets.
scxtargs = read_files(scxdir, mx[3])
scxtargs = scxtargs[~scxtargs["OVERRIDE"]]
# ADM match primary targets to non-OVERRIDE secondary targets.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if nside is not None and pix is not None:
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(primtargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(primtargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(scndout, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
desi_mask = desimasks[0]
targs[desicols[0]][umtargs] |= desi_mask.SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# APC Secondary target bits only affect PRIORITY, NUMOBS and
# APC obsconditions for specific DESI_TARGET bits
# APC See https://github.com/desihub/desitarget/pull/530
# APC Only consider primary targets with secondary bits set
scnd_update = (targs[desicols[0]] & desi_mask['SCND_ANY']) != 0
if np.any(scnd_update):
# APC Allow changes to primaries if the DESI_TARGET bitmask has
# APC only the following bits set, in any combination.
log.info(
'Testing if secondary targets can update {} matched primaries'.
format(scnd_update.sum()))
update_from_scnd_bits = desi_mask['SCND_ANY'] | desi_mask[
'MWS_ANY'] | desi_mask['STD_BRIGHT'] | desi_mask[
'STD_FAINT'] | desi_mask['STD_WD']
scnd_update &= ((targs[desicols[0]] & ~update_from_scnd_bits) == 0)
log.info(
'Setting new priority, numobs and obsconditions from secondary for {} matched primaries'
.format(scnd_update.sum()))
# APC Primary and secondary obsconditions are or'd
scnd_obscon = set_obsconditions(targs[scnd_update], scnd=True)
targs['OBSCONDITIONS'][scnd_update] &= scnd_obscon
# APC bit of a hack here
# APC Check for _BRIGHT, _DARK split in column names
darkbright = 'NUMOBS_INIT_DARK' in targs.dtype.names
if darkbright:
ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"]
else:
ender, obscon = [""], [
"DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18"
]
# APC secondaries can increase priority and numobs
for edr, oc in zip(ender, obscon):
pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr
scnd_priority, scnd_numobs = initial_priority_numobs(
targs[scnd_update], obscon=oc, scnd=True)
targs[nc][scnd_update] = np.maximum(targs[nc][scnd_update],
scnd_numobs)
targs[pc][scnd_update] = np.maximum(targs[pc][scnd_update],
scnd_priority)
# ADM update the secondary targets with the primary information.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM the maximum priority will be used to break ties in the
# ADM unlikely event that a secondary matches two primaries.
hipri = np.maximum(targs["PRIORITY_INIT_DARK"],
targs["PRIORITY_INIT_BRIGHT"])
scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs]
# ADM write the secondary targets that have updated TARGETIDs.
ii = scxtargs["TARGETID"] != -1
nmatches = np.sum(ii)
log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format(
nmatches, scndout,
time() - start))
if nmatches > 0:
hdr = fitsio.FITSHDR()
hdr["SURVEY"] = surv
fitsio.write(scndout,
scxtargs[ii],
extname='SCND_TARG',
header=hdr,
clobber=True)
log.info('Done...t={:.1f}s'.format(time() - start))
return targs
