def setUp(self):
self.targets = Table()
self.types = np.array(['ELG', 'LRG', 'QSO', 'QSO', 'ELG'])
self.priorities = [Mx[t].priorities['UNOBS'] for t in self.types]
self.post_prio = [Mx[t].priorities['MORE_ZGOOD'] for t in self.types]
self.post_prio[0] = 2 # ELG
self.post_prio[1] = 2 # LRG...all one-pass
self.post_prio[2] = 2 # lowz QSO
nt = len(self.types)
# ADM add some "extra" columns that are needed for observations.
for col in ["RA", "DEC", "PARALLAX", "PMRA", "PMDEC", "REF_EPOCH"]:
self.targets[col] = np.zeros(nt, dtype=mtldatamodel[col].dtype)
self.targets['DESI_TARGET'] = [Mx[t].mask for t in self.types]
for col in ['BGS_TARGET', 'MWS_TARGET', 'SUBPRIORITY']:
self.targets[col] = np.zeros(nt, dtype=mtldatamodel[col].dtype)
n = len(self.targets)
self.targets['ZFLUX'] = 10**((22.5 - np.linspace(20, 22, n)) / 2.5)
self.targets['TARGETID'] = list(range(n))
# ADM determine the initial PRIORITY and NUMOBS.
pinit, ninit = initial_priority_numobs(self.targets,
obscon="DARK|GRAY")
self.targets["PRIORITY_INIT"] = pinit
self.targets["NUMOBS_INIT"] = ninit
# - reverse the order for zcat to make sure joins work
self.zcat = Table()
self.zcat['TARGETID'] = self.targets['TARGETID'][-2::-1]
self.zcat['Z'] = [2.5, 1.0, 0.5, 1.0]
self.zcat['ZWARN'] = [0, 0, 0, 0]
self.zcat['NUMOBS'] = [1, 1, 1, 1]
self.zcat['SPECTYPE'] = ['QSO', 'QSO', 'GALAXY', 'GALAXY']
def setUp(self):
self.targets = Table()
self.types = np.array(['ELG', 'LRG', 'QSO', 'QSO', 'ELG'])
self.priorities = [Mx[t].priorities['UNOBS'] for t in self.types]
self.post_prio = [Mx[t].priorities['MORE_ZGOOD'] for t in self.types]
self.post_prio[0] = 2 # ELG
self.post_prio[1] = 2 # LRG...all one-pass
self.post_prio[2] = 2 # lowz QSO
self.targets['DESI_TARGET'] = [Mx[t].mask for t in self.types]
self.targets['BGS_TARGET'] = np.zeros(len(self.types), dtype=np.int64)
self.targets['MWS_TARGET'] = np.zeros(len(self.types), dtype=np.int64)
n = len(self.targets)
self.targets['ZFLUX'] = 10**((22.5 - np.linspace(20, 22, n)) / 2.5)
self.targets['TARGETID'] = list(range(n))
# ADM determine the initial PRIORITY and NUMOBS.
pinit, ninit = initial_priority_numobs(self.targets)
self.targets["PRIORITY_INIT"] = pinit
self.targets["NUMOBS_INIT"] = ninit
# - reverse the order for zcat to make sure joins work
self.zcat = Table()
self.zcat['TARGETID'] = self.targets['TARGETID'][-2::-1]
self.zcat['Z'] = [2.5, 1.0, 0.5, 1.0]
self.zcat['ZWARN'] = [0, 0, 0, 0]
self.zcat['NUMOBS'] = [1, 1, 1, 1]
self.zcat['SPECTYPE'] = ['QSO', 'QSO', 'GALAXY', 'GALAXY']
def test_cmx_priorities(self):
"""Test that priority calculation can handle commissioning files.
"""
t = self.targets.copy()
z = self.zcat
# ADM restructure the table to look like a commissioning table.
t.rename_column('DESI_TARGET', 'CMX_TARGET')
t.remove_column('BGS_TARGET')
t.remove_column('MWS_TARGET')
# - No targeting bits set is priority=0
self.assertTrue(np.all(calc_priority(t, z, "GRAY|DARK") == 0))
# ADM retrieve the cmx_mask.
colnames, masks, _ = main_cmx_or_sv(t)
cmx_mask = masks[0]
# ADM test handling of unobserved SV0_BGS and SV0_MWS
for name, obscon in [("SV0_BGS", "BRIGHT"), ("SV0_MWS", "POOR")]:
t['CMX_TARGET'] = cmx_mask[name]
self.assertTrue(
np.all(
calc_priority(t, z, obscon) ==
cmx_mask[name].priorities['UNOBS']))
# ADM done is Done, regardless of ZWARN.
for name, obscon in [("SV0_BGS", "BRIGHT"), ("SV0_MWS", "POOR")]:
t['CMX_TARGET'] = cmx_mask[name]
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
# APC: Use NUMOBS_INIT here to avoid hardcoding NOBS corresponding to "done".
numobs_done = t['NUMOBS_INIT'][0]
z['NUMOBS'] = [0, numobs_done, numobs_done]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, obscon, zcat=z)["PRIORITY"]
self.assertEqual(p[0], cmx_mask[name].priorities['UNOBS'])
self.assertEqual(p[1], cmx_mask[name].priorities['DONE'])
self.assertEqual(p[2], cmx_mask[name].priorities['DONE'])
# BGS ZGOOD targets always have lower priority than MWS targets that
# are not DONE.
lowest_bgs_priority_zgood = cmx_mask['SV0_BGS'].priorities[
'MORE_ZGOOD']
lowest_mws_priority_unobs = cmx_mask['SV0_MWS'].priorities['UNOBS']
lowest_mws_priority_zwarn = cmx_mask['SV0_MWS'].priorities[
'MORE_ZWARN']
lowest_mws_priority_zgood = cmx_mask['SV0_MWS'].priorities[
'MORE_ZGOOD']
lowest_mws_priority = min(lowest_mws_priority_unobs,
lowest_mws_priority_zwarn,
lowest_mws_priority_zgood)
self.assertLess(lowest_bgs_priority_zgood, lowest_mws_priority)
def setUp(self):
self.targets = Table()
# This is a dual identity case. In all cases the target is both QSO and ELG.
# The first case is a true QSO with lowz.
# The second case is a true QSO with highz.
# The third case is an ELG.
self.type_A = np.array(['QSO', 'QSO', 'ELG'])
self.type_B = np.array(['ELG', 'ELG', 'QSO'])
self.priorities_A = np.array(
[Mx[t].priorities['UNOBS'] for t in self.type_A])
self.priorities_B = np.array(
[Mx[t].priorities['UNOBS'] for t in self.type_B])
self.priorities = np.maximum(
self.priorities_A,
self.priorities_B) # get the maximum between the two.
nt = len(self.type_A)
# ADM add some "extra" columns that are needed for observations.
for col in ["RA", "DEC", "PARALLAX", "PMRA", "PMDEC", "REF_EPOCH"]:
self.targets[col] = np.zeros(nt, dtype=mtldatamodel[col].dtype)
Amx = np.array([Mx[t].mask for t in self.type_A])
Bmx = np.array([Mx[t].mask for t in self.type_B])
self.targets['DESI_TARGET'] = Amx | Bmx
for col in ['BGS_TARGET', 'MWS_TARGET', 'SUBPRIORITY']:
self.targets[col] = np.zeros(nt, dtype=mtldatamodel[col].dtype)
n = len(self.targets)
self.targets['ZFLUX'] = 10**((22.5 - np.linspace(20, 22, n)) / 2.5)
self.targets['TARGETID'] = list(range(n))
pinit, ninit = initial_priority_numobs(self.targets)
self.targets["PRIORITY_INIT"] = pinit
self.targets["NUMOBS_INIT"] = ninit
# - reverse the order for zcat to make sure joins work.
self.zcat = Table()
self.zcat['TARGETID'] = self.targets['TARGETID'][::-1]
self.zcat['Z'] = [1.0, 1.5, 2.5]
self.zcat['ZWARN'] = [0, 0, 0]
self.zcat['NUMOBS'] = [1, 1, 1]
self.zcat['SPECTYPE'] = ['QSO', 'QSO', 'GALAXY']
# priorities and numobs more after measuring redshifts.
self.post_prio = [0 for t in self.type_A]
self.post_numobs_more = [0 for t in self.type_A]
self.post_prio[0] = Mx['QSO'].priorities['MORE_ZGOOD'] # highz QSO.
self.post_prio[1] = Mx['QSO'].priorities['DONE'] # Lowz QSO, DONE.
self.post_prio[2] = Mx['ELG'].priorities['DONE'] # ELG, DONE.
self.post_numobs_more[0] = 3
self.post_numobs_more[1] = 0
self.post_numobs_more[2] = 0
def setUp(self):
self.targets = Table()
# This is a dual identity case. In all cases the target is both QSO and ELG.
# The first case is a true QSO with lowz.
# The second case is a true QSO with highz.
# The third case is an ELG.
self.type_A = np.array(['QSO', 'QSO', 'ELG'])
self.type_B = np.array(['ELG', 'ELG', 'QSO'])
self.priorities_A = np.array(
[Mx[t].priorities['UNOBS'] for t in self.type_A])
self.priorities_B = np.array(
[Mx[t].priorities['UNOBS'] for t in self.type_B])
self.priorities = np.maximum(
self.priorities_A,
self.priorities_B) # get the maximum between the two.
self.targets['DESI_TARGET'] = np.array([
Mx[t].mask for t in self.type_A
]) | np.array([Mx[t].mask for t in self.type_B])
self.targets['BGS_TARGET'] = np.zeros(len(self.type_A), dtype=np.int64)
self.targets['MWS_TARGET'] = np.zeros(len(self.type_A), dtype=np.int64)
n = len(self.targets)
self.targets['ZFLUX'] = 10**((22.5 - np.linspace(20, 22, n)) / 2.5)
self.targets['TARGETID'] = list(range(n))
pinit, ninit = initial_priority_numobs(self.targets)
self.targets["PRIORITY_INIT"] = pinit
self.targets["NUMOBS_INIT"] = ninit
# - reverse the order for zcat to make sure joins work.
self.zcat = Table()
self.zcat['TARGETID'] = self.targets['TARGETID'][::-1]
self.zcat['Z'] = [1.0, 1.5, 2.5]
self.zcat['ZWARN'] = [0, 0, 0]
self.zcat['NUMOBS'] = [1, 1, 1]
self.zcat['SPECTYPE'] = ['QSO', 'QSO', 'GALAXY']
# priorities and numobs more after measuring redshifts.
self.post_prio = [0 for t in self.type_A]
self.post_numobs_more = [0 for t in self.type_A]
self.post_prio[0] = Mx['QSO'].priorities['MORE_ZGOOD'] # highz QSO.
self.post_prio[1] = Mx['QSO'].priorities['DONE'] # Lowz QSO, DONE.
self.post_prio[2] = Mx['ELG'].priorities['DONE'] # ELG, DONE.
self.post_numobs_more[0] = 3
self.post_numobs_more[1] = 0
self.post_numobs_more[2] = 0
def test_endless_bgs(self):
"""Test BGS targets always get another observation in bright time.
"""
# ADM create a set of BGS FAINT/BGS_BRIGHT targets
# ADM (perhaps) also another target class.
bgstargets = self.targets.copy()
bgstargets["DESI_TARGET"] = Mx["BGS_ANY"]
bgstargets[
"BGS_TARGET"] = bgs_mask["BGS_FAINT"] | bgs_mask["BGS_BRIGHT"]
# ADM set their initial conditions for the bright-time survey.
pinit, ninit = initial_priority_numobs(bgstargets, obscon="BRIGHT")
bgstargets["PRIORITY_INIT"] = pinit
bgstargets["NUMOBS_INIT"] = ninit
# ADM create a copy of the zcat.
bgszcat = self.zcat.copy()
# ADM run through MTL.
mtl = make_mtl(bgstargets, obscon="BRIGHT", zcat=bgszcat)
# ADM all BGS targets should always have NUMOBS_MORE=1.
self.assertTrue(np.all(bgszcat["NUMOBS_MORE"] == 1))
def test_merged_qso(self):
"""Test QSO tracers that are also other target types get 1 observation.
"""
# ADM there are other tests of this kind in test_multiple_mtl.py.
# ADM create a set of targets that are QSOs and
# ADM (perhaps) also another target class.
qtargets = self.targets.copy()
qtargets["DESI_TARGET"] |= Mx["QSO"]
# ADM give them all a "tracer" redshift (below a LyA QSO).
qzcat = self.zcat.copy()
qzcat["Z"] = 0.5
# ADM set their initial conditions to be that of a QSO.
pinit, ninit = initial_priority_numobs(qtargets, obscon="DARK|GRAY")
qtargets["PRIORITY_INIT"] = pinit
qtargets["NUMOBS_INIT"] = ninit
# ADM run through MTL.
mtl = make_mtl(qtargets, obscon="DARK|GRAY", zcat=qzcat)
# ADM all confirmed tracer quasars should have NUMOBS_MORE=0.
self.assertTrue(np.all(qzcat["NUMOBS_MORE"] == 0))
def test_priorities(self):
"""Test that priorities are set correctly for both the main survey and SV.
"""
# ADM loop through once for SV and once for the main survey.
for prefix in ["", "SV1_"]:
t = self.targets.copy()
z = self.zcat.copy()
main_names = ['DESI_TARGET', 'BGS_TARGET', 'MWS_TARGET']
for name in main_names:
t.rename_column(name, prefix + name)
# ADM retrieve the mask and column names for this survey flavor.
colnames, masks, _ = main_cmx_or_sv(t)
desi_target, bgs_target, mws_target = colnames
desi_mask, bgs_mask, mws_mask = masks
# - No targeting bits set is priority=0
self.assertTrue(np.all(calc_priority(t, z) == 0))
# - test QSO > (LRG_1PASS | LRG_2PASS) > ELG
t[desi_target] = desi_mask.ELG
self.assertTrue(
np.all(
calc_priority(t, z) == desi_mask.ELG.priorities['UNOBS']))
t[desi_target] |= desi_mask.LRG_1PASS
self.assertTrue(
np.all(
calc_priority(t, z) == desi_mask.LRG.priorities['UNOBS']))
t[desi_target] |= desi_mask.LRG_2PASS
self.assertTrue(
np.all(
calc_priority(t, z) == desi_mask.LRG.priorities['UNOBS']))
t[desi_target] |= desi_mask.QSO
self.assertTrue(
np.all(
calc_priority(t, z) == desi_mask.QSO.priorities['UNOBS']))
# - different states -> different priorities
# - Done is Done, regardless of ZWARN.
t[desi_target] = desi_mask.ELG
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 1, 1]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, z)["PRIORITY"]
self.assertEqual(p[0], desi_mask.ELG.priorities['UNOBS'])
self.assertEqual(p[1], desi_mask.ELG.priorities['DONE'])
self.assertEqual(p[2], desi_mask.ELG.priorities['DONE'])
# - BGS FAINT targets are never DONE, only MORE_ZGOOD.
t[desi_target] = desi_mask.BGS_ANY
t[bgs_target] = bgs_mask.BGS_FAINT
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 1, 1]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, z)["PRIORITY"]
self.assertEqual(p[0], bgs_mask.BGS_FAINT.priorities['UNOBS'])
self.assertEqual(p[1], bgs_mask.BGS_FAINT.priorities['MORE_ZWARN'])
self.assertEqual(p[2], bgs_mask.BGS_FAINT.priorities['MORE_ZGOOD'])
# BGS_FAINT: {UNOBS: 2000, MORE_ZWARN: 2000, MORE_ZGOOD: 1000, DONE: 2, OBS: 1, DONOTOBSERVE: 0}
# - BGS BRIGHT targets are never DONE, only MORE_ZGOOD.
t[desi_target] = desi_mask.BGS_ANY
t[bgs_target] = bgs_mask.BGS_BRIGHT
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 1, 1]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, z)["PRIORITY"]
self.assertEqual(p[0], bgs_mask.BGS_BRIGHT.priorities['UNOBS'])
self.assertEqual(p[1],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZWARN'])
self.assertEqual(p[2],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZGOOD'])
# BGS_BRIGHT: {UNOBS: 2100, MORE_ZWARN: 2100, MORE_ZGOOD: 1000, DONE: 2, OBS: 1, DONOTOBSERVE: 0}
# BGS targets are NEVER done even after 100 observations
t[desi_target] = desi_mask.BGS_ANY
t[bgs_target] = bgs_mask.BGS_BRIGHT
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 100, 100]
z['ZWARN'] = [1, 1, 0]
p = calc_priority(t, z)
self.assertEqual(p[0], bgs_mask.BGS_BRIGHT.priorities['UNOBS'])
self.assertEqual(p[1],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZWARN'])
self.assertEqual(p[2],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZGOOD'])
# BGS ZGOOD targets always have lower priority than MWS targets that
# are not DONE.
# ADM first discard N/S informational bits from bitmask as these
# ADM should never trump the other bits.
bgs_names = [
name for name in bgs_mask.names()
if 'NORTH' not in name and 'SOUTH' not in name
]
mws_names = [
name for name in mws_mask.names()
if 'NORTH' not in name and 'SOUTH' not in name
]
lowest_bgs_priority_zgood = min(
[bgs_mask[n].priorities['MORE_ZGOOD'] for n in bgs_names])
lowest_mws_priority_unobs = min(
[mws_mask[n].priorities['UNOBS'] for n in mws_names])
lowest_mws_priority_zwarn = min(
[mws_mask[n].priorities['MORE_ZWARN'] for n in mws_names])
lowest_mws_priority_zgood = min(
[mws_mask[n].priorities['MORE_ZGOOD'] for n in mws_names])
lowest_mws_priority = min(lowest_mws_priority_unobs,
lowest_mws_priority_zwarn,
lowest_mws_priority_zgood)
self.assertLess(lowest_bgs_priority_zgood, lowest_mws_priority)
def test_priorities(self):
"""Test that priorities are set correctly for both the main survey and SV.
"""
# ADM loop through once for SV and once for the main survey.
for prefix in ["", "SV1_"]:
t = self.targets.copy()
z = self.zcat.copy()
main_names = ['DESI_TARGET', 'BGS_TARGET', 'MWS_TARGET']
for name in main_names:
t.rename_column(name, prefix + name)
# ADM retrieve the mask and column names for this survey flavor.
colnames, masks, _ = main_cmx_or_sv(t)
desi_target, bgs_target, mws_target = colnames
desi_mask, bgs_mask, mws_mask = masks
# - No targeting bits set is priority=0
self.assertTrue(np.all(calc_priority(t, z, "BRIGHT") == 0))
# ADM test QSO > LRG > ELG for main survey and SV.
t[desi_target] = desi_mask.ELG
self.assertTrue(
np.all(
calc_priority(t, z, "GRAY|DARK") ==
desi_mask.ELG.priorities['UNOBS']))
t[desi_target] |= desi_mask.LRG
self.assertTrue(
np.all(
calc_priority(t, z, "GRAY|DARK") ==
desi_mask.LRG.priorities['UNOBS']))
t[desi_target] |= desi_mask.QSO
self.assertTrue(
np.all(
calc_priority(t, z, "GRAY|DARK") ==
desi_mask.QSO.priorities['UNOBS']))
# - different states -> different priorities
# - Done is Done, regardless of ZWARN.
t[desi_target] = desi_mask.ELG
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 1, 1]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, "GRAY|DARK", zcat=z)["PRIORITY"]
self.assertEqual(p[0], desi_mask.ELG.priorities['UNOBS'])
self.assertEqual(p[1], desi_mask.ELG.priorities['DONE'])
self.assertEqual(p[2], desi_mask.ELG.priorities['DONE'])
# ADM In BRIGHT conditions BGS FAINT targets are
# ADM never DONE, only MORE_ZGOOD.
t[desi_target] = desi_mask.BGS_ANY
t[bgs_target] = bgs_mask.BGS_FAINT
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 1, 1]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, "BRIGHT", zcat=z)["PRIORITY"]
self.assertEqual(p[0], bgs_mask.BGS_FAINT.priorities['UNOBS'])
self.assertEqual(p[1], bgs_mask.BGS_FAINT.priorities['MORE_ZWARN'])
self.assertEqual(p[2], bgs_mask.BGS_FAINT.priorities['MORE_ZGOOD'])
# BGS_FAINT: {UNOBS: 2000, MORE_ZWARN: 2000, MORE_ZGOOD: 1000, DONE: 2, OBS: 1, DONOTOBSERVE: 0}
# ADM but in DARK conditions, BGS_FAINT should behave as
# ADM for other target classes.
z = self.zcat.copy()
z['NUMOBS'] = [0, 1, 1]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, "DARK|GRAY", zcat=z)["PRIORITY"]
self.assertEqual(p[0], bgs_mask.BGS_FAINT.priorities['UNOBS'])
self.assertEqual(p[1], bgs_mask.BGS_FAINT.priorities['DONE'])
self.assertEqual(p[2], bgs_mask.BGS_FAINT.priorities['DONE'])
# ADM In BRIGHT conditions BGS BRIGHT targets are
# ADM never DONE, only MORE_ZGOOD.
t[desi_target] = desi_mask.BGS_ANY
t[bgs_target] = bgs_mask.BGS_BRIGHT
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 1, 1]
z['ZWARN'] = [1, 1, 0]
p = make_mtl(t, "BRIGHT", zcat=z)["PRIORITY"]
self.assertEqual(p[0], bgs_mask.BGS_BRIGHT.priorities['UNOBS'])
self.assertEqual(p[1],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZWARN'])
self.assertEqual(p[2],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZGOOD'])
# BGS_BRIGHT: {UNOBS: 2100, MORE_ZWARN: 2100, MORE_ZGOOD: 1000, DONE: 2, OBS: 1, DONOTOBSERVE: 0}
# ADM In BRIGHT conditions BGS targets are
# ADM NEVER done even after 100 observations
t[desi_target] = desi_mask.BGS_ANY
t[bgs_target] = bgs_mask.BGS_BRIGHT
t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t)
z['NUMOBS'] = [0, 100, 100]
z['ZWARN'] = [1, 1, 0]
p = calc_priority(t, z, "BRIGHT")
self.assertEqual(p[0], bgs_mask.BGS_BRIGHT.priorities['UNOBS'])
self.assertEqual(p[1],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZWARN'])
self.assertEqual(p[2],
bgs_mask.BGS_BRIGHT.priorities['MORE_ZGOOD'])
# BGS ZGOOD targets always have lower priority than MWS targets that
# are not DONE. Exempting the MWS "BACKUP" targets.
# ADM first discard N/S informational bits from bitmask as these
# ADM should never trump the other bits.
bgs_names = [
name for name in bgs_mask.names()
if 'NORTH' not in name and 'SOUTH' not in name
]
mws_names = [
name for name in mws_mask.names() if 'NORTH' not in name
and 'SOUTH' not in name and 'BACKUP' not in name
]
lowest_mws_priority_unobs = [
mws_mask[n].priorities['UNOBS'] for n in mws_names
]
lowest_bgs_priority_zgood = np.min(
[bgs_mask[n].priorities['MORE_ZGOOD'] for n in bgs_names])
# ADM MORE_ZGOOD and MORE_ZWARN are only meaningful if a
# ADM target class requests more than 1 observation (except
# ADM for BGS, which has a numobs=infinity exception)
lowest_mws_priority_zwarn = [
mws_mask[n].priorities['MORE_ZWARN'] for n in mws_names
if mws_mask[n].numobs > 1
]
lowest_mws_priority_zgood = [
mws_mask[n].priorities['MORE_ZGOOD'] for n in mws_names
if mws_mask[n].numobs > 1
]
lowest_mws_priority = np.min(
np.concatenate([
lowest_mws_priority_unobs, lowest_mws_priority_zwarn,
lowest_mws_priority_zgood
]))
self.assertLess(lowest_bgs_priority_zgood, lowest_mws_priority)
def finalize_secondary(scxtargs,
scnd_mask,
survey='main',
sep=1.,
darkbright=False):
"""Assign secondary targets a realistic TARGETID, finalize columns.
Parameters
----------
scxtargs : :class:`~numpy.ndarray`
An array of secondary targets, must contain the columns `RA`,
`DEC` and `TARGETID`. `TARGETID` should be -1 for objects
that lack a `TARGETID`.
scnd_mask : :class:`desiutil.bitmask.BitMask`
A mask corresponding to a set of secondary targets, e.g, could
be ``from desitarget.targetmask import scnd_mask`` for the
main survey mask.
survey : :class:`str`, optional, defaults to "main"
string indicating whether we are working in the context of the
Main Survey (`main`) or SV (e.g. `sv1`, `sv2` etc.). Used to
set the `RELEASE` number in the `TARGETID` (see Notes).
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match secondary targets to
themselves in ARCSECONDS.
darkbright : :class:`bool`, optional, defaults to ``False``
If sent, then split `NUMOBS_INIT` and `PRIORITY_INIT` into
`NUMOBS_INIT_DARK`, `NUMOBS_INIT_BRIGHT`, `PRIORITY_INIT_DARK`
and `PRIORITY_INIT_BRIGHT` and calculate values appropriate
to "BRIGHT" and "DARK|GRAY" observing conditions.
Returns
-------
:class:`~numpy.ndarray`
The array of secondary targets, with the `TARGETID` bit updated
to be unique and reasonable and the `SCND_TARGET` column renamed
based on the flavor of `scnd_mask`.
Notes
-----
- Secondaries without `OVERRIDE` are also matched to themselves
Such matches are given the same `TARGETID` (that of the primary
if they match a primary) and the bitwise or of `SCND_TARGET` and
`OBSCONDITIONS` bits across matches. The highest `PRIORITY_INIT`
is retained, and others are set to -1. Only secondaries with
priorities that are not -1 are written to the main file. If
multiple matching secondary targets have the same (highest)
priority, the first one encountered retains its `PRIORITY_INIT`
- The secondary `TARGETID` is designed to be reproducible. It
combines `BRICKID` based on location, `OBJID` based on the
order of the targets in the secondary file (`SCND_ORDER`) and
`RELEASE` from the secondary bit number (`SCND_TARGET`) and the
input `survey`. `RELEASE` is set to ((X-1)*100)+np.log2(scnd_bit)
with X from the `survey` string survey=svX and scnd_bit from
`SCND_TARGET`. For the main survey (survey="main") X-1 is 5.
"""
# ADM assign new TARGETIDs to targets without a primary match.
nomatch = scxtargs["TARGETID"] == -1
# ADM get the BRICKIDs for each source.
brxid = bricks.brickid(scxtargs["RA"], scxtargs["DEC"])
# ADM ensure unique secondary bits for different iterations of SV
# ADM and the Main Survey.
if survey == 'main':
Xm1 = 5
elif survey[0:2] == 'sv':
# ADM the re.search just extracts the numbers in the string.
Xm1 = int(re.search(r'\d+', survey).group()) - 1
# ADM we've allowed a max of up to sv5 (!). Fail if surpassed.
if Xm1 >= 5:
msg = "Only coded for up to 'sv5', not {}!!!".format(survey)
log.critical(msg)
raise ValueError(msg)
else:
msg = "allowed surveys: 'main', 'svX', not {}!!!".format(survey)
log.critical(msg)
raise ValueError(msg)
# ADM the RELEASE for each source is the `SCND_TARGET` bit NUMBER.
release = (Xm1 * 100) + np.log2(scxtargs["SCND_TARGET_INIT"]).astype('int')
# ADM build the OBJIDs based on the values of SCND_ORDER.
t0 = time()
log.info("Begin assigning OBJIDs to bricks...")
# ADM So as not to overwhelm the bit-limits for OBJID
# ADM rank by SCND_ORDER for each brick and bit combination.
# ADM First, create a unique ID based on brxid and release.
scnd_order = scxtargs["SCND_ORDER"]
sorter = (1000 * brxid) + release
# ADM sort the unique IDs and split based on where they change.
argsort = np.argsort(sorter)
w = np.where(np.diff(sorter[argsort]))[0]
soperbrxbit = np.split(scnd_order[argsort], w + 1)
# ADM loop through each (brxid, release) and sort on scnd_order.
# ADM double argsort returns the ascending ranked order of the entry
# ADM (whereas a single argsort returns the indexes for ordering).
sortperbrxbit = [np.argsort(np.argsort(so)) for so in soperbrxbit]
# ADM finally unroll the (brxid, release) combinations...
sortedobjid = np.array(list(itertools.chain.from_iterable(sortperbrxbit)))
# ADM ...and reorder based on the initial argsort.
objid = np.zeros_like(sortedobjid) - 1
objid[argsort] = sortedobjid
log.info("Assigned OBJIDs to bricks in {:.1f}s".format(time() - t0))
# ADM check that the objid array was entirely populated.
assert np.all(objid != -1)
# ADM assemble the TARGETID, SCND objects have RELEASE==0.
targetid = encode_targetid(objid=objid, brickid=brxid, release=release)
# ADM a check that the generated TARGETIDs are unique.
if len(set(targetid)) != len(targetid):
msg = "duplicate TARGETIDs generated for secondary targets!!!"
log.critical(msg)
raise ValueError(msg)
# ADM assign the unique TARGETIDs to the secondary objects.
scxtargs["TARGETID"][nomatch] = targetid[nomatch]
log.debug("Assigned {} targetids to unmatched secondaries".format(
len(targetid[nomatch])))
# ADM match secondaries to themselves, to ensure duplicates
# ADM share a TARGETID. Don't match special (OVERRIDE) targets
# ADM or sources that have already been matched to a primary.
w = np.where(~scxtargs["OVERRIDE"] & nomatch)[0]
if len(w) > 0:
log.info("Matching secondary targets to themselves...t={:.1f}s".format(
time() - t0))
# ADM use astropy for the matching. At NERSC, astropy matches
# ADM ~20M objects to themselves in about 10 minutes.
c = SkyCoord(scxtargs["RA"][w] * u.deg, scxtargs["DEC"][w] * u.deg)
m1, m2, _, _ = c.search_around_sky(c, sep * u.arcsec)
log.info("Done with matching...t={:.1f}s".format(time() - t0))
# ADM restrict only to unique matches (and exclude self-matches).
uniq = m1 > m2
m1, m2 = m1[uniq], m2[uniq]
# ADM set same TARGETID for any matches. m2 must come first, here.
scxtargs["TARGETID"][w[m2]] = scxtargs["TARGETID"][w[m1]]
# ADM Ensure secondary targets with matching TARGETIDs have all the
# ADM relevant SCND_TARGET bits set. By definition, targets with
# ADM OVERRIDE set never have matching TARGETIDs.
wnoov = np.where(~scxtargs["OVERRIDE"])[0]
if len(wnoov) > 0:
for _, inds in duplicates(scxtargs["TARGETID"][wnoov]):
scnd_targ = 0
for ind in inds:
scnd_targ |= scxtargs["SCND_TARGET"][wnoov[ind]]
scxtargs["SCND_TARGET"][wnoov[inds]] = scnd_targ
log.info("Done checking SCND_TARGET...t={:.1f}s".format(time() - t0))
# ADM change the data model depending on whether the mask
# ADM is an SVX (X = 1, 2, etc.) mask or not. Nothing will
# ADM change if the mask has no preamble.
prepend = scnd_mask._name[:-9].upper()
scxtargs = rfn.rename_fields(scxtargs,
{'SCND_TARGET': prepend + 'SCND_TARGET'})
# APC same thing for DESI_TARGET
scxtargs = rfn.rename_fields(scxtargs,
{'DESI_TARGET': prepend + 'DESI_TARGET'})
# APC Remove duplicate targetids from secondary-only targets
alldups = []
for _, dups in duplicates(scxtargs['TARGETID']):
# Retain the duplicate with highest priority, breaking ties
# on lowest index in list of duplicates
dups = np.delete(dups, np.argmax(scxtargs['PRIORITY_INIT'][dups]))
alldups.append(dups)
alldups = np.hstack(alldups)
log.debug(
"Flagging {} duplicate secondary targetids with PRIORITY_INIT=-1".
format(len(alldups)))
# ADM and remove the INIT fields in prep for a dark/bright split.
scxtargs = rfn.drop_fields(scxtargs, ["PRIORITY_INIT", "NUMOBS_INIT"])
# ADM set initial priorities, numobs and obsconditions for both
# ADM BRIGHT and DARK|GRAY conditions, if requested.
nscx = len(scxtargs)
nodata = np.zeros(nscx, dtype='int') - 1
if darkbright:
ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"]
else:
ender, obscon = [""], ["DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18"]
cols, vals, forms = [], [], []
for edr, oc in zip(ender, obscon):
cols += ["{}_INIT{}".format(pn, edr) for pn in ["PRIORITY", "NUMOBS"]]
vals += [nodata, nodata]
forms += ['>i8', '>i8']
# ADM write the output array.
newdt = [dt for dt in zip(cols, forms)]
done = np.array(np.zeros(nscx), dtype=scxtargs.dtype.descr + newdt)
for col in scxtargs.dtype.names:
done[col] = scxtargs[col]
for col, val in zip(cols, vals):
done[col] = val
# ADM add the actual PRIORITY/NUMOBS values.
for edr, oc in zip(ender, obscon):
pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr
done[pc], done[nc] = initial_priority_numobs(done,
obscon=oc,
scnd=True)
# APC Flagged duplicates are removed in io.write_secondary
done[pc][alldups] = -1
# APC add secondary flag in DESI_TARGET
cols, mx, surv = main_cmx_or_sv(done, scnd=True)
done[cols[0]] = mx[0]['SCND_ANY']
# ADM set the OBSCONDITIONS.
done["OBSCONDITIONS"] = set_obsconditions(done, scnd=True)
return done
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
primtargs : :class:`~numpy.ndarray`
An array of primary targets.
scndout : :class`~numpy.ndarray`
Name of a sub-directory to which to write the information in
`desitarget.secondary.outdatamodel` with `TARGETID` and (the
highest) `PRIORITY_INIT` updated with matching primary info.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
pix : :class:`list`, optional, defaults to `None`
Limit secondary targets to (NESTED) HEALpixels that touch
pix at the supplied `nside`, as a speed-up.
nside : :class:`int`, optional, defaults to `None`
The (NESTED) HEALPixel nside to be used with `pixlist`.
Returns
-------
:class:`~numpy.ndarray`
The array of primary targets, with the `SCND_TARGET` bit
populated for matches to secondary targets
"""
# ADM add a SCND_TARGET column to the primary targets.
dt = primtargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(primtargs), dtype=dt)
for col in primtargs.dtype.names:
targs[col] = primtargs[col]
# ADM check if this is an SV or main survey file.
cols, mx, surv = main_cmx_or_sv(targs, scnd=True)
log.info('running on the {} survey...'.format(surv))
if surv != 'main':
scxdir = os.path.join(scxdir, surv)
# ADM read in non-OVERRIDE secondary targets.
scxtargs = read_files(scxdir, mx[3])
scxtargs = scxtargs[~scxtargs["OVERRIDE"]]
# ADM match primary targets to non-OVERRIDE secondary targets.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if nside is not None and pix is not None:
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(primtargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(primtargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(scndout, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
desi_mask = desimasks[0]
targs[desicols[0]][umtargs] |= desi_mask.SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# APC Secondary target bits only affect PRIORITY, NUMOBS and
# APC obsconditions for specific DESI_TARGET bits
# APC See https://github.com/desihub/desitarget/pull/530
# APC Only consider primary targets with secondary bits set
scnd_update = (targs[desicols[0]] & desi_mask['SCND_ANY']) != 0
if np.any(scnd_update):
# APC Allow changes to primaries if the DESI_TARGET bitmask has
# APC only the following bits set, in any combination.
log.info(
'Testing if secondary targets can update {} matched primaries'.
format(scnd_update.sum()))
update_from_scnd_bits = desi_mask['SCND_ANY'] | desi_mask[
'MWS_ANY'] | desi_mask['STD_BRIGHT'] | desi_mask[
'STD_FAINT'] | desi_mask['STD_WD']
scnd_update &= ((targs[desicols[0]] & ~update_from_scnd_bits) == 0)
log.info(
'Setting new priority, numobs and obsconditions from secondary for {} matched primaries'
.format(scnd_update.sum()))
# APC Primary and secondary obsconditions are or'd
scnd_obscon = set_obsconditions(targs[scnd_update], scnd=True)
targs['OBSCONDITIONS'][scnd_update] &= scnd_obscon
# APC bit of a hack here
# APC Check for _BRIGHT, _DARK split in column names
darkbright = 'NUMOBS_INIT_DARK' in targs.dtype.names
if darkbright:
ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"]
else:
ender, obscon = [""], [
"DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18"
]
# APC secondaries can increase priority and numobs
for edr, oc in zip(ender, obscon):
pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr
scnd_priority, scnd_numobs = initial_priority_numobs(
targs[scnd_update], obscon=oc, scnd=True)
targs[nc][scnd_update] = np.maximum(targs[nc][scnd_update],
scnd_numobs)
targs[pc][scnd_update] = np.maximum(targs[pc][scnd_update],
scnd_priority)
# ADM update the secondary targets with the primary information.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM the maximum priority will be used to break ties in the
# ADM unlikely event that a secondary matches two primaries.
hipri = np.maximum(targs["PRIORITY_INIT_DARK"],
targs["PRIORITY_INIT_BRIGHT"])
scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs]
# ADM write the secondary targets that have updated TARGETIDs.
ii = scxtargs["TARGETID"] != -1
nmatches = np.sum(ii)
log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format(
nmatches, scndout,
time() - start))
if nmatches > 0:
hdr = fitsio.FITSHDR()
hdr["SURVEY"] = surv
fitsio.write(scndout,
scxtargs[ii],
extname='SCND_TARG',
header=hdr,
clobber=True)
log.info('Done...t={:.1f}s'.format(time() - start))
return targs