def test_select_targets(self):
"""Test select targets works with either data or filenames
"""
# ADM only test the LRG cuts for speed. There's a
# ADM full run through all classes in test_cuts_basic.
tc = ["LRG"]
for filelist in [self.tractorfiles, self.sweepfiles]:
# ADM set backup to False as the Gaia unit test
# ADM files only cover a limited pixel range.
targets = cuts.select_targets(filelist,
numproc=1,
tcnames=tc,
backup=False)
t1 = cuts.select_targets(filelist[0:1],
numproc=1,
tcnames=tc,
backup=False)
t2 = cuts.select_targets(filelist[0],
numproc=1,
tcnames=tc,
backup=False)
for col in t1.dtype.names:
try:
notNaN = ~np.isnan(t1[col])
except TypeError: # - can't check string columns for NaN
notNaN = np.ones(len(t1), dtype=bool)
self.assertTrue(np.all(t1[col][notNaN] == t2[col][notNaN]))
def test_qso_selection_options(self):
targetfile = self.tractorfiles[0]
for qso_selection in cuts.qso_selection_options:
results = cuts.select_targets(targetfile,
qso_selection=qso_selection)
with self.assertRaises(ValueError):
results = cuts.select_targets(targetfile,
numproc=1,
qso_selection='blatfoo')
def test_targets_spatial(self):
"""Test applying RA/Dec/HEALpixel inputs to sweeps recovers same targets
"""
# ADM only test some of the galaxy cuts for speed. There's a
# ADM full run through all classes in test_cuts_basic.
tc = ["LRG", "ELG", "BGS"]
infiles = self.sweepfiles[2]
targets = cuts.select_targets(infiles, numproc=1, tcnames=tc)
# ADM test the RA/Dec box input.
radecbox = [
np.min(targets["RA"]) - 0.01,
np.max(targets["RA"]) + 0.01,
np.min(targets["DEC"]) - 0.01,
np.max(targets["DEC"] + 0.01)
]
t1 = cuts.select_targets(infiles,
numproc=1,
tcnames=tc,
radecbox=radecbox)
# ADM test the RA/Dec/radius cap input.
centra, centdec = 0.5 * (radecbox[0] + radecbox[1]), 0.5 * (
radecbox[2] + radecbox[3])
# ADM 20 degrees should be a large enough radius for the sweeps.
maxrad = 20.
radecrad = centra, centdec, maxrad
t2 = cuts.select_targets(infiles,
numproc=1,
tcnames=tc,
radecrad=radecrad)
# ADM test the pixel input.
nside = pixarea2nside(box_area(radecbox))
pixlist = hp_in_box(nside, radecbox)
t3 = cuts.select_targets(infiles,
numproc=1,
tcnames=tc,
nside=nside,
pixlist=pixlist)
# ADM sort each set of targets on TARGETID to compare them.
targets = targets[np.argsort(targets["TARGETID"])]
t1 = t1[np.argsort(t1["TARGETID"])]
t2 = t2[np.argsort(t2["TARGETID"])]
t3 = t3[np.argsort(t3["TARGETID"])]
# ADM test the same targets were recovered and that
# ADM each recovered target has the same bits set.
for targs in t1, t2, t3:
for col in "TARGETID", "DESI_TARGET", "BGS_TARGET", "MWS_TARGET":
self.assertTrue(np.all(targs[col] == targets[col]))
def test_select_targets(self):
for filelist in [self.tractorfiles, self.sweepfiles]:
targets = cuts.select_targets(filelist, numproc=1)
t1 = cuts.select_targets(filelist[0:1], numproc=1)
t2 = cuts.select_targets(filelist[0], numproc=1)
for col in t1.dtype.names:
try:
notNaN = ~np.isnan(t1[col])
except TypeError: #- can't check string columns for NaN
notNan = np.ones(len(t1), dtype=bool)
self.assertTrue(np.all(t1[col][notNaN] == t2[col][notNaN]))
def test_select_targets(self):
for filelist in [self.tractorfiles, self.sweepfiles]:
targets = cuts.select_targets(filelist, numproc=1)
t1 = cuts.select_targets(filelist[0:1], numproc=1)
t2 = cuts.select_targets(filelist[0], numproc=1)
for col in t1.dtype.names:
try:
notNaN = ~np.isnan(t1[col])
except TypeError: #- can't check string columns for NaN
notNan = np.ones(len(t1), dtype=bool)
self.assertTrue(np.all(t1[col][notNaN]==t2[col][notNaN]))
def test_qso_selection_options(self):
"""Test the QSO selection options are passed correctly
"""
tc = ["QSO"]
targetfile = self.tractorfiles[0]
for qso_selection in cuts.qso_selection_options:
results = cuts.select_targets(targetfile,
tcnames=tc, qso_selection=qso_selection)
with self.assertRaises(ValueError):
results = cuts.select_targets(targetfile, numproc=1,
tcnames=tc, qso_selection='blatfoo')
def test_qso_selection_options(self):
"""Test the QSO selection options are passed correctly
"""
tc = ["QSO"]
targetfile = self.tractorfiles[0]
for qso_selection in cuts.qso_selection_options:
# ADM set backup to False as the Gaia unit test
# ADM files only cover a limited pixel range.
results = cuts.select_targets(targetfile, backup=False,
tcnames=tc, qso_selection=qso_selection)
with self.assertRaises(ValueError):
results = cuts.select_targets(targetfile, numproc=1, backup=False,
tcnames=tc, qso_selection='blatfoo')
def main():
ns = ap.parse_args()
t0 = time()
objects = read_mock_durham(ns.src_core, ns.src_photo)
t1 = time()
targetflags = select_targets(objects)
keep = (targetflags !=0)
targets = objects[keep]
targetflag = targetflags[keep]
t2 = time()
numobs = calc_numobs(targets, targetflag)
targets = desitarget.targets.finalize(targets, targetflag, numobs)
t3 = time()
write_targets(ns.dest, targets)
t4 = time()
if ns.verbose:
print ('written {} targets to {}'.format(len(targets), ns.dest))
print('Read mock file {:.1f} sec'.format(t1-t0))
print('Make target selection {:.1f} sec'.format(t2-t1))
print('Compute numobs, finalize target selection {:.1f} sec'.format(t3-t2))
print('Write output {:.1f} sec'.format(t4-t3))
def test_missing_files(self):
"""Test the code will die gracefully if input files are missing
"""
with self.assertRaises(ValueError):
targets = cuts.select_targets([
'blat.foo1234',
], numproc=1)
def test_targets_spatial_inputs(self):
"""Test the code fails if more than one spatial input is passed
"""
# ADM set up some fake inputs.
pixlist = [0, 1]
radecbox = [2, 3, 4, 5]
radecrad = [6, 7, 8]
# ADM we should throw an error every time we pass 2 inputs that aren't NoneType.
timesthrown = 0
for i in range(3):
inputs = [pixlist, radecbox, radecrad]
inputs[i] = None
try:
cuts.select_targets(self.sweepfiles, numproc=1,
pixlist=inputs[0], radecbox=inputs[1], radecrad=inputs[2])
except ValueError:
timesthrown += 1
self.assertEqual(timesthrown, 3)
def test_sv_cuts(self):
"""Test SV cuts work.
"""
# ADM find all svX sub-directories in the desitarget directory.
fns = glob(resource_filename('desitarget', 'sv*'))
svlist = [os.path.basename(fn) for fn in fns if os.path.isdir(fn)]
for survey in svlist:
desicol, bgscol, mwscol = [
"{}_{}_TARGET".format(survey.upper(), tc)
for tc in ["DESI", "BGS", "MWS"]
]
for filelist in [self.tractorfiles, self.sweepfiles]:
# ADM set backup to False as the Gaia unit test
# ADM files only cover a limited pixel range.
targets = cuts.select_targets(filelist,
survey=survey,
backup=False)
for col in desicol, bgscol, mwscol:
self.assertTrue(col in targets.dtype.names)
self.assertEqual(len(targets),
np.count_nonzero(targets[desicol]))
# ADM this test should be fine as long as the main survey BGS
# ADM bits don't get divorced from the SV survey bits.
bgs1 = (targets[desicol] & desi_mask.BGS_ANY) != 0
bgs2 = targets[bgscol] != 0
self.assertTrue(np.all(bgs1 == bgs2))
# ADM backup targets can only be run on sweep files.
for filelist in self.sweepfiles:
# ADM also test the backup targets in the pixels covered
# ADM by the Gaia unit test files.
targets = cuts.select_targets(filelist,
survey=survey,
nside=self.nside,
pixlist=self.pix)
for col in desicol, bgscol, mwscol:
self.assertTrue(col in targets.dtype.names)
self.assertEqual(len(targets),
np.count_nonzero(targets[desicol]))
def test_parallel_select(self):
for nproc in [1,2]:
for filelist in [self.tractorfiles, self.sweepfiles]:
targets = cuts.select_targets(filelist, numproc=nproc)
self.assertTrue('DESI_TARGET' in targets.dtype.names)
self.assertTrue('BGS_TARGET' in targets.dtype.names)
self.assertTrue('MWS_TARGET' in targets.dtype.names)
self.assertEqual(len(targets), np.count_nonzero(targets['DESI_TARGET']))
bgs1 = (targets['DESI_TARGET'] & desi_mask.BGS_ANY) != 0
bgs2 = targets['BGS_TARGET'] != 0
self.assertTrue(np.all(bgs1 == bgs2))
def test_parallel_select(self):
for nproc in [1, 2]:
for filelist in [self.tractorfiles, self.sweepfiles]:
targets = cuts.select_targets(filelist, numproc=nproc)
self.assertTrue('DESI_TARGET' in targets.dtype.names)
self.assertTrue('BGS_TARGET' in targets.dtype.names)
self.assertTrue('MWS_TARGET' in targets.dtype.names)
self.assertEqual(len(targets),
np.count_nonzero(targets['DESI_TARGET']))
bgs1 = (targets['DESI_TARGET'] & desi_mask.BGS_ANY) != 0
bgs2 = targets['BGS_TARGET'] != 0
self.assertTrue(np.all(bgs1 == bgs2))
def test_select_targets_sandbox(self):
from desitarget import sandbox
ntot = 0
for filename in self.tractorfiles + self.sweepfiles:
targets = cuts.select_targets(filename, numproc=1, sandbox=True)
objects = Table.read(filename)
if 'BRICK_PRIMARY' in objects.colnames:
objects.remove_column('BRICK_PRIMARY')
desi_target, bgs_target, mws_target = \
sandbox.cuts.apply_sandbox_cuts(objects)
n = np.count_nonzero(desi_target) + \
np.count_nonzero(bgs_target) + \
np.count_nonzero(mws_target)
self.assertEqual(len(targets), n)
ntot += n
self.assertGreater(ntot, 0, 'No targets selected by sandbox.cuts')
def test_parallel_select(self):
"""Test multiprocessing parallelization works
"""
# ADM only test the ELG, BGS cuts for speed. There's a
# ADM full run through all classes in test_cuts_basic.
tc = ["ELG", "BGS"]
for nproc in [1, 2]:
for filelist in [self.tractorfiles, self.sweepfiles]:
targets = cuts.select_targets(filelist,
numproc=nproc, tcnames=tc)
self.assertTrue('DESI_TARGET' in targets.dtype.names)
self.assertTrue('BGS_TARGET' in targets.dtype.names)
self.assertTrue('MWS_TARGET' in targets.dtype.names)
self.assertEqual(len(targets), np.count_nonzero(targets['DESI_TARGET']))
bgs1 = (targets['DESI_TARGET'] & desi_mask.BGS_ANY) != 0
bgs2 = targets['BGS_TARGET'] != 0
self.assertTrue(np.all(bgs1 == bgs2))
def test_backup(self):
"""Test BACKUP targets are selected.
"""
# ADM only test the ELG, BGS cuts for speed. There's a
# ADM full run through all classes in test_cuts_basic.
tc = ["ELG", "BGS"]
# ADM BACKUP targets can only run on the sweep files.
for filelist in self.sweepfiles:
# ADM limit to pixels covered in the Gaia unit test files.
targets = cuts.select_targets(filelist, numproc=1, tcnames=tc,
nside=self.nside, pixlist=self.pix)
self.assertTrue('DESI_TARGET' in targets.dtype.names)
self.assertTrue('BGS_TARGET' in targets.dtype.names)
self.assertTrue('MWS_TARGET' in targets.dtype.names)
self.assertEqual(len(targets), np.count_nonzero(targets['DESI_TARGET']))
bgs1 = (targets['DESI_TARGET'] & desi_mask.BGS_ANY) != 0
bgs2 = targets['BGS_TARGET'] != 0
self.assertTrue(np.all(bgs1 == bgs2))
def test_sv_cuts(self):
"""Test SV cuts work.
"""
for filelist in [self.tractorfiles, self.sweepfiles]:
# ADM increase maxsv as we add more iterations of SV!!!
maxsv = 1
svlist = ['sv{}'.format(i) for i in range(1, maxsv+1)]
for survey in svlist:
desicol, bgscol, mwscol = ["{}_{}_TARGET".format(survey.upper(), tc)
for tc in ["DESI", "BGS", "MWS"]]
targets = cuts.select_targets(filelist, survey=survey)
for col in desicol, bgscol, mwscol:
self.assertTrue(col in targets.dtype.names)
self.assertEqual(len(targets), np.count_nonzero(targets[desicol]))
# ADM this test should be fine as long as the main survey BGS
# ADM bits don't get divorced from the SV survey bits.
bgs1 = (targets[desicol] & desi_mask.BGS_ANY) != 0
bgs2 = targets[bgscol] != 0
self.assertTrue(np.all(bgs1 == bgs2))
def test_missing_files(self):
with self.assertRaises(ValueError):
targets = cuts.select_targets(['blat.foo1234',], numproc=1)
def test_missing_files(self):
with self.assertRaises(ValueError):
targets = cuts.select_targets([
'blat.foo1234',
], numproc=1)
#infiles = np.concatenate((infilessouth, infilesnorth)).tolist()
infiles = infilessouth
#
import time
start = time.time()
def flux_to_mag(flux):
mag = 22.5 - 2.5 * np.log10(flux)
return mag
targets = select_targets(infiles,
numproc=4,
mask=True,
tcnames=["BGS"],
survey='main',
backup=False)
bits = {'bright': 1, 'faint': 0, 'wise': 2}
#['BGS_FAINT', 'BGS_BRIGHT', 'BGS_FAINT_EXT', 'BGS_LOWQ', 'BGS_FIBMAG', ]
end = time.time()
print('Total run time: %f sec' % (end - start))
rmag = flux_to_mag(targets['FLUX_R'] / targets['MW_TRANSMISSION_R'])
for name, bit in zip(bits.keys(), bits.values()):
bgsbit = ((targets['BGS_TARGET'] & 2**(bit)) != 0)
print(name, np.sum(bgsbit))
print('rmag min: %f, rmag max: %f' %
(rmag[bgsbit].min(), rmag[bgsbit].max()))
