def test_select_targets(self): """Test select targets works with either data or filenames """ # ADM only test the LRG cuts for speed. There's a # ADM full run through all classes in test_cuts_basic. tc = ["LRG"] for filelist in [self.tractorfiles, self.sweepfiles]: # ADM set backup to False as the Gaia unit test # ADM files only cover a limited pixel range. targets = cuts.select_targets(filelist, numproc=1, tcnames=tc, backup=False) t1 = cuts.select_targets(filelist[0:1], numproc=1, tcnames=tc, backup=False) t2 = cuts.select_targets(filelist[0], numproc=1, tcnames=tc, backup=False) for col in t1.dtype.names: try: notNaN = ~np.isnan(t1[col]) except TypeError: # - can't check string columns for NaN notNaN = np.ones(len(t1), dtype=bool) self.assertTrue(np.all(t1[col][notNaN] == t2[col][notNaN]))
def test_qso_selection_options(self): targetfile = self.tractorfiles[0] for qso_selection in cuts.qso_selection_options: results = cuts.select_targets(targetfile, qso_selection=qso_selection) with self.assertRaises(ValueError): results = cuts.select_targets(targetfile, numproc=1, qso_selection='blatfoo')
def test_targets_spatial(self): """Test applying RA/Dec/HEALpixel inputs to sweeps recovers same targets """ # ADM only test some of the galaxy cuts for speed. There's a # ADM full run through all classes in test_cuts_basic. tc = ["LRG", "ELG", "BGS"] infiles = self.sweepfiles[2] targets = cuts.select_targets(infiles, numproc=1, tcnames=tc) # ADM test the RA/Dec box input. radecbox = [ np.min(targets["RA"]) - 0.01, np.max(targets["RA"]) + 0.01, np.min(targets["DEC"]) - 0.01, np.max(targets["DEC"] + 0.01) ] t1 = cuts.select_targets(infiles, numproc=1, tcnames=tc, radecbox=radecbox) # ADM test the RA/Dec/radius cap input. centra, centdec = 0.5 * (radecbox[0] + radecbox[1]), 0.5 * ( radecbox[2] + radecbox[3]) # ADM 20 degrees should be a large enough radius for the sweeps. maxrad = 20. radecrad = centra, centdec, maxrad t2 = cuts.select_targets(infiles, numproc=1, tcnames=tc, radecrad=radecrad) # ADM test the pixel input. nside = pixarea2nside(box_area(radecbox)) pixlist = hp_in_box(nside, radecbox) t3 = cuts.select_targets(infiles, numproc=1, tcnames=tc, nside=nside, pixlist=pixlist) # ADM sort each set of targets on TARGETID to compare them. targets = targets[np.argsort(targets["TARGETID"])] t1 = t1[np.argsort(t1["TARGETID"])] t2 = t2[np.argsort(t2["TARGETID"])] t3 = t3[np.argsort(t3["TARGETID"])] # ADM test the same targets were recovered and that # ADM each recovered target has the same bits set. for targs in t1, t2, t3: for col in "TARGETID", "DESI_TARGET", "BGS_TARGET", "MWS_TARGET": self.assertTrue(np.all(targs[col] == targets[col]))
def test_select_targets(self): for filelist in [self.tractorfiles, self.sweepfiles]: targets = cuts.select_targets(filelist, numproc=1) t1 = cuts.select_targets(filelist[0:1], numproc=1) t2 = cuts.select_targets(filelist[0], numproc=1) for col in t1.dtype.names: try: notNaN = ~np.isnan(t1[col]) except TypeError: #- can't check string columns for NaN notNan = np.ones(len(t1), dtype=bool) self.assertTrue(np.all(t1[col][notNaN] == t2[col][notNaN]))
def test_select_targets(self): for filelist in [self.tractorfiles, self.sweepfiles]: targets = cuts.select_targets(filelist, numproc=1) t1 = cuts.select_targets(filelist[0:1], numproc=1) t2 = cuts.select_targets(filelist[0], numproc=1) for col in t1.dtype.names: try: notNaN = ~np.isnan(t1[col]) except TypeError: #- can't check string columns for NaN notNan = np.ones(len(t1), dtype=bool) self.assertTrue(np.all(t1[col][notNaN]==t2[col][notNaN]))
def test_qso_selection_options(self): """Test the QSO selection options are passed correctly """ tc = ["QSO"] targetfile = self.tractorfiles[0] for qso_selection in cuts.qso_selection_options: results = cuts.select_targets(targetfile, tcnames=tc, qso_selection=qso_selection) with self.assertRaises(ValueError): results = cuts.select_targets(targetfile, numproc=1, tcnames=tc, qso_selection='blatfoo')
def test_qso_selection_options(self): """Test the QSO selection options are passed correctly """ tc = ["QSO"] targetfile = self.tractorfiles[0] for qso_selection in cuts.qso_selection_options: # ADM set backup to False as the Gaia unit test # ADM files only cover a limited pixel range. results = cuts.select_targets(targetfile, backup=False, tcnames=tc, qso_selection=qso_selection) with self.assertRaises(ValueError): results = cuts.select_targets(targetfile, numproc=1, backup=False, tcnames=tc, qso_selection='blatfoo')
def main(): ns = ap.parse_args() t0 = time() objects = read_mock_durham(ns.src_core, ns.src_photo) t1 = time() targetflags = select_targets(objects) keep = (targetflags !=0) targets = objects[keep] targetflag = targetflags[keep] t2 = time() numobs = calc_numobs(targets, targetflag) targets = desitarget.targets.finalize(targets, targetflag, numobs) t3 = time() write_targets(ns.dest, targets) t4 = time() if ns.verbose: print ('written {} targets to {}'.format(len(targets), ns.dest)) print('Read mock file {:.1f} sec'.format(t1-t0)) print('Make target selection {:.1f} sec'.format(t2-t1)) print('Compute numobs, finalize target selection {:.1f} sec'.format(t3-t2)) print('Write output {:.1f} sec'.format(t4-t3))
def test_missing_files(self): """Test the code will die gracefully if input files are missing """ with self.assertRaises(ValueError): targets = cuts.select_targets([ 'blat.foo1234', ], numproc=1)
def test_targets_spatial_inputs(self): """Test the code fails if more than one spatial input is passed """ # ADM set up some fake inputs. pixlist = [0, 1] radecbox = [2, 3, 4, 5] radecrad = [6, 7, 8] # ADM we should throw an error every time we pass 2 inputs that aren't NoneType. timesthrown = 0 for i in range(3): inputs = [pixlist, radecbox, radecrad] inputs[i] = None try: cuts.select_targets(self.sweepfiles, numproc=1, pixlist=inputs[0], radecbox=inputs[1], radecrad=inputs[2]) except ValueError: timesthrown += 1 self.assertEqual(timesthrown, 3)
def test_sv_cuts(self): """Test SV cuts work. """ # ADM find all svX sub-directories in the desitarget directory. fns = glob(resource_filename('desitarget', 'sv*')) svlist = [os.path.basename(fn) for fn in fns if os.path.isdir(fn)] for survey in svlist: desicol, bgscol, mwscol = [ "{}_{}_TARGET".format(survey.upper(), tc) for tc in ["DESI", "BGS", "MWS"] ] for filelist in [self.tractorfiles, self.sweepfiles]: # ADM set backup to False as the Gaia unit test # ADM files only cover a limited pixel range. targets = cuts.select_targets(filelist, survey=survey, backup=False) for col in desicol, bgscol, mwscol: self.assertTrue(col in targets.dtype.names) self.assertEqual(len(targets), np.count_nonzero(targets[desicol])) # ADM this test should be fine as long as the main survey BGS # ADM bits don't get divorced from the SV survey bits. bgs1 = (targets[desicol] & desi_mask.BGS_ANY) != 0 bgs2 = targets[bgscol] != 0 self.assertTrue(np.all(bgs1 == bgs2)) # ADM backup targets can only be run on sweep files. for filelist in self.sweepfiles: # ADM also test the backup targets in the pixels covered # ADM by the Gaia unit test files. targets = cuts.select_targets(filelist, survey=survey, nside=self.nside, pixlist=self.pix) for col in desicol, bgscol, mwscol: self.assertTrue(col in targets.dtype.names) self.assertEqual(len(targets), np.count_nonzero(targets[desicol]))
def test_parallel_select(self): for nproc in [1,2]: for filelist in [self.tractorfiles, self.sweepfiles]: targets = cuts.select_targets(filelist, numproc=nproc) self.assertTrue('DESI_TARGET' in targets.dtype.names) self.assertTrue('BGS_TARGET' in targets.dtype.names) self.assertTrue('MWS_TARGET' in targets.dtype.names) self.assertEqual(len(targets), np.count_nonzero(targets['DESI_TARGET'])) bgs1 = (targets['DESI_TARGET'] & desi_mask.BGS_ANY) != 0 bgs2 = targets['BGS_TARGET'] != 0 self.assertTrue(np.all(bgs1 == bgs2))
def test_parallel_select(self): for nproc in [1, 2]: for filelist in [self.tractorfiles, self.sweepfiles]: targets = cuts.select_targets(filelist, numproc=nproc) self.assertTrue('DESI_TARGET' in targets.dtype.names) self.assertTrue('BGS_TARGET' in targets.dtype.names) self.assertTrue('MWS_TARGET' in targets.dtype.names) self.assertEqual(len(targets), np.count_nonzero(targets['DESI_TARGET'])) bgs1 = (targets['DESI_TARGET'] & desi_mask.BGS_ANY) != 0 bgs2 = targets['BGS_TARGET'] != 0 self.assertTrue(np.all(bgs1 == bgs2))
def test_select_targets_sandbox(self): from desitarget import sandbox ntot = 0 for filename in self.tractorfiles + self.sweepfiles: targets = cuts.select_targets(filename, numproc=1, sandbox=True) objects = Table.read(filename) if 'BRICK_PRIMARY' in objects.colnames: objects.remove_column('BRICK_PRIMARY') desi_target, bgs_target, mws_target = \ sandbox.cuts.apply_sandbox_cuts(objects) n = np.count_nonzero(desi_target) + \ np.count_nonzero(bgs_target) + \ np.count_nonzero(mws_target) self.assertEqual(len(targets), n) ntot += n self.assertGreater(ntot, 0, 'No targets selected by sandbox.cuts')
def test_parallel_select(self): """Test multiprocessing parallelization works """ # ADM only test the ELG, BGS cuts for speed. There's a # ADM full run through all classes in test_cuts_basic. tc = ["ELG", "BGS"] for nproc in [1, 2]: for filelist in [self.tractorfiles, self.sweepfiles]: targets = cuts.select_targets(filelist, numproc=nproc, tcnames=tc) self.assertTrue('DESI_TARGET' in targets.dtype.names) self.assertTrue('BGS_TARGET' in targets.dtype.names) self.assertTrue('MWS_TARGET' in targets.dtype.names) self.assertEqual(len(targets), np.count_nonzero(targets['DESI_TARGET'])) bgs1 = (targets['DESI_TARGET'] & desi_mask.BGS_ANY) != 0 bgs2 = targets['BGS_TARGET'] != 0 self.assertTrue(np.all(bgs1 == bgs2))
def test_backup(self): """Test BACKUP targets are selected. """ # ADM only test the ELG, BGS cuts for speed. There's a # ADM full run through all classes in test_cuts_basic. tc = ["ELG", "BGS"] # ADM BACKUP targets can only run on the sweep files. for filelist in self.sweepfiles: # ADM limit to pixels covered in the Gaia unit test files. targets = cuts.select_targets(filelist, numproc=1, tcnames=tc, nside=self.nside, pixlist=self.pix) self.assertTrue('DESI_TARGET' in targets.dtype.names) self.assertTrue('BGS_TARGET' in targets.dtype.names) self.assertTrue('MWS_TARGET' in targets.dtype.names) self.assertEqual(len(targets), np.count_nonzero(targets['DESI_TARGET'])) bgs1 = (targets['DESI_TARGET'] & desi_mask.BGS_ANY) != 0 bgs2 = targets['BGS_TARGET'] != 0 self.assertTrue(np.all(bgs1 == bgs2))
def test_sv_cuts(self): """Test SV cuts work. """ for filelist in [self.tractorfiles, self.sweepfiles]: # ADM increase maxsv as we add more iterations of SV!!! maxsv = 1 svlist = ['sv{}'.format(i) for i in range(1, maxsv+1)] for survey in svlist: desicol, bgscol, mwscol = ["{}_{}_TARGET".format(survey.upper(), tc) for tc in ["DESI", "BGS", "MWS"]] targets = cuts.select_targets(filelist, survey=survey) for col in desicol, bgscol, mwscol: self.assertTrue(col in targets.dtype.names) self.assertEqual(len(targets), np.count_nonzero(targets[desicol])) # ADM this test should be fine as long as the main survey BGS # ADM bits don't get divorced from the SV survey bits. bgs1 = (targets[desicol] & desi_mask.BGS_ANY) != 0 bgs2 = targets[bgscol] != 0 self.assertTrue(np.all(bgs1 == bgs2))
def test_missing_files(self): with self.assertRaises(ValueError): targets = cuts.select_targets(['blat.foo1234',], numproc=1)
def test_missing_files(self): with self.assertRaises(ValueError): targets = cuts.select_targets([ 'blat.foo1234', ], numproc=1)
#infiles = np.concatenate((infilessouth, infilesnorth)).tolist() infiles = infilessouth # import time start = time.time() def flux_to_mag(flux): mag = 22.5 - 2.5 * np.log10(flux) return mag targets = select_targets(infiles, numproc=4, mask=True, tcnames=["BGS"], survey='main', backup=False) bits = {'bright': 1, 'faint': 0, 'wise': 2} #['BGS_FAINT', 'BGS_BRIGHT', 'BGS_FAINT_EXT', 'BGS_LOWQ', 'BGS_FIBMAG', ] end = time.time() print('Total run time: %f sec' % (end - start)) rmag = flux_to_mag(targets['FLUX_R'] / targets['MW_TRANSMISSION_R']) for name, bit in zip(bits.keys(), bits.values()): bgsbit = ((targets['BGS_TARGET'] & 2**(bit)) != 0) print(name, np.sum(bgsbit)) print('rmag min: %f, rmag max: %f' % (rmag[bgsbit].min(), rmag[bgsbit].max()))