def _test_table_row(self, targets): """Test cuts work with tables from several I/O libraries """ # ADM only test the ELG cuts for speed. There's a # ADM full run through all classes in test_cuts_basic. tc = ["ELG"] # ADM add the DR7/DR8 data columns if they aren't there yet. # ADM can remove this once DR8 is finalized. if "MASKBITS" not in targets.dtype.names: targets = io.add_dr8_columns(targets) self.assertFalse(cuts._is_row(targets)) self.assertTrue(cuts._is_row(targets[0])) desi, bgs, mws = cuts.apply_cuts(targets, tcnames=tc) self.assertEqual(len(desi), len(targets)) self.assertEqual(len(bgs), len(targets)) self.assertEqual(len(mws), len(targets)) desi, bgs, mws = cuts.apply_cuts(targets[0], tcnames=tc) self.assertTrue(isinstance(desi, numbers.Integral), 'DESI_TARGET mask not an int') self.assertTrue(isinstance(bgs, numbers.Integral), 'BGS_TARGET mask not an int') self.assertTrue(isinstance(mws, numbers.Integral), 'MWS_TARGET mask not an int')
def test_cuts_basic(self): #- Cuts work with either data or filenames desi, bgs, mws = cuts.apply_cuts(self.tractorfiles[0]) desi, bgs, mws = cuts.apply_cuts(self.sweepfiles[0]) data = io.read_tractor(self.tractorfiles[0]) desi, bgs, mws = cuts.apply_cuts(data) data = io.read_tractor(self.sweepfiles[0]) desi, bgs, mws = cuts.apply_cuts(data)
def test_cuts_noprimary(self): #- cuts should work with or without "primary" targets = Table.read(self.sweepfiles[0]) desi1, bgs1, mws1 = cuts.apply_cuts(targets) targets.remove_column('BRICK_PRIMARY') desi2, bgs2, mws2 = cuts.apply_cuts(targets) self.assertTrue(np.all(desi1==desi2)) self.assertTrue(np.all(bgs1==bgs2)) self.assertTrue(np.all(mws1==mws2))
def test_cuts_noprimary(self): #- cuts should work with or without "primary" #- BRICK_PRIMARY was removed from the sweeps in dr3 (@moustakas) targets = Table.read(self.sweepfiles[0]) if 'BRICK_PRIMARY' in targets.colnames: desi1, bgs1, mws1 = cuts.apply_cuts(targets) targets.remove_column('BRICK_PRIMARY') desi2, bgs2, mws2 = cuts.apply_cuts(targets) self.assertTrue(np.all(desi1 == desi2)) self.assertTrue(np.all(bgs1 == bgs2)) self.assertTrue(np.all(mws1 == mws2))
def _test_table_row(self, targets): self.assertFalse(cuts._is_row(targets)) self.assertTrue(cuts._is_row(targets[0])) desi, bgs, mws = cuts.apply_cuts(targets) self.assertEqual(len(desi), len(targets)) self.assertEqual(len(bgs), len(targets)) self.assertEqual(len(mws), len(targets)) desi, bgs, mws = cuts.apply_cuts(targets[0]) self.assertTrue(isinstance(desi, int), 'DESI_TARGET mask not an int') self.assertTrue(isinstance(bgs, int), 'BGS_TARGET mask not an int') self.assertTrue(isinstance(mws, int), 'MWS_TARGET mask not an int')
def test_cuts_basic(self): """Test cuts work with either data or filenames """ # ADM only test the "BGS" class for speed. # ADM with one run of all target classes for coverage. tc = ["BGS"] desi, bgs, mws = cuts.apply_cuts(self.tractorfiles[0], tcnames=tc) desi, bgs, mws = cuts.apply_cuts(self.sweepfiles[0], tcnames=tc) data = io.read_tractor(self.tractorfiles[0]) desi, bgs, mws = cuts.apply_cuts(data, tcnames=tc) data = io.read_tractor(self.sweepfiles[0]) desi, bgs, mws = cuts.apply_cuts(data) bgs_any1 = (desi & desi_mask.BGS_ANY != 0) bgs_any2 = (bgs != 0) self.assertTrue(np.all(bgs_any1 == bgs_any2))
def _test_table_row(self, targets): self.assertFalse(cuts._is_row(targets)) self.assertTrue(cuts._is_row(targets[0])) desi, bgs, mws = cuts.apply_cuts(targets) self.assertEqual(len(desi), len(targets)) self.assertEqual(len(bgs), len(targets)) self.assertEqual(len(mws), len(targets)) desi, bgs, mws = cuts.apply_cuts(targets[0]) self.assertTrue(isinstance(desi, numbers.Integral), 'DESI_TARGET mask not an int') self.assertTrue(isinstance(bgs, numbers.Integral), 'BGS_TARGET mask not an int') self.assertTrue(isinstance(mws, numbers.Integral), 'MWS_TARGET mask not an int')
# from astropy.io import fits from desitarget.cuts import apply_cuts from desitarget.cmx import cmx_cuts from desitarget.io import read_tractor from desitarget.targets import finalize from desitarget.QA import _load_systematics # from desitarget.gaiamatch import find_gaia_files start = time() tractordir = '/global/cfs/cdirs/cosmo/data/legacysurvey/dr8/south/tractor/330/' # tractordir = '/project/projectdirs/cosmo/data/legacysurvey/dr7/tractor/330/' # tractordir = '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/tractor/330' # tractordir = '/data/legacysurvey/dr3.1/tractor/330/' for brick in ['3301m002', '3301m007', '3303p000']: filepath = '{}/tractor-{}.fits'.format(tractordir, brick) desi_target, bgs_target, mws_target = apply_cuts(filepath) # ADM as nobody is testing the MWS in the sandbox, yet, we need to # ADM ensure we ignore MWS targets for testing the main algorithms. yes = np.where((desi_target != 0) & (mws_target == 0))[0] no = np.where(desi_target == 0)[0] keep = np.concatenate([yes[0:3], no[0:3]]) data, hdr = read_tractor(filepath, header=True) # ADM the FRACDEV and FRACDEV_IVAR columns can # ADM contain some NaNs, which break testing. wnan = np.where(data["FRACDEV"] != data["FRACDEV"]) if len(wnan[0]) > 0: data["FRACDEV"][wnan] = 0. wnan = np.where(data["FRACDEV_IVAR"] != data["FRACDEV_IVAR"]) if len(wnan[0]) > 0: data["FRACDEV_IVAR"][wnan] = 0.
print('Returned {:d} files'.format(len(files))) for j, file in enumerate(files): all_paths.append( os.path.join(os.environ['DESI_DR2'], 'tractor', dir, file)) return all_paths path = get_file_paths(['034']) all_lrgs = Table() all_elgs = Table() all_qsos = Table() all_bgs = Table() for i in range(len(path)): desi_target, bgs_target, mws_target = cuts.apply_cuts(path[i]) t = Table.read(path[i]) lrgs = t[(desi_target & desi_mask.LRG).astype(bool)] elgs = t[(desi_target & desi_mask.ELG).astype(bool)] qsos = t[(desi_target & desi_mask.QSO).astype(bool)] bgs = t[(bgs_target & bgs_mask.BGS_BRIGHT).astype(bool)] all_lrgs = vstack([all_lrgs, lrgs]) all_elgs = vstack([all_elgs, elgs]) all_qsos = vstack([all_qsos, qsos]) all_bgs = vstack([all_bgs, bgs]) all_lrgs.write('lrg_cuts034.fits', format='fits') all_elgs.write('elg_cuts034.fits', format='fits') all_qsos.write('qso_cuts034.fits', format='fits')
#ADM as of DR4, we read in DR3 files and use desitarget.io #ADM to transform the format to the post-DR3 data model. #ADM Should eventually update to read in DR5 files directly from os.path import basename import numpy as np #from astropy.io import fits from desitarget.cuts import apply_cuts from desitarget.io import read_tractor import fitsio tractordir = '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/tractor/330' #tractordir = '/data/legacysurvey/dr3.1/tractor/330/' for brick in ['3301m002', '3301m007', '3303p000']: filepath = '{}/tractor-{}.fits'.format(tractordir, brick) desi_target = apply_cuts(filepath)[0] yes = np.where(desi_target != 0)[0] no = np.where(desi_target == 0)[0] keep = np.concatenate([yes[0:3], no[0:3]]) # data, hdr = fits.getdata(filepath, header=True) # fits.writeto('t/'+basename(filepath), data[keep], header=hdr) data, hdr = read_tractor(filepath, header=True) fitsio.write('t/' + basename(filepath), data[keep], header=hdr) sweepdir = '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1' #sweepdir = '/data/legacysurvey/dr2p/sweep/' for radec in ['310m005-320p000', '320m005-330p000', '330m005-340p000']: filepath = '{}/sweep-{}.fits'.format(sweepdir, radec) desi_target = apply_cuts(filepath)[0] yes = np.where(desi_target != 0)[0] no = np.where(desi_target == 0)[0]