def _test_table_row(self, targets):
"""Test cuts work with tables from several I/O libraries
"""
# ADM only test the ELG cuts for speed. There's a
# ADM full run through all classes in test_cuts_basic.
tc = ["ELG"]
# ADM add the DR7/DR8 data columns if they aren't there yet.
# ADM can remove this once DR8 is finalized.
if "MASKBITS" not in targets.dtype.names:
targets = io.add_dr8_columns(targets)
self.assertFalse(cuts._is_row(targets))
self.assertTrue(cuts._is_row(targets[0]))
desi, bgs, mws = cuts.apply_cuts(targets, tcnames=tc)
self.assertEqual(len(desi), len(targets))
self.assertEqual(len(bgs), len(targets))
self.assertEqual(len(mws), len(targets))
desi, bgs, mws = cuts.apply_cuts(targets[0], tcnames=tc)
self.assertTrue(isinstance(desi, numbers.Integral),
'DESI_TARGET mask not an int')
self.assertTrue(isinstance(bgs, numbers.Integral),
'BGS_TARGET mask not an int')
self.assertTrue(isinstance(mws, numbers.Integral),
'MWS_TARGET mask not an int')
def test_cuts_basic(self):
#- Cuts work with either data or filenames
desi, bgs, mws = cuts.apply_cuts(self.tractorfiles[0])
desi, bgs, mws = cuts.apply_cuts(self.sweepfiles[0])
data = io.read_tractor(self.tractorfiles[0])
desi, bgs, mws = cuts.apply_cuts(data)
data = io.read_tractor(self.sweepfiles[0])
desi, bgs, mws = cuts.apply_cuts(data)
def test_cuts_basic(self):
#- Cuts work with either data or filenames
desi, bgs, mws = cuts.apply_cuts(self.tractorfiles[0])
desi, bgs, mws = cuts.apply_cuts(self.sweepfiles[0])
data = io.read_tractor(self.tractorfiles[0])
desi, bgs, mws = cuts.apply_cuts(data)
data = io.read_tractor(self.sweepfiles[0])
desi, bgs, mws = cuts.apply_cuts(data)
def test_cuts_noprimary(self):
#- cuts should work with or without "primary"
targets = Table.read(self.sweepfiles[0])
desi1, bgs1, mws1 = cuts.apply_cuts(targets)
targets.remove_column('BRICK_PRIMARY')
desi2, bgs2, mws2 = cuts.apply_cuts(targets)
self.assertTrue(np.all(desi1==desi2))
self.assertTrue(np.all(bgs1==bgs2))
self.assertTrue(np.all(mws1==mws2))
def test_cuts_noprimary(self):
#- cuts should work with or without "primary"
#- BRICK_PRIMARY was removed from the sweeps in dr3 (@moustakas)
targets = Table.read(self.sweepfiles[0])
if 'BRICK_PRIMARY' in targets.colnames:
desi1, bgs1, mws1 = cuts.apply_cuts(targets)
targets.remove_column('BRICK_PRIMARY')
desi2, bgs2, mws2 = cuts.apply_cuts(targets)
self.assertTrue(np.all(desi1 == desi2))
self.assertTrue(np.all(bgs1 == bgs2))
self.assertTrue(np.all(mws1 == mws2))
def _test_table_row(self, targets):
self.assertFalse(cuts._is_row(targets))
self.assertTrue(cuts._is_row(targets[0]))
desi, bgs, mws = cuts.apply_cuts(targets)
self.assertEqual(len(desi), len(targets))
self.assertEqual(len(bgs), len(targets))
self.assertEqual(len(mws), len(targets))
desi, bgs, mws = cuts.apply_cuts(targets[0])
self.assertTrue(isinstance(desi, int), 'DESI_TARGET mask not an int')
self.assertTrue(isinstance(bgs, int), 'BGS_TARGET mask not an int')
self.assertTrue(isinstance(mws, int), 'MWS_TARGET mask not an int')
def test_cuts_basic(self):
"""Test cuts work with either data or filenames
"""
# ADM only test the "BGS" class for speed.
# ADM with one run of all target classes for coverage.
tc = ["BGS"]
desi, bgs, mws = cuts.apply_cuts(self.tractorfiles[0], tcnames=tc)
desi, bgs, mws = cuts.apply_cuts(self.sweepfiles[0], tcnames=tc)
data = io.read_tractor(self.tractorfiles[0])
desi, bgs, mws = cuts.apply_cuts(data, tcnames=tc)
data = io.read_tractor(self.sweepfiles[0])
desi, bgs, mws = cuts.apply_cuts(data)
bgs_any1 = (desi & desi_mask.BGS_ANY != 0)
bgs_any2 = (bgs != 0)
self.assertTrue(np.all(bgs_any1 == bgs_any2))
def _test_table_row(self, targets):
self.assertFalse(cuts._is_row(targets))
self.assertTrue(cuts._is_row(targets[0]))
desi, bgs, mws = cuts.apply_cuts(targets)
self.assertEqual(len(desi), len(targets))
self.assertEqual(len(bgs), len(targets))
self.assertEqual(len(mws), len(targets))
desi, bgs, mws = cuts.apply_cuts(targets[0])
self.assertTrue(isinstance(desi, numbers.Integral),
'DESI_TARGET mask not an int')
self.assertTrue(isinstance(bgs, numbers.Integral),
'BGS_TARGET mask not an int')
self.assertTrue(isinstance(mws, numbers.Integral),
'MWS_TARGET mask not an int')
# from astropy.io import fits
from desitarget.cuts import apply_cuts
from desitarget.cmx import cmx_cuts
from desitarget.io import read_tractor
from desitarget.targets import finalize
from desitarget.QA import _load_systematics
# from desitarget.gaiamatch import find_gaia_files
start = time()
tractordir = '/global/cfs/cdirs/cosmo/data/legacysurvey/dr8/south/tractor/330/'
# tractordir = '/project/projectdirs/cosmo/data/legacysurvey/dr7/tractor/330/'
# tractordir = '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/tractor/330'
# tractordir = '/data/legacysurvey/dr3.1/tractor/330/'
for brick in ['3301m002', '3301m007', '3303p000']:
filepath = '{}/tractor-{}.fits'.format(tractordir, brick)
desi_target, bgs_target, mws_target = apply_cuts(filepath)
# ADM as nobody is testing the MWS in the sandbox, yet, we need to
# ADM ensure we ignore MWS targets for testing the main algorithms.
yes = np.where((desi_target != 0) & (mws_target == 0))[0]
no = np.where(desi_target == 0)[0]
keep = np.concatenate([yes[0:3], no[0:3]])
data, hdr = read_tractor(filepath, header=True)
# ADM the FRACDEV and FRACDEV_IVAR columns can
# ADM contain some NaNs, which break testing.
wnan = np.where(data["FRACDEV"] != data["FRACDEV"])
if len(wnan[0]) > 0:
data["FRACDEV"][wnan] = 0.
wnan = np.where(data["FRACDEV_IVAR"] != data["FRACDEV_IVAR"])
if len(wnan[0]) > 0:
data["FRACDEV_IVAR"][wnan] = 0.
print('Returned {:d} files'.format(len(files)))
for j, file in enumerate(files):
all_paths.append(
os.path.join(os.environ['DESI_DR2'], 'tractor', dir, file))
return all_paths
path = get_file_paths(['034'])
all_lrgs = Table()
all_elgs = Table()
all_qsos = Table()
all_bgs = Table()
for i in range(len(path)):
desi_target, bgs_target, mws_target = cuts.apply_cuts(path[i])
t = Table.read(path[i])
lrgs = t[(desi_target & desi_mask.LRG).astype(bool)]
elgs = t[(desi_target & desi_mask.ELG).astype(bool)]
qsos = t[(desi_target & desi_mask.QSO).astype(bool)]
bgs = t[(bgs_target & bgs_mask.BGS_BRIGHT).astype(bool)]
all_lrgs = vstack([all_lrgs, lrgs])
all_elgs = vstack([all_elgs, elgs])
all_qsos = vstack([all_qsos, qsos])
all_bgs = vstack([all_bgs, bgs])
all_lrgs.write('lrg_cuts034.fits', format='fits')
all_elgs.write('elg_cuts034.fits', format='fits')
all_qsos.write('qso_cuts034.fits', format='fits')
#ADM as of DR4, we read in DR3 files and use desitarget.io
#ADM to transform the format to the post-DR3 data model.
#ADM Should eventually update to read in DR5 files directly
from os.path import basename
import numpy as np
#from astropy.io import fits
from desitarget.cuts import apply_cuts
from desitarget.io import read_tractor
import fitsio
tractordir = '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/tractor/330'
#tractordir = '/data/legacysurvey/dr3.1/tractor/330/'
for brick in ['3301m002', '3301m007', '3303p000']:
filepath = '{}/tractor-{}.fits'.format(tractordir, brick)
desi_target = apply_cuts(filepath)[0]
yes = np.where(desi_target != 0)[0]
no = np.where(desi_target == 0)[0]
keep = np.concatenate([yes[0:3], no[0:3]])
# data, hdr = fits.getdata(filepath, header=True)
# fits.writeto('t/'+basename(filepath), data[keep], header=hdr)
data, hdr = read_tractor(filepath, header=True)
fitsio.write('t/' + basename(filepath), data[keep], header=hdr)
sweepdir = '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1'
#sweepdir = '/data/legacysurvey/dr2p/sweep/'
for radec in ['310m005-320p000', '320m005-330p000', '330m005-340p000']:
filepath = '{}/sweep-{}.fits'.format(sweepdir, radec)
desi_target = apply_cuts(filepath)[0]
yes = np.where(desi_target != 0)[0]
no = np.where(desi_target == 0)[0]
