def test_cuts_basic(self): #- Cuts work with either data or filenames desi, bgs, mws = cuts.apply_cuts(self.tractorfiles[0]) desi, bgs, mws = cuts.apply_cuts(self.sweepfiles[0]) data = io.read_tractor(self.tractorfiles[0]) desi, bgs, mws = cuts.apply_cuts(data) data = io.read_tractor(self.sweepfiles[0]) desi, bgs, mws = cuts.apply_cuts(data)
def test_tractor_columns(self): tscolumns = io.tscolumns + ['BRICK_PRIMARY',] tractorfile = io.list_tractorfiles(self.datadir)[0] data = io.read_tractor(tractorfile) self.assertEqual(set(data.dtype.names), set(tscolumns)) columns = ['BX', 'BY'] data = io.read_tractor(tractorfile, columns=columns) self.assertEqual(set(data.dtype.names), set(columns)) data = io.read_tractor(tractorfile, columns=tuple(columns)) self.assertEqual(set(data.dtype.names), set(columns))
def test_readwrite_tractor(self): tractorfile = io.list_tractorfiles(self.datadir)[0] sweepfile = io.list_sweepfiles(self.datadir)[0] data = io.read_tractor(sweepfile) data = io.read_tractor(tractorfile) self.assertEqual(len(data), 6) #- test data has 6 objects per file data, hdr = io.read_tractor(tractorfile, header=True) self.assertEqual(len(data), 6) #- test data has 6 objects per file io.write_targets(self.testfile, data, indir=self.datadir) d2, h2 = fits.getdata(self.testfile, header=True) self.assertEqual(h2['DEPVER02'], self.datadir) self.assertEqual(data.dtype.names, d2.dtype.names) for column in data.dtype.names: self.assertTrue(np.all(data[column] == d2[column]))
def _select_targets_file(filename): '''Returns targets in filename that pass the cuts''' from desitarget import io objects = io.read_tractor(filename, columns=columns) keep = apply_cuts(objects) return io.fix_tractor_dr1_dtype(objects[keep])
def _get_bright_stars(filename): '''Retrieves bright stars from a sweeps/Tractor file''' objs = io.read_tractor(filename) #ADM write the fluxes as an array instead of as named columns fluxes = objs[bandnames].view( objs[bandnames].dtype[0]).reshape(objs[bandnames].shape + (-1, )) #ADM Retain rows for which ANY band is brighter than maglim w = np.where(np.any(fluxes > fluxlim, axis=1)) if len(w[0]) > 0: return objs[w]
def test_unextinct_fluxes(self): targets = io.read_tractor(self.tractorfiles[0]) t1 = cuts.unextinct_fluxes(targets) self.assertTrue(isinstance(t1, np.ndarray)) t2 = cuts.unextinct_fluxes(Table(targets)) self.assertTrue(isinstance(t2, Table)) for col in ['GFLUX', 'RFLUX', 'ZFLUX', 'W1FLUX', 'W2FLUX', 'WFLUX']: self.assertIn(col, t1.dtype.names) self.assertIn(col, t2.dtype.names) self.assertTrue(np.all(t1[col] == t2[col]))
def test_unextinct_fluxes(self): targets = io.read_tractor(self.tractorfiles[0]) t1 = cuts.unextinct_fluxes(targets) self.assertTrue(isinstance(t1, np.ndarray)) t2 = cuts.unextinct_fluxes(Table(targets)) self.assertTrue(isinstance(t2, Table)) for col in ['GFLUX', 'RFLUX', 'ZFLUX', 'W1FLUX', 'W2FLUX']: self.assertIn(col, t1.dtype.names) self.assertIn(col, t2.dtype.names) self.assertTrue(np.all(t1[col] == t2[col]))
def _get_bright_sources(filename): """Retrieves bright sources from a sweeps/Tractor file""" objs = io.read_tractor(filename) # ADM write the fluxes as an array instead of as named columns. # ADM Retain rows for which ANY band is brighter than maglim. ok = np.zeros(objs[bandnames[0]].shape, dtype=bool) for i, bandname in enumerate(bandnames): ok |= (objs[bandname] > fluxlim[i]) w = np.where(ok) if len(w[0]) > 0: return objs[w]
def test_cuts_basic(self): """Test cuts work with either data or filenames """ # ADM test for tractor files. # ADM No QSO cuts for speed. This doesn't affect coverage. cmx, pshift = cuts.apply_cuts(self.tractorfiles[0], cmxdir=self.cmxdir, noqso=True) data = io.read_tractor(self.tractorfiles[0]) cmx2, pshift2 = cuts.apply_cuts(data, cmxdir=self.cmxdir, noqso=True) self.assertTrue(np.all(cmx == cmx2)) self.assertTrue(np.all(pshift == pshift2)) # ADM test for sweeps files. # ADM No QSO cuts for speed. This doesn't affect coverage. cmx, pshift = cuts.apply_cuts(self.sweepfiles[0], cmxdir=self.cmxdir, noqso=True) data = io.read_tractor(self.sweepfiles[0]) cmx2, pshift2 = cuts.apply_cuts(data, cmxdir=self.cmxdir, noqso=True) self.assertTrue(np.all(cmx == cmx2)) self.assertTrue(np.all(pshift == pshift2))
def test_fix_dr1(self): '''test the DR1 TYPE dype fix (make everything S4)''' # - First, break it files = io.list_sweepfiles(self.datadir) objects = io.read_tractor(files[0]) dt = objects.dtype.descr for i in range(len(dt)): if dt[i][0] == 'TYPE': dt[i] = ('TYPE', 'S10') break badobjects = objects.astype(np.dtype(dt)) newobjects = io.fix_tractor_dr1_dtype(badobjects) self.assertEqual(newobjects['TYPE'].dtype, np.dtype('S4'))
def test_fix_dr1(self): '''test the DR1 TYPE dype fix (make everything S4)''' #- First, break it files = io.list_sweepfiles(self.datadir) objects = io.read_tractor(files[0]) dt = objects.dtype.descr for i in range(len(dt)): if dt[i][0] == 'TYPE': dt[i] = ('TYPE', 'S10') break badobjects = objects.astype(np.dtype(dt)) newobjects = io.fix_tractor_dr1_dtype(badobjects) self.assertEqual(newobjects['TYPE'].dtype, np.dtype('S4'))
def apply_cuts(objects): """Apply the cuts we want.""" #zfaint = 22.0 #keep = np.where((np.sum((objects['DECAM_NOBS'][:,[1,2,4]]>=3)*1,axis=1)==3)* # (np.sum((objects['DECAM_ANYMASK'][:,[1,2,4]]>0)*1,axis=1)==0)* # (objects['DECAM_FLUX'][:,4]<(10**(-0.4*(zfaint-22.5)))))[0] if isinstance(objects, (str, unicode)): from desitarget import io objects = io.read_tractor(objects) #- ensure uppercase column names if astropy Table if isinstance(objects, (Table, Row)): for col in objects.columns.itervalues(): if not col.name.isupper(): col.name = col.name.upper() #- undo Milky Way extinction flux = unextinct_fluxes(objects) gflux = flux['GFLUX'] rflux = flux['RFLUX'] zflux = flux['ZFLUX'] w1flux = flux['W1FLUX'] wflux = flux['WFLUX'] #- DR1 has targets off the edge of the brick; trim to just this brick try: primary = objects['BRICK_PRIMARY'] except (KeyError, ValueError): if _is_row(objects): primary = True else: primary = np.ones_like(objects, dtype=bool) blue = isBLUE(primary=primary, gflux=gflux, zflux=zflux, rflux=rflux) #keep = np.where((blue==1))[0] #keep = np.where((blue==1)* # (np.sum((objects['DECAM_ANYMASK'][:,[1,2,4]]>0)*1,axis=1)==0))[0] keep = np.where( (blue == 1) * (np.sum( (objects['DECAM_ANYMASK'][:, [1, 2, 4]] > 0) * 1, axis=1) == 0) * (np.sum( (objects['DECAM_FLUX'][:, [1, 2, 4]] > 0) * 1, axis=1) == 3))[0] return keep
def apply_cuts(objects): """Apply the cuts we want.""" #zfaint = 22.0 #keep = np.where((np.sum((objects['DECAM_NOBS'][:,[1,2,4]]>=3)*1,axis=1)==3)* # (np.sum((objects['DECAM_ANYMASK'][:,[1,2,4]]>0)*1,axis=1)==0)* # (objects['DECAM_FLUX'][:,4]<(10**(-0.4*(zfaint-22.5)))))[0] if isinstance(objects, (str, unicode)): from desitarget import io objects = io.read_tractor(objects) #- ensure uppercase column names if astropy Table if isinstance(objects, (Table, Row)): for col in objects.columns.itervalues(): if not col.name.isupper(): col.name = col.name.upper() #- undo Milky Way extinction flux = unextinct_fluxes(objects) gflux = flux['GFLUX'] rflux = flux['RFLUX'] zflux = flux['ZFLUX'] w1flux = flux['W1FLUX'] wflux = flux['WFLUX'] #- DR1 has targets off the edge of the brick; trim to just this brick try: primary = objects['BRICK_PRIMARY'] except (KeyError, ValueError): if _is_row(objects): primary = True else: primary = np.ones_like(objects, dtype=bool) blue = isBLUE(primary=primary, gflux=gflux, zflux=zflux, rflux=rflux) #keep = np.where((blue==1))[0] #keep = np.where((blue==1)* # (np.sum((objects['DECAM_ANYMASK'][:,[1,2,4]]>0)*1,axis=1)==0))[0] keep = np.where((blue==1)* (np.sum((objects['DECAM_ANYMASK'][:,[1,2,4]]>0)*1,axis=1)==0)* (np.sum((objects['DECAM_FLUX'][:,[1,2,4]]>0)*1,axis=1)==3))[0] return keep
def _select_targets_file(filename): '''Returns targets in filename that pass the cuts''' from desitarget import io objects = io.read_tractor(filename) desi_target, bgs_target, mws_target = apply_cuts(objects) #- desi_target includes BGS_ANY and MWS_ANY, so we can filter just #- on desi_target != 0 keep = (desi_target != 0) objects = objects[keep] desi_target = desi_target[keep] bgs_target = bgs_target[keep] mws_target = mws_target[keep] #- Add *_target mask columns targets = desitarget.targets.finalize( objects, desi_target, bgs_target, mws_target) return io.fix_tractor_dr1_dtype(targets)
def _check_input_files(filename): '''Check for corrupted values in a file''' from functools import partial from os.path import getsize #ADM read in Tractor or sweeps files objects = io.read_tractor(filename) #ADM if everything is OK the default meassage will be "OK" filemessageroot = 'OK' filemessageend = '' #ADM columns that shouldn't have zero values cols = [ 'BRICKID', # 'RA_IVAR', 'DEC_IVAR', 'MW_TRANSMISSION_G', 'MW_TRANSMISSION_R', 'MW_TRANSMISSION_Z', # 'WISE_FLUX', # 'WISE_MW_TRANSMISSION','DCHISQ' ] #ADM for each of these columnes that shouldn't have zero values, #ADM loop through and look for zero values for colname in cols: if np.min(objects[colname]) == 0: filemessageroot = "WARNING...some values are zero for" filemessageend += " " + colname #ADM now, loop through entries in the file and search for 4096-byte #ADM blocks that are all zeros (a sign of corruption in file-writing) #ADM Note that fits files are padded by 2880 bytes, so we only want to #ADM process the file length (in bytes) - 2880 bytestop = getsize(filename) - 2880 with open(filename, 'rb') as f: for block_number, data in enumerate( iter(partial(f.read, 4096), b'')): if not any(data): if block_number * 4096 < bytestop: filemessageroot = "WARNING...some values are zero for" filemessageend += ' 4096-byte-block-#{0}'.format( block_number) return [filename, filemessageroot + filemessageend]
def _get_gaia_matches(fnwdir): '''wrapper on match_gaia_to_primary() given a file name''' # ADM extract the output file name. fn = os.path.basename(fnwdir) outfile = '{}/{}'.format(outdir, fn.replace(".fits", ender)) # ADM read in the objects. objs, hdr = io.read_tractor(fnwdir, header=True) # ADM match to Gaia sources. gaiainfo = match_gaia_to_primary(objs) log.info('Done with Gaia match for {} primary objects...t = {:.1f}s' .format(len(objs), time()-start)) # ADM remove the GAIA_RA, GAIA_DEC columns as they aren't # ADM in the imaging surveys data model. gaiainfo = pop_gaia_coords(gaiainfo) # ADM add the Gaia column information to the sweeps array. for col in gaiainfo.dtype.names: objs[col] = gaiainfo[col] fitsio.write(outfile, objs, extname='SWEEP', header=hdr, clobber=True) return True
def _select_targets_file(filename): '''Returns targets in filename that pass the cuts''' objects = io.read_tractor(filename) cmx_target, priority_shift = apply_cuts(objects, cmxdir=cmxdir) return _finalize_targets(objects, cmx_target, priority_shift)
def apply_cuts(objects, cmxdir=None): """Perform commissioning (cmx) target selection on objects, return target mask arrays Parameters ---------- objects: numpy structured array with UPPERCASE columns needed for target selection, OR a string tractor/sweep filename cmxdir : :class:`str`, optional, defaults to :envvar:`CMX_DIR` Directory in which to find commmissioning files to which to match, such as the CALSPEC stars. If not specified, the cmx directory is taken to be the value of the :envvar:`CMX_DIR` environment variable. Returns ------- :class:`~numpy.ndarray` commissioning target selection bitmask flags for each object See desitarget.cmx.cmx_targetmask.cmx_mask for the definition of each bit """ # -Check if objects is a filename instead of the actual data if isinstance(objects, str): objects = io.read_tractor(objects) # -Ensure uppercase column names if astropy Table if isinstance(objects, (Table, Row)): for col in list(objects.columns.values()): if not col.name.isupper(): col.name = col.name.upper() # ADM retrieve/check the cmxdir. cmxdir = _get_cmxdir(cmxdir) # ADM As we need the column names. colnames = _get_colnames(objects) photsys_north, photsys_south, obs_rflux, gflux, rflux, zflux, \ w1flux, w2flux, rfiberflux, objtype, release, gfluxivar, rfluxivar, zfluxivar, \ gnobs, rnobs, znobs, gfracflux, rfracflux, zfracflux, \ gfracmasked, rfracmasked, zfracmasked, \ gfracin, rfracin, zfracin, gallmask, rallmask, zallmask, \ gsnr, rsnr, zsnr, w1snr, w2snr, dchisq, deltaChi2, brightstarinblob = \ _prepare_optical_wise(objects, colnames=colnames) # ADM in addition, cmx needs ra and dec. ra, dec = objects["RA"], objects["DEC"] # ADM Currently only coded for objects with Gaia matches # ADM (e.g. DR6 or above). Fail for earlier Data Releases. if np.any(release < 6000): log.critical('Commissioning cuts only coded for DR6 or above') raise ValueError if (np.max(objects['PMRA']) == 0.) & np.any(release < 7000): d = "/project/projectdirs/desi/target/gaia_dr2_match_dr6" log.info("Zero objects have a proper motion.") log.critical( "Did you mean to send the Gaia-matched sweeps in, e.g., {}?". format(d)) raise IOError # Process the Gaia inputs for target selection. gaia, pmra, pmdec, parallax, parallaxovererror, parallaxerr, gaiagmag, gaiabmag, \ gaiarmag, gaiaaen, gaiadupsource, Grr, gaiaparamssolved, gaiabprpfactor, \ gaiasigma5dmax, galb = _prepare_gaia(objects, colnames=colnames) # ADM a couple of extra columns; the observed g/z fluxes. obs_gflux, obs_zflux = objects['FLUX_G'], objects['FLUX_Z'] # ADM initially, every object passes the cuts (is True). # ADM need to guard against the case of a single row being passed. # ADM initially every class has a priority shift of zero. if _is_row(objects): primary = np.bool_(True) priority_shift = np.array(0) else: primary = np.ones_like(objects, dtype=bool) priority_shift = np.zeros_like(objects, dtype=int) # ADM determine if an object passes the default logic for cmx stars. isgood = passesSTD_logic(gfracflux=gfracflux, rfracflux=rfracflux, zfracflux=zfracflux, objtype=objtype, gaia=gaia, pmra=pmra, pmdec=pmdec, aen=gaiaaen, dupsource=gaiadupsource, paramssolved=gaiaparamssolved, primary=primary) # ADM determine if an object is a "dither" star. std_dither, shift_dither = isSTD_dither(obs_gflux=obs_gflux, obs_rflux=obs_rflux, obs_zflux=obs_zflux, isgood=isgood, primary=primary) # ADM set up an initial priority shift. # ADM determine if an object is a bright test star. std_test = isSTD_test(obs_gflux=obs_gflux, obs_rflux=obs_rflux, obs_zflux=obs_zflux, isgood=isgood, primary=primary) # ADM determine if an object matched a CALSPEC standard. std_calspec = isSTD_calspec(ra=ra, dec=dec, cmxdir=cmxdir, primary=primary) # ADM determine if an object is SV0_STD_BRIGHT. Resembles first # ADM iteration of SV, but locked in cmx_cuts (and could be altered). sv0_std_bright = isSV0_STD_bright(gflux=gflux, rflux=rflux, zflux=zflux, pmra=pmra, pmdec=pmdec, parallax=parallax, gaiagmag=gaiagmag, isgood=isgood, primary=primary) # ADM determine if an object is SV0_BGS sv0_bgs = isSV0_BGS(rflux=rflux, objtype=objtype, primary=primary) # ADM determine if an object is SV0_MWS sv0_mws = isSV0_MWS(rflux=rflux, obs_rflux=obs_rflux, objtype=objtype, gaiagmag=gaiagmag, gaiabmag=gaiabmag, gaiarmag=gaiarmag, pmra=pmra, pmdec=pmdec, parallax=parallax, parallaxovererror=parallaxovererror, photbprpexcessfactor=gaiabprpfactor, astrometricsigma5dmax=gaiasigma5dmax, galb=galb, gaia=gaia, primary=primary) # ADM Construct the targetflag bits. cmx_target = std_dither * cmx_mask.STD_GAIA cmx_target |= std_test * cmx_mask.STD_TEST cmx_target |= std_calspec * cmx_mask.STD_CALSPEC cmx_target |= sv0_std_bright * cmx_mask.SV0_STD_BRIGHT cmx_target |= sv0_bgs * cmx_mask.SV0_BGS cmx_target |= sv0_mws * cmx_mask.SV0_MWS # ADM update the priority with any shifts. # ADM we may need to update this logic if there are other shifts. priority_shift[std_dither] = shift_dither[std_dither] return cmx_target, priority_shift
def apply_cuts(objects, qso_selection='randomforest'): """Perform target selection on objects, returning target mask arrays Args: objects: numpy structured array with UPPERCASE columns needed for target selection, OR a string tractor/sweep filename Options: qso_selection : algorithm to use for QSO selection; valid options are 'colorcuts' and 'randomforest' Returns: (desi_target, bgs_target, mws_target) where each element is an ndarray of target selection bitmask flags for each object Bugs: If objects is a astropy Table with lowercase column names, this converts them to UPPERCASE in-place, thus modifying the input table. To avoid this, pass in objects.copy() instead. See desitarget.targetmask for the definition of each bit """ #- Check if objects is a filename instead of the actual data if isinstance(objects, str): objects = io.read_tractor(objects) #- ensure uppercase column names if astropy Table if isinstance(objects, (Table, Row)): for col in list(objects.columns.values()): if not col.name.isupper(): col.name = col.name.upper() obs_rflux = objects[ 'FLUX_R'] # observed r-band flux (used for F standards, below) #- undo Milky Way extinction flux = unextinct_fluxes(objects) gflux = flux['GFLUX'] rflux = flux['RFLUX'] zflux = flux['ZFLUX'] w1flux = flux['W1FLUX'] w2flux = flux['W2FLUX'] objtype = objects['TYPE'] gfluxivar = objects['FLUX_IVAR_G'] gfracflux = objects['FRACFLUX_G'].T # note transpose rfracflux = objects['FRACFLUX_R'].T # note transpose zfracflux = objects['FRACFLUX_Z'].T # note transpose gsnr = objects['FLUX_G'] * np.sqrt(objects['FLUX_IVAR_G']) rsnr = objects['FLUX_R'] * np.sqrt(objects['FLUX_IVAR_R']) zsnr = objects['FLUX_Z'] * np.sqrt(objects['FLUX_IVAR_Z']) w1snr = objects['FLUX_W1'] * np.sqrt(objects['FLUX_IVAR_W1']) w2snr = objects['FLUX_W2'] * np.sqrt(objects['FLUX_IVAR_W2']) # Delta chi2 between PSF and SIMP morphologies; note the sign.... dchisq = objects['DCHISQ'] deltaChi2 = dchisq[..., 0] - dchisq[..., 1] #ADM remove handful of NaN values from DCHISQ values and make them unselectable w = np.where(deltaChi2 != deltaChi2) #ADM this is to catch the single-object case for unit tests if len(w[0]) > 0: deltaChi2[w] = -1e6 #- DR1 has targets off the edge of the brick; trim to just this brick try: primary = objects['BRICK_PRIMARY'] except (KeyError, ValueError): if _is_row(objects): primary = True else: primary = np.ones_like(objects, dtype=bool) lrg = isLRG(primary=primary, gflux=gflux, rflux=rflux, zflux=zflux, w1flux=w1flux, gflux_ivar=gfluxivar, rflux_snr=rsnr, zflux_snr=zsnr, w1flux_snr=w1snr) elg = isELG(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux) bgs_bright = isBGS_bright(primary=primary, rflux=rflux, objtype=objtype) bgs_faint = isBGS_faint(primary=primary, rflux=rflux, objtype=objtype) if qso_selection == 'colorcuts': qso = isQSO_cuts(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux, w1flux=w1flux, w2flux=w2flux, deltaChi2=deltaChi2, objtype=objtype, w1snr=w1snr, w2snr=w2snr) elif qso_selection == 'randomforest': qso = isQSO_randomforest(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux, w1flux=w1flux, w2flux=w2flux, deltaChi2=deltaChi2, objtype=objtype) else: raise ValueError( 'Unknown qso_selection {}; valid options are {}'.format( qso_selection, qso_selection_options)) #ADM Make sure to pass all of the needed columns! At one point we stopped #ADM passing objtype, which meant no standards were being returned. fstd = isFSTD(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux, gfracflux=gfracflux, rfracflux=rfracflux, zfracflux=zfracflux, gsnr=gsnr, rsnr=rsnr, zsnr=zsnr, obs_rflux=obs_rflux, objtype=objtype) fstd_bright = isFSTD(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux, gfracflux=gfracflux, rfracflux=rfracflux, zfracflux=zfracflux, gsnr=gsnr, rsnr=rsnr, zsnr=zsnr, obs_rflux=obs_rflux, objtype=objtype, bright=True) # Construct the targetflag bits; currently our only cuts are DECam based # (i.e. South). This should really be refactored into a dedicated function. desi_target = lrg * desi_mask.LRG_SOUTH desi_target |= elg * desi_mask.ELG_SOUTH desi_target |= qso * desi_mask.QSO_SOUTH desi_target |= lrg * desi_mask.LRG desi_target |= elg * desi_mask.ELG desi_target |= qso * desi_mask.QSO # Standards; still need to set STD_WD desi_target |= fstd * desi_mask.STD_FSTAR desi_target |= fstd_bright * desi_mask.STD_BRIGHT # BGS, bright and faint bgs_target = bgs_bright * bgs_mask.BGS_BRIGHT bgs_target |= bgs_bright * bgs_mask.BGS_BRIGHT_SOUTH bgs_target |= bgs_faint * bgs_mask.BGS_FAINT bgs_target |= bgs_faint * bgs_mask.BGS_FAINT_SOUTH # Nothing for MWS yet; will be GAIA-based. if isinstance(bgs_target, numbers.Integral): mws_target = 0 else: mws_target = np.zeros_like(bgs_target) # Are any BGS or MWS bit set? Tell desi_target too. desi_target |= (bgs_target != 0) * desi_mask.BGS_ANY desi_target |= (mws_target != 0) * desi_mask.MWS_ANY return desi_target, bgs_target, mws_target
def apply_cuts(objects): """Perform target selection on objects, returning target mask arrays Args: objects: numpy structured array with UPPERCASE columns needed for target selection, OR a string tractor/sweep filename Returns: (desi_target, bgs_target, mws_target) where each element is an ndarray of target selection bitmask flags for each object Bugs: If objects is a astropy Table with lowercase column names, this converts them to UPPERCASE in-place, thus modifying the input table. To avoid this, pass in objects.copy() instead. See desitarget.targetmask for the definition of each bit """ #- Check if objects is a filename instead of the actual data if isinstance(objects, (str, unicode)): from desitarget import io objects = io.read_tractor(objects) #- ensure uppercase column names if astropy Table if isinstance(objects, (Table, Row)): for col in objects.columns.itervalues(): if not col.name.isupper(): col.name = col.name.upper() #- undo Milky Way extinction flux = unextinct_fluxes(objects) gflux = flux['GFLUX'] rflux = flux['RFLUX'] zflux = flux['ZFLUX'] w1flux = flux['W1FLUX'] w2flux = flux['W2FLUX'] objtype = objects['TYPE'] wise_snr = objects['WISE_FLUX'] * np.sqrt(objects['WISE_FLUX_IVAR']) #- DR1 has targets off the edge of the brick; trim to just this brick try: primary = objects['BRICK_PRIMARY'] except (KeyError, ValueError): if _is_row(objects): primary = True else: primary = np.ones_like(objects, dtype=bool) lrg = isLRG(primary=primary, zflux=zflux, rflux=rflux, w1flux=w1flux) elg = isELG(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux) bgs = isBGS(primary=primary, rflux=rflux, objtype=objtype) qso = isQSO(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux, w1flux=w1flux, w2flux=w2flux, objtype=objtype, wise_snr=wise_snr) #----- Standard stars fstd = isFSTD_colors(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux) fstd &= psflike(objtype) fracflux = objects['DECAM_FRACFLUX'].T signal2noise = objects['DECAM_FLUX'] * np.sqrt(objects['DECAM_FLUX_IVAR']) with warnings.catch_warnings(): # FIXME: what warnings are we ignoring? warnings.simplefilter('ignore') for j in (1,2,4): #- g, r, z fstd &= fracflux[j] < 0.04 fstd &= signal2noise[..., j] > 10 #- observed flux; no Milky Way extinction obs_rflux = objects['DECAM_FLUX'][..., 2] fstd &= obs_rflux < 10**((22.5-16.0)/2.5) fstd &= obs_rflux > 10**((22.5-19.0)/2.5) #----- #- construct the targetflag bits #- Currently our only cuts are DECam based (i.e. South) desi_target = lrg * desi_mask.LRG_SOUTH desi_target |= elg * desi_mask.ELG_SOUTH desi_target |= qso * desi_mask.QSO_SOUTH desi_target |= lrg * desi_mask.LRG desi_target |= elg * desi_mask.ELG desi_target |= qso * desi_mask.QSO desi_target |= fstd * desi_mask.STD_FSTAR bgs_target = bgs * bgs_mask.BGS_BRIGHT bgs_target |= bgs * bgs_mask.BGS_BRIGHT_SOUTH #- nothing for MWS yet; will be GAIA-based if isinstance(bgs_target, int): mws_target = 0 else: mws_target = np.zeros_like(bgs_target) #- Are any BGS or MWS bit set? Tell desi_target too. desi_target |= (bgs_target != 0) * desi_mask.BGS_ANY desi_target |= (mws_target != 0) * desi_mask.MWS_ANY return desi_target, bgs_target, mws_target
def make_bright_star_mask( bands, maglim, numproc=4, rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1', infilename=None, outfilename=None, verbose=False): """Make a bright star mask from a structure of bright stars drawn from the sweeps Parameters ---------- bands : :class:`str` A magnitude band from the sweeps, e.g., "G", "R", "Z". Can pass multiple bands as string, e.g. "GRZ", in which case maglim has to be a list of the same length as the string maglim : :class:`float` The upper limit in that magnitude band for which to assemble a list of bright stars. Can pass a list of magnitude limits, in which case bands has to be a string of the same length (e.g., "GRZ" for [12.3,12.7,12.6] numproc : :class:`int`, optional Number of processes over which to parallelize rootdirname : :class:`str`, optional, defaults to dr3 Root directory containing either sweeps or tractor files...e.g. for dr3 this might be /global/project/projectdirs/cosmo/data/legacysurvey/dr3/sweeps/dr3.1 infilename : :class:`str`, optional, if this exists, then the list of bright stars is read in from the file of this name if this is not passed, then code defaults to deriving the recarray of bright stars via a call to collect_bright_stars outfilename : :class:`str`, optional, defaults to not writing anything to file (FITS) File name to which to write the output bright star mask verbose : :class:`bool`, optional Send to write progress to screen Returns ------- :class:`recarray` The bright star mask in the form RA, DEC, TARGETID, IN_RADIUS, NEAR_RADIUS (may also be written to file if "outfilename" is passed) The radii are in ARCMINUTES TARGETID is as calculated in :mod:`desitarget.targets.encode_targetid` Notes ----- - IN_RADIUS is a smaller radius that corresponds to the IN_BRIGHT_OBJECT bit in data/targetmask.yaml - NEAR_RADIUS is a radius that corresponds to the NEAR_BRIGHT_OBJECT bit in data/targetmask.yaml - Currently uses the radius-as-a-function-of-B-mag for Tycho stars from the BOSS mask (in every band) to set the NEAR_RADIUS: R = (0.0802B*B - 1.860B + 11.625) (see Eqn. 9 of https://arxiv.org/pdf/1203.6594.pdf) and half that radius to set the IN_RADIUS. - It's an open question as to what the correct radii are for DESI observations """ #ADM set bands to uppercase if passed as lower case bands = bands.upper() #ADM the band names and nobs columns as arrays instead of strings bandnames = np.array(["FLUX_" + band for band in bands]) nobsnames = np.array(["NOBS_" + band for band in bands]) #ADM force the input maglim to be a list (in case a single value was passed) if type(maglim) == type(16) or type(maglim) == type(16.): maglim = [maglim] if len(bandnames) != len(maglim): raise IOError( 'bands has to be the same length as maglim and {} does not equal {}' .format(len(bandnames), len(maglim))) #ADM change input magnitude(s) to a flux to test against fluxlim = 10.**((22.5 - np.array(maglim)) / 2.5) if infilename is not None: objs = io.read_tractor(infilename) else: objs = collect_bright_stars(bands, maglim, numproc, rootdirname, outfilename, verbose) #ADM write the fluxes and bands as arrays instead of named columns fluxes = objs[bandnames].view( objs[bandnames].dtype[0]).reshape(objs[bandnames].shape + (-1, )) nobs = objs[nobsnames].view( objs[nobsnames].dtype[0]).reshape(objs[nobsnames].shape + (-1, )) #ADM set any observations with NOBS = 0 to have small flux so glitches don't end up as bright star masks. w = np.where(nobs == 0) if len(w[0]) > 0: fluxes[w] = 0. #ADM limit to the passed faint limit w = np.where(np.any(fluxes > fluxlim, axis=1)) fluxes = fluxes[w] objs = objs[w] #ADM grab the (GRZ) magnitudes for observations #ADM and record only the largest flux (smallest magnitude) fluxmax = np.max(fluxes, axis=1) mags = 22.5 - 2.5 * np.log10(fluxmax) #ADM convert the largest magnitude into radii for "in" and "near" bright objects. This will require #ADM more consideration to determine the truly correct numbers for DESI near_radius = (0.0802 * mags * mags - 1.860 * mags + 11.625) in_radius = 0.5 * (0.0802 * mags * mags - 1.860 * mags + 11.625) #ADM calculate the TARGETID targetid = encode_targetid(objid=objs['OBJID'], brickid=objs['BRICKID'], release=objs['RELEASE']) #ADM create an output recarray that is just RA, Dec, TARGETID and the radius done = objs[['RA', 'DEC']].copy() done = rfn.append_fields(done, ["TARGETID", "IN_RADIUS", "NEAR_RADIUS"], [targetid, in_radius, near_radius], usemask=False, dtypes=['>i8', '<f8', '<f8']) if outfilename is not None: fitsio.write(outfilename, done, clobber=True) return done
def apply_cuts(objects, cmxdir=None, noqso=False): """Commissioning (cmx) target selection, return target mask arrays. Parameters ---------- objects: numpy structured array with UPPERCASE columns needed for target selection, OR a string tractor/sweep filename cmxdir : :class:`str`, optional, defaults to :envvar:`CMX_DIR` Directory to find commmissioning files to which to match, such as the CALSPEC stars. If not specified, the cmx directory is taken to be the value of :envvar:`CMX_DIR`. noqso : :class:`boolean`, optional, defaults to ``False`` If passed, do not run the quasar selection. All QSO bits will be set to zero. Intended use is to speed unit tests. Returns ------- :class:`~numpy.ndarray` commissioning target selection bitmask flags for each object. See desitarget.cmx.cmx_targetmask.cmx_mask for bit definitions. """ # -Check if objects is a filename instead of the actual data if isinstance(objects, str): objects = io.read_tractor(objects) # -Ensure uppercase column names if astropy Table if isinstance(objects, (Table, Row)): for col in list(objects.columns.values()): if not col.name.isupper(): col.name = col.name.upper() # ADM retrieve/check the cmxdir. cmxdir = _get_cmxdir(cmxdir) # ADM As we need the column names. colnames = _get_colnames(objects) photsys_north, photsys_south, obs_rflux, gflux, rflux, zflux, \ w1flux, w2flux, rfiberflux, objtype, release, \ gfluxivar, rfluxivar, zfluxivar, \ gnobs, rnobs, znobs, gfracflux, rfracflux, zfracflux, \ gfracmasked, rfracmasked, zfracmasked, \ gfracin, rfracin, zfracin, gallmask, rallmask, zallmask, \ gsnr, rsnr, zsnr, w1snr, w2snr, dchisq, deltaChi2, maskbits = \ _prepare_optical_wise(objects) # ADM in addition, cmx needs ra and dec. ra, dec = objects["RA"], objects["DEC"] # ADM Currently only coded for objects with Gaia matches # ADM (e.g. DR6 or above). Fail for earlier Data Releases. if np.any(release < 6000): log.critical('Commissioning cuts only coded for DR6 or above') raise ValueError if (np.max(objects['PMRA']) == 0.) & np.any(release < 7000): d = "/project/projectdirs/desi/target/gaia_dr2_match_dr6" log.info("Zero objects have a proper motion.") log.critical( "Did you mean to send the Gaia-matched sweeps in, e.g., {}?". format(d)) raise IOError # Process the Gaia inputs for target selection. gaia, pmra, pmdec, parallax, parallaxovererror, parallaxerr, gaiagmag, gaiabmag, \ gaiarmag, gaiaaen, gaiadupsource, Grr, gaiaparamssolved, gaiabprpfactor, \ gaiasigma5dmax, galb = _prepare_gaia(objects, colnames=colnames) # ADM a couple of extra columns; the observed g/z fluxes. obs_gflux, obs_zflux = objects['FLUX_G'], objects['FLUX_Z'] # ADM initially, every object passes the cuts (is True). # ADM need to guard against the case of a single row being passed. # ADM initially every class has a priority shift of zero. if _is_row(objects): primary = np.bool_(True) priority_shift = np.array(0) else: primary = np.ones_like(objects, dtype=bool) priority_shift = np.zeros_like(objects, dtype=int) # ADM determine if an object passes the default logic for cmx stars. isgood = passesSTD_logic(gfracflux=gfracflux, rfracflux=rfracflux, zfracflux=zfracflux, objtype=objtype, gaia=gaia, pmra=pmra, pmdec=pmdec, aen=gaiaaen, dupsource=gaiadupsource, paramssolved=gaiaparamssolved, primary=primary) # ADM determine if an object is a "dither" star. # ADM and priority shift. std_dither, shift_dither = isSTD_dither(obs_gflux=obs_gflux, obs_rflux=obs_rflux, obs_zflux=obs_zflux, isgood=isgood, primary=primary) # ADM determine if an object is a bright test star. std_test = isSTD_test(obs_gflux=obs_gflux, obs_rflux=obs_rflux, obs_zflux=obs_zflux, isgood=isgood, primary=primary) # ADM determine if an object matched a CALSPEC standard. std_calspec = isSTD_calspec(ra=ra, dec=dec, cmxdir=cmxdir, primary=primary) # ADM determine if an object is SV0_BGS. sv0_bgs = isSV0_BGS(rflux=rflux, objtype=objtype, primary=primary) # ADM determine if an object is SV0_MWS or WD. sv0_mws, sv0_wd = isSV0_MWS(rflux=rflux, obs_rflux=obs_rflux, objtype=objtype, gaiagmag=gaiagmag, gaiabmag=gaiabmag, gaiarmag=gaiarmag, pmra=pmra, pmdec=pmdec, parallax=parallax, parallaxerr=parallaxerr, parallaxovererror=parallaxovererror, photbprpexcessfactor=gaiabprpfactor, astrometricsigma5dmax=gaiasigma5dmax, gaiaaen=gaiaaen, paramssolved=gaiaparamssolved, galb=galb, gaia=gaia, primary=primary) # ADM determine if an object is SV0_LRG. sv0_lrg = isSV0_LRG(gflux=gflux, rflux=rflux, zflux=zflux, w1flux=w1flux, rflux_snr=rsnr, zflux_snr=zsnr, w1flux_snr=w1snr, gflux_ivar=gfluxivar, primary=primary) # ADM determine if an object is SV0_ELG. sv0_elg = isSV0_ELG(primary=primary, gflux=gflux, rflux=rflux, zflux=zflux, gsnr=gsnr, rsnr=rsnr, zsnr=zsnr, maskbits=maskbits) # ADM determine if an object is SV0_QSO. if noqso: # ADM don't run quasar cuts if requested, for speed. sv0_qso = ~primary else: sv0_qso = isSV0_QSO(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux, w1flux=w1flux, w2flux=w2flux, objtype=objtype, dchisq=dchisq, maskbits=maskbits) # ADM run the SV0 STD target types for both faint and bright. # ADM Make sure to pass all of the needed columns! At one point we stopped # ADM passing objtype, which meant no standards were being returned. sv0_std_classes = [] for bright in [False, True]: sv0_std_classes.append( isSV0_STD(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux, gfracflux=gfracflux, rfracflux=rfracflux, zfracflux=zfracflux, gfracmasked=gfracmasked, rfracmasked=rfracmasked, objtype=objtype, zfracmasked=zfracmasked, gnobs=gnobs, rnobs=rnobs, znobs=znobs, gfluxivar=gfluxivar, rfluxivar=rfluxivar, zfluxivar=zfluxivar, gaia=gaia, astrometricexcessnoise=gaiaaen, paramssolved=gaiaparamssolved, pmra=pmra, pmdec=pmdec, parallax=parallax, dupsource=gaiadupsource, gaiagmag=gaiagmag, gaiabmag=gaiabmag, gaiarmag=gaiarmag, bright=bright)) sv0_std_faint, sv0_std_bright = sv0_std_classes # ADM the nominal main survey cuts for standard stars. These are currently # ADM identical to the SV0 cuts, so treat accordingly: std_faint, std_bright = sv0_std_classes # ADM Construct the target flag bits. cmx_target = std_dither * cmx_mask.STD_GAIA cmx_target |= std_test * cmx_mask.STD_TEST cmx_target |= std_calspec * cmx_mask.STD_CALSPEC cmx_target |= sv0_std_faint * cmx_mask.SV0_STD_FAINT cmx_target |= sv0_std_bright * cmx_mask.SV0_STD_BRIGHT cmx_target |= sv0_bgs * cmx_mask.SV0_BGS cmx_target |= sv0_mws * cmx_mask.SV0_MWS cmx_target |= sv0_lrg * cmx_mask.SV0_LRG cmx_target |= sv0_elg * cmx_mask.SV0_ELG cmx_target |= sv0_qso * cmx_mask.SV0_QSO cmx_target |= sv0_wd * cmx_mask.SV0_WD cmx_target |= std_faint * cmx_mask.STD_FAINT cmx_target |= std_bright * cmx_mask.STD_BRIGHT # ADM update the priority with any shifts. # ADM we may need to update this logic if there are other shifts. priority_shift[std_dither] = shift_dither[std_dither] return cmx_target, priority_shift
def apply_sandbox_cuts(objects, FoMthresh=None, MethodELG='XD'): """Perform target selection on objects, returning target mask arrays Args: objects: numpy structured array with UPPERCASE columns needed for target selection, OR a string tractor/sweep filename FoMthresh: If this is passed, then run apply_XD_globalerror and return the Figure of Merits calculated for the ELGs in a file "FoM.fits" in the current working directory. MethodELG: Three methods available for ELGs XD: Extreme deconvolution RF_spectro: Random Forest trained with spectro z (VIPERS and DEEP2) RF_photo: Random Forest trained with photo z (HSC) Returns: (desi_target, bgs_target, mws_target) where each element is an ndarray of target selection bitmask flags for each object If FoMthresh is passed where FoM are the Figure of Merit values calculated by apply_XD_globalerror Bugs: If objects is a astropy Table with lowercase column names, this converts them to UPPERCASE in-place, thus modifying the input table. To avoid this, pass in objects.copy() instead. See desitarget.targetmask for the definition of each bit """ #- Check if objects is a filename instead of the actual data if isinstance(objects, str): from desitarget import io objects = io.read_tractor(objects) #- ensure uppercase column names if astropy Table if isinstance(objects, (Table, Row)): for col in list(objects.columns.values()): if not col.name.isupper(): col.name = col.name.upper() #- undo Milky Way extinction flux = unextinct_fluxes(objects) gflux = flux['GFLUX'] rflux = flux['RFLUX'] zflux = flux['ZFLUX'] w1flux = flux['W1FLUX'] w2flux = flux['W2FLUX'] objtype = objects['TYPE'] gflux_ivar = objects['FLUX_IVAR_G'] rflux_ivar = objects['FLUX_IVAR_R'] zflux_ivar = objects['FLUX_IVAR_Z'] gflux_snr = objects['FLUX_G'] * np.sqrt(objects['FLUX_IVAR_G']) rflux_snr = objects['FLUX_R'] * np.sqrt(objects['FLUX_IVAR_R']) zflux_snr = objects['FLUX_Z'] * np.sqrt(objects['FLUX_IVAR_Z']) w1flux_snr = objects['FLUX_W1'] * np.sqrt(objects['FLUX_IVAR_W1']) #- DR1 has targets off the edge of the brick; trim to just this brick try: primary = objects['BRICK_PRIMARY'] except (KeyError, ValueError): if _is_row(objects): primary = True else: primary = np.ones_like(objects, dtype=bool) lrg = isLRG_2016v3(gflux=gflux, rflux=rflux, zflux=zflux, w1flux=w1flux, gflux_ivar=gflux_ivar, rflux_snr=rflux_snr, zflux_snr=zflux_snr, w1flux_snr=w1flux_snr, primary=primary) if FoMthresh is not None: if (MethodELG == 'XD'): elg, FoM = apply_XD_globalerror( objects, FoMthresh, glim=23.8, rlim=23.4, zlim=22.4, gr_ref=0.5, rz_ref=0.5, reg_r=1e-4 / (0.025**2 * 0.05), f_i=[1., 1., 0., 0.25, 0., 0.25, 0.], gmin=21., gmax=24.) elif (MethodELG == 'RF_photo'): elg, FoM = isELG_randomforest(pcut=abs(FoMthresh), primary=primary, zflux=zflux, rflux=rflux, gflux=gflux, w1flux=w1flux, w2flux=w2flux, training='photo') elif (MethodELG == 'RF_spectro'): elg, FoM = isELG_randomforest(pcut=abs(FoMthresh), primary=primary, zflux=zflux, rflux=rflux, gflux=gflux, w1flux=w1flux, w2flux=w2flux, training='spectro') #- construct the targetflag bits #- Currently our only cuts are DECam based (i.e. South) desi_target = lrg * desi_mask.LRG_SOUTH #desi_target |= elg * desi_mask.ELG_SOUTH #desi_target |= qso * desi_mask.QSO_SOUTH desi_target |= lrg * desi_mask.LRG if FoMthresh is not None: desi_target |= elg * desi_mask.ELG #desi_target |= qso * desi_mask.QSO #desi_target |= fstd * desi_mask.STD_FSTAR bgs_target = np.zeros_like(desi_target) #bgs_target = bgs_bright * bgs_mask.BGS_BRIGHT #bgs_target |= bgs_bright * bgs_mask.BGS_BRIGHT_SOUTH #bgs_target |= bgs_faint * bgs_mask.BGS_FAINT #bgs_target |= bgs_faint * bgs_mask.BGS_FAINT_SOUTH #- nothing for MWS yet; will be GAIA-based #if isinstance(bgs_target, numbers.Integral): # mws_target = 0 #else: # mws_target = np.zeros_like(bgs_target) mws_target = np.zeros_like(desi_target) #- Are any BGS or MWS bit set? Tell desi_target too. desi_target |= (bgs_target != 0) * desi_mask.BGS_ANY desi_target |= (mws_target != 0) * desi_mask.MWS_ANY if FoMthresh is not None: keep = (desi_target != 0) write_fom_targets(objects[keep], FoM[keep], desi_target[keep], bgs_target[keep], mws_target[keep]) return desi_target, bgs_target, mws_target
# from desitarget.gaiamatch import find_gaia_files start = time() tractordir = '/global/cfs/cdirs/cosmo/data/legacysurvey/dr8/south/tractor/330/' # tractordir = '/project/projectdirs/cosmo/data/legacysurvey/dr7/tractor/330/' # tractordir = '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/tractor/330' # tractordir = '/data/legacysurvey/dr3.1/tractor/330/' for brick in ['3301m002', '3301m007', '3303p000']: filepath = '{}/tractor-{}.fits'.format(tractordir, brick) desi_target, bgs_target, mws_target = apply_cuts(filepath) # ADM as nobody is testing the MWS in the sandbox, yet, we need to # ADM ensure we ignore MWS targets for testing the main algorithms. yes = np.where((desi_target != 0) & (mws_target == 0))[0] no = np.where(desi_target == 0)[0] keep = np.concatenate([yes[0:3], no[0:3]]) data, hdr = read_tractor(filepath, header=True) # ADM the FRACDEV and FRACDEV_IVAR columns can # ADM contain some NaNs, which break testing. wnan = np.where(data["FRACDEV"] != data["FRACDEV"]) if len(wnan[0]) > 0: data["FRACDEV"][wnan] = 0. wnan = np.where(data["FRACDEV_IVAR"] != data["FRACDEV_IVAR"]) if len(wnan[0]) > 0: data["FRACDEV_IVAR"][wnan] = 0. # ADM the "CONTINUE" comment keyword is not yet implemented # ADM in fitsio, so delete it to prevent fitsio barfing on headers. hdr.delete("CONTINUE") fitsio.write('t/' + basename(filepath), data[keep],
def make_bright_source_mask( bands, maglim, numproc=4, rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr5/sweep/5.0', infilename=None, outfilename=None): """Make a mask of bright sources from a structure of bright sources drawn from the sweeps. Parameters ---------- bands : :class:`str` A magnitude band from the sweeps, e.g., "G", "R", "Z". Can pass multiple bands as string, e.g. ``"GRZ"``, in which case maglim has to be a list of the same length as the string. maglim : :class:`float` The upper limit in that magnitude band for which to assemble a list of bright sources. Can pass a list of magnitude limits, in which case bands has to be a string of the same length (e.g., ``"GRZ"`` for [12.3,12.7,12.6]). numproc : :class:`int`, optional Number of processes over which to parallelize. rootdirname : :class:`str`, optional, defaults to dr3 Root directory containing either sweeps or tractor files...e.g. for dr5 this might be ``/global/project/projectdirs/cosmo/data/legacysurvey/dr5/sweep/dr5.0``. This is only used if ``infilename`` is not passed. infilename : :class:`str`, optional, if this exists, then the list of bright sources is read in from the file of this name. if this is not passed, then code defaults to deriving the recarray of bright sources from ``rootdirname`` via a call to ``collect_bright_sources``. outfilename : :class:`str`, optional, defaults to not writing anything to file (FITS) File name to which to write the output bright source mask. Returns ------- :class:`recarray` - The bright source mask in the form ``RA`, ``DEC``, ``TARGETID``, ``IN_RADIUS``, ``NEAR_RADIUS``, ``E1``, ``E2``, ``TYPE`` (may also be written to file if ``outfilename`` is passed). - ``TARGETID`` is as calculated in :mod:`desitarget.targets.encode_targetid`. - The radii are in ARCSECONDS (they default to equivalents of half-light radii for ellipses). - ``E1`` and ``E2`` are the ellipticity components as defined at the bottom of, e.g.: http://legacysurvey.org/dr5/catalogs/. - ``TYPE`` is the ``TYPE`` from the sweeps files, see, e.g.: http://legacysurvey.org/dr5/files/#sweep-catalogs. Notes ----- - ``IN_RADIUS`` is a smaller radius that corresponds to the ``IN_BRIGHT_OBJECT`` bit in ``data/targetmask.yaml`` (and is in ARCSECONDS). - ``NEAR_RADIUS`` is a radius that corresponds to the ``NEAR_BRIGHT_OBJECT`` bit in ``data/targetmask.yaml`` (and is in ARCSECONDS). - Currently uses the radius-as-a-function-of-B-mag for Tycho stars from the BOSS mask (in every band) to set the ``NEAR_RADIUS``: R = (0.0802B*B - 1.860B + 11.625) (see Eqn. 9 of https://arxiv.org/pdf/1203.6594.pdf) and half that radius to set the ``IN_RADIUS``. We convert this from arcminutes to arcseconds. - It's an open question as to what the correct radii are for DESI observations. """ # ADM set bands to uppercase if passed as lower case. bands = bands.upper() # ADM the band names and nobs columns as arrays instead of strings. bandnames = np.array(["FLUX_" + band for band in bands]) nobsnames = np.array(["NOBS_" + band for band in bands]) # ADM force the input maglim to be a list (in case a single value was passed). if isinstance(maglim, int) or isinstance(maglim, float): maglim = [maglim] if len(bandnames) != len(maglim): msg = "bands has to be the same length as maglim and {} does not equal {}".format( len(bandnames), len(maglim)) raise IOError(msg) # ADM change input magnitude(s) to a flux to test against. fluxlim = 10.**((22.5 - np.array(maglim)) / 2.5) if infilename is not None: objs = io.read_tractor(infilename) else: objs = collect_bright_sources(bands, maglim, numproc, rootdirname, outfilename) # ADM write the fluxes and bands as arrays instead of named columns # ADM to circumvent a numpy future warning, this requires two copies of the fluxes array. fluxcheck = objs[bandnames].view( objs[bandnames].dtype[0]).reshape(objs[bandnames].shape + (-1, )) fluxes = fluxcheck.copy() nobs = objs[nobsnames].view( objs[nobsnames].dtype[0]).reshape(objs[nobsnames].shape + (-1, )) # ADM set any observations with NOBS = 0 to have small flux so glitches don't end up as bright object masks. w = np.where(nobs == 0) if len(w[0]) > 0: fluxes[w] = 0. # ADM limit to the passed faint limit. w = np.where(np.any(fluxes > fluxlim, axis=1)) fluxes = fluxes[w] objs = objs[w] # ADM grab the (GRZ) magnitudes for observations # ADM and record only the largest flux (smallest magnitude). fluxmax = np.max(fluxes, axis=1) mags = 22.5 - 2.5 * np.log10(fluxmax) # ADM each object's TYPE. objtype = objs["TYPE"] # ADM calculate the TARGETID. targetid = encode_targetid(objid=objs['OBJID'], brickid=objs['BRICKID'], release=objs['RELEASE']) # ADM first set the shape parameters assuming everything is an exponential # ADM this will correctly assign e1, e2 of 0 to things with zero shape. in_radius = objs['SHAPEEXP_R'] e1 = objs['SHAPEEXP_E1'] e2 = objs['SHAPEEXP_E2'] # ADM now to account for deVaucouleurs objects, or things that are dominated by # ADM deVaucouleurs profiles, update objects with a larger "DEV" than "EXP" radius. wdev = np.where(objs['SHAPEDEV_R'] > objs['SHAPEEXP_R']) if len(wdev[0]) > 0: in_radius[wdev] = objs[wdev]['SHAPEDEV_R'] e1[wdev] = objs[wdev]['SHAPEDEV_E1'] e2[wdev] = objs[wdev]['SHAPEDEV_E2'] # ADM finally use the Tycho radius (see the notes above) for PSF or star-like objects. # ADM More consideration will be needed to derive correct numbers for this for DESI!!! # ADM this calculation was for "near" Tycho objects and was in arcmin, so we convert # ADM it to arcsec and multiply it by infac (see the top of the module). tycho_in_radius = infac * (0.0802 * mags * mags - 1.860 * mags + 11.625) * 60. wpsf = np.where(_psflike(objtype)) in_radius[wpsf] = tycho_in_radius[wpsf] # ADM set "near" as a multiple of "in" radius using the factor at the top of the code. near_radius = in_radius * nearfac # ADM create an output recarray that is just RA, Dec, TARGETID and the radius. done = objs[['RA', 'DEC']].copy() done = rfn.append_fields( done, ["TARGETID", "IN_RADIUS", "NEAR_RADIUS", "E1", "E2", "TYPE"], [targetid, in_radius, near_radius, e1, e2, objtype], usemask=False, dtypes=['>i8', '>f4', '>f4', '>f4', '>f4', '|S4']) if outfilename is not None: fitsio.write(outfilename, done, clobber=True) return done