def test_readwrite_tractor(self): tractorfile = io.list_tractorfiles(self.datadir)[0] sweepfile = io.list_sweepfiles(self.datadir)[0] data = io.read_tractor(sweepfile) self.assertEqual(len(data), 6) #- test data has 6 objects per file data = io.read_tractor(tractorfile) self.assertEqual(len(data), 6) #- test data has 6 objects per file data, hdr = io.read_tractor(tractorfile, header=True) self.assertEqual(len(data), 6) #- test data has 6 objects per file #ADM check PHOTSYS got added in writing targets io.write_targets(self.testfile, data, indir=self.datadir) #ADM use fits read wrapper in io to correctly handle whitespace d2, h2 = io.whitespace_fits_read(self.testfile, header=True) self.assertEqual( list(data.dtype.names) + ["PHOTSYS"], list(d2.dtype.names)) #ADM check PHOTSYS and HPXPIXEL got added writing targets with NSIDE request io.write_targets(self.testfile, data, nside=64, indir=self.datadir) #ADM use fits read wrapper in io to correctly handle whitespace d2, h2 = io.whitespace_fits_read(self.testfile, header=True) self.assertEqual( list(data.dtype.names) + ["HPXPIXEL", "PHOTSYS"], list(d2.dtype.names)) for column in data.dtype.names: self.assertTrue(np.all(data[column] == d2[column]))
def test_cuts_basic(self): #- Cuts work with either data or filenames desi, bgs, mws = cuts.apply_cuts(self.tractorfiles[0]) desi, bgs, mws = cuts.apply_cuts(self.sweepfiles[0]) data = io.read_tractor(self.tractorfiles[0]) desi, bgs, mws = cuts.apply_cuts(data) data = io.read_tractor(self.sweepfiles[0]) desi, bgs, mws = cuts.apply_cuts(data)
def test_tractor_columns(self): tscolumns = list(io.tsdatamodel.dtype.names) + [ 'BRICK_PRIMARY', ] tractorfile = io.list_tractorfiles(self.datadir)[0] data = io.read_tractor(tractorfile) self.assertEqual(set(data.dtype.names), set(tscolumns)) # columns = ['BX', 'BY'] columns = ['RA', 'DEC'] data = io.read_tractor(tractorfile, columns=columns) self.assertEqual(set(data.dtype.names), set(columns)) data = io.read_tractor(tractorfile, columns=tuple(columns)) self.assertEqual(set(data.dtype.names), set(columns))
def setUp(self): #ADM at this nskymin you always seem to get at least 1 bad position self.nskymin = 5000000 self.navoid = 2. self.psfsize = psfsize #ADM location of input test file self.datadir = resource_filename('lvmtarget.test', 't') self.sweepfile = self.datadir + '/sweep-320m005-330p000.fits' #ADM read in the test sweeps file self.objs = io.read_tractor(self.sweepfile) #ADM need to ensure that one object has a large enough half-light radius #ADM to cover matching sky positions to larger objects self.objs['SHAPEDEV_R'][0] = (self.psfsize * self.navoid) + 1e-8 self.objs['SHAPEDEV_E1'][0] = -0.22389728 self.objs['SHAPEDEV_E2'][0] = 0.42635256 #ADM create a "maximum" search distance that is as large as the #ADM diagonal across all objects in the test sweeps file #ADM note that this is only close to correct because the test #ADM file is near 0o Declination cmax = SkyCoord( max(self.objs["RA"]) * u.degree, max(self.objs["DEC"]) * u.degree) cmin = SkyCoord( min(self.objs["RA"]) * u.degree, min(self.objs["DEC"]) * u.degree) self.maxrad = cmax.separation(cmin).arcsec
def _select_sandbox_targets_file(filename): '''Returns targets in filename that pass the sandbox cuts''' from lvmtarget.sandbox.cuts import apply_sandbox_cuts objects = io.read_tractor(filename) desi_target, bgs_target, mws_target = apply_sandbox_cuts(objects,FoMthresh,Method) return _finalize_targets(objects, desi_target, bgs_target, mws_target)
def _get_bright_stars(filename): '''Retrieves bright stars from a sweeps/Tractor file''' objs = io.read_tractor(filename) #ADM write the fluxes as an array instead of as named columns fluxes = objs[bandnames].view( objs[bandnames].dtype[0]).reshape(objs[bandnames].shape + (-1, )) #ADM Retain rows for which ANY band is brighter than maglim w = np.where(np.any(fluxes > fluxlim, axis=1)) if len(w[0]) > 0: return objs[w]
def test_unextinct_fluxes(self): targets = io.read_tractor(self.tractorfiles[0]) t1 = cuts.unextinct_fluxes(targets) self.assertTrue(isinstance(t1, np.ndarray)) t2 = cuts.unextinct_fluxes(Table(targets)) self.assertTrue(isinstance(t2, Table)) for col in ['GFLUX', 'RFLUX', 'ZFLUX', 'W1FLUX', 'W2FLUX']: self.assertIn(col, t1.dtype.names) self.assertIn(col, t2.dtype.names) self.assertTrue(np.all(t1[col] == t2[col]))
def test_fix_dr1(self): '''test the DR1 TYPE dype fix (make everything S4)''' #- First, break it files = io.list_sweepfiles(self.datadir) objects = io.read_tractor(files[0]) dt = objects.dtype.descr for i in range(len(dt)): if dt[i][0] == 'TYPE': dt[i] = ('TYPE', 'S10') break badobjects = objects.astype(np.dtype(dt)) newobjects = io.fix_tractor_dr1_dtype(badobjects) self.assertEqual(newobjects['TYPE'].dtype, np.dtype('S4'))
def _check_input_files(filename): '''Check for corrupted values in a file''' from functools import partial from os.path import getsize #ADM read in Tractor or sweeps files objects = io.read_tractor(filename) #ADM if everything is OK the default meassage will be "OK" filemessageroot = 'OK' filemessageend = '' #ADM columns that shouldn't have zero values cols = [ 'BRICKID', # 'RA_IVAR', 'DEC_IVAR', 'MW_TRANSMISSION_G', 'MW_TRANSMISSION_R', 'MW_TRANSMISSION_Z', # 'WISE_FLUX', # 'WISE_MW_TRANSMISSION','DCHISQ' ] #ADM for each of these columnes that shouldn't have zero values, #ADM loop through and look for zero values for colname in cols: if np.min(objects[colname]) == 0: filemessageroot = "WARNING...some values are zero for" filemessageend += " "+colname #ADM now, loop through entries in the file and search for 4096-byte #ADM blocks that are all zeros (a sign of corruption in file-writing) #ADM Note that fits files are padded by 2880 bytes, so we only want to #ADM process the file length (in bytes) - 2880 bytestop = getsize(filename) -2880 with open(filename, 'rb') as f: for block_number, data in enumerate(iter(partial(f.read, 4096), b'')): if not any(data): if block_number*4096 < bytestop: filemessageroot = "WARNING...some values are zero for" filemessageend += ' 4096-byte-block-#{0}'.format(block_number) return [filename,filemessageroot+filemessageend]
def calculate_separations(objs, navoid=2.): """Generate an array of separations (in arcseconds) for a set of objects Parameters ---------- objs : :class:`~numpy.ndarray` numpy structured array with UPPERCASE columns, OR a string tractor/sweep filename. Must contain at least the columns "RA", "DEC", "SHAPEDEV_R", "SHAPEEXP_R" navoid : :class:`float`, optional, defaults to 2. the number of times the galaxy half-light radius (or seeing) to avoid objects out to when placing sky fibers Returns ------- :class:`float` an array of maximum separations (in arcseconds) based on de Vaucouleurs, Exponential or point-source half-light radii """ #ADM check if input objs is a filename or the actual data if isinstance(objs, str): objs = io.read_tractor(objs) nobjs = len(objs) #ADM possible choices for separation based on de Vaucouleurs and Exponential profiles #ADM or a minimum of psfsize arcseconds for point sources ("the seeing") #ADM the default psfsize is supplied at the top of this code sepchoices = np.vstack( [objs["SHAPEDEV_R"], objs["SHAPEEXP_R"], np.ones(nobjs) * psfsize]).T #ADM the maximum separation from de Vaucoulers/exponential/PSF choices sep = navoid * np.max(sepchoices, axis=1) return sep
def make_sky_targets(objs, navoid=2., nskymin=None): """Generate sky targets and translate them into the typical format for DESI targets Parameters ---------- objs : :class:`~numpy.ndarray` numpy structured array with UPPERCASE columns, OR a string tractor/sweep filename. Must contain at least the columns "RA", "DEC", "SHAPEDEV_R", "SHAPEEXP_R" navoid : :class:`float`, optional, defaults to 2. the number of times the galaxy half-light radius (or seeing) to avoid objects out to when placing sky fibers nskymin : :class:`float`, optional, defaults to reading from lvmmodel.io the minimum DENSITY of sky fibers to generate Returns ------- :class:`~numpy.ndarray` a structured array of good and bad sky positions in the DESI target format """ #ADM initialize the default logger from lvmutil.log import get_logger, DEBUG log = get_logger(DEBUG) start = time() #ADM check if input objs is a filename or the actual data if isinstance(objs, str): objs = io.read_tractor(objs) log.info('Generating sky positions...t = {:.1f}s'.format(time() - start)) #ADM generate arrays of good and bad objects for this sweeps file (or set) ragood, decgood, rabad, decbad = generate_sky_positions(objs, navoid=navoid, nskymin=nskymin) ngood = len(ragood) nbad = len(rabad) nskies = ngood + nbad #ADM retrieve the standard DESI target array dt = io.tsdatamodel.dtype skies = np.zeros(nskies, dtype=dt) #ADM populate the output recarray with the RA/Dec of the good and bad sky locations skies["RA"][0:ngood], skies["DEC"][0:ngood] = ragood, decgood skies["RA"][ngood:nskies], skies["DEC"][ngood:nskies] = rabad, decbad #ADM create an array of target bits with the SKY information set desi_target = np.zeros(nskies, dtype='>i8') desi_target[0:ngood] |= desi_mask.SKY desi_target[ngood:nskies] |= desi_mask.BADSKY log.info('Looking up brick information...t = {:.1f}s'.format(time() - start)) #ADM add the brick information for the sky targets b = brick.Bricks(bricksize=0.25) skies["BRICKID"] = b.brickid(skies["RA"], skies["DEC"]) skies["BRICKNAME"] = b.brickname(skies["RA"], skies["DEC"]) #ADM set the data release from the passed sweeps objects dr = np.max(objs['RELEASE']) #ADM check the passed sweeps objects have the same release checker = np.min(objs['RELEASE']) if dr != checker: raise IOError( 'Multiple data releases present in same input sweeps objects?!') skies["RELEASE"] = dr #ADM set the objid (just use a sequential number as setting skies #ADM to 1 in the TARGETID will make these unique #ADM *MAKE SURE TO SET THE BRIGHT STAR SAFE LOCATIONS OF THE MAXIMUM SKY OBJID*!!! skies["OBJID"] = np.arange(nskies) log.info('Finalizing target bits...t = {:.1f}s'.format(time() - start)) #ADM add target bit columns to the output array, note that mws_target #ADM and bgs_target should be zeros for all sky objects dum = np.zeros_like(desi_target) skies = finalize(skies, desi_target, dum, dum, sky=1) log.info('Done...t = {:.1f}s'.format(time() - start)) return skies
def plot_sky_positions(ragood, decgood, rabad, decbad, objs, navoid=2., limits=None, plotname=None): """plot an example set of sky positions to check if they avoid real objects Parameters ---------- ragood : :class:`~numpy.array` array of RA coordinates for good sky positions decgood : :class:`~numpy.array` array of Dec coordinates for good sky positions rabad : :class:`~numpy.array` array of RA coordinates for bad sky positions, i.e. positions that ARE within the avoidance zones of the "objs" decbad : :class:`~numpy.array` array of Dec coordinates for bad sky positions, i.e. positions that ARE within the avoidance zones of the "objs" objs : :class:`~numpy.ndarray` numpy structured array with UPPERCASE columns, OR a string tractor/sweep filename. Must contain at least the columns "RA", "DEC", "SHAPEDEV_R", "SHAPEEXP_R" navoid : :class:`float`, optional, defaults to 2. the number of times the galaxy half-light radius (or seeing) that objects (objs) were avoided out to when generating sky positions limits : :class:`~numpy.array`, optional, defaults to None plot limits in the form [ramin, ramax, decmin, decmax] if None is passed, then the entire area is plotted plotname : :class:`str`, defaults to None If a name is passed use matplotlib's savefig command to save the plot to that file name. Otherwise, display the plot Returns ------- Nothing """ import matplotlib.pyplot as plt from lvmtarget.brightstar import ellipses from matplotlib.patches import Polygon from matplotlib.collections import PatchCollection #ADM initialize the default logger from lvmutil.log import get_logger, DEBUG log = get_logger(DEBUG) start = time() #ADM set up the figure and the axis labels fig, ax = plt.subplots(figsize=(8, 8)) ax.set_xlabel('RA (o)') ax.set_ylabel('Dec (o)') #ADM check if input objs is a filename or the actual data if isinstance(objs, str): objs = io.read_tractor(objs) #ADM coordinate limits for the passed objs ramin, ramax = np.min(objs["RA"]), np.max(objs["RA"]) decmin, decmax = np.min(objs["DEC"]), np.max(objs["DEC"]) #ADM guard against wraparound bug (which should never be an issue for the sweeps, anyway) if ramax - ramin > 180.: ramax -= 360. #ADM the avoidance separation (in arcseconds) for each object based on #ADM its half-light radius/profile sep = calculate_separations(objs, navoid) #ADM the maximum such separation for any object in the passed set IN DEGREES maxrad = max(sep) / 3600. #ADM limit the figure range based on the passed objs if limits is None: ralo, rahi = ramin, ramax declo, dechi = decmin, decmax else: ralo, rahi, declo, dechi = limits dum = plt.axis([ralo, rahi, declo, dechi]) #ADM plot good and bad sky positions ax.scatter(ragood, decgood, marker='.', facecolors='none', edgecolors='k') ax.scatter(rabad, decbad, marker='.', facecolors='r') #ADM the size that defines a PSF versus an elliptical avoidance zone sepsplit = (psfsize * navoid) + 1e-8 smallsepw = np.where(sep <= sepsplit)[0] bigsepw = np.where(sep > sepsplit)[0] #ADM first the PSF or "small separation objects"... #ADM set up the ellipse shapes based on sizes of the past avoidance zones #ADM remembering to stretch by the COS term to de-project the sky minoraxis = sep / 3600. majoraxis = minoraxis / np.cos(np.radians(objs["DEC"])) log.info('Plotting avoidance zones...t = {:.1f}s'.format(time() - start)) #ADM plot the avoidance zones as circles, stretched by their DEC position out = ellipses(objs[smallsepw]["RA"], objs[smallsepw]["DEC"], 2 * majoraxis[smallsepw], 2 * minoraxis[smallsepw], alpha=0.4, edgecolor='none') #ADM now the elliptical or "large separation objects"... #ADM loop through the DEV and EXP shapes, and create polygons #ADM of them to plot (where they're defined) patches = [] for i, valobj in enumerate(bigsepw): if objs[valobj]["SHAPEEXP_R"] > 0: #ADM points on the ellipse boundary for EXP objects ras, decs = ellipse_boundary(objs[valobj]["RA"], objs[valobj]["DEC"], objs[valobj]["SHAPEEXP_R"] * navoid, objs[valobj]["SHAPEEXP_E1"], objs[valobj]["SHAPEEXP_E2"]) polygon = Polygon(np.array(list(zip(ras, decs))), True) patches.append(polygon) if objs[valobj]["SHAPEDEV_R"] > 0: #ADM points on the ellipse boundary for DEV objects ras, decs = ellipse_boundary(objs[valobj]["RA"], objs[valobj]["DEC"], objs[valobj]["SHAPEDEV_R"] * navoid, objs[valobj]["SHAPEDEV_E1"], objs[valobj]["SHAPEDEV_E2"]) polygon = Polygon(np.array(list(zip(ras, decs))), True) patches.append(polygon) p = PatchCollection(patches, alpha=0.4, facecolors='b', edgecolors='b') ax.add_collection(p) #ADM display the plot, if requested if plotname is None: log.info('Displaying plot...t = {:.1f}s'.format(time() - start)) plt.show() else: plt.savefig(plotname) log.info('Done...t = {:.1f}s'.format(time() - start)) return
def generate_sky_positions(objs, navoid=2., nskymin=None): """Use a basic avoidance-of-other-objects approach to generate sky positions Parameters ---------- objs : :class:`~numpy.ndarray` numpy structured array with UPPERCASE columns, OR a string tractor/sweep filename. Must contain at least the columns "RA", "DEC", "SHAPEDEV_R", "SHAPEEXP_R" navoid : :class:`float`, optional, defaults to 2. the number of times the galaxy half-light radius (or seeing) to avoid objects out to when placing sky fibers nskymin : :class:`float`, optional, defaults to reading from lvmmodel.io the minimum DENSITY of sky fibers to generate Returns ------- ragood : :class:`~numpy.array` array of RA coordinates for good sky positions decgood : :class:`~numpy.array` array of Dec coordinates for good sky positions rabad : :class:`~numpy.array` array of RA coordinates for bad sky positions, i.e. positions that ARE within navoid half-light radii of a galaxy (or navoid*psfsize arcseconds for a PSF object) decbad : :class:`~numpy.array` array of Dec coordinates for bad sky positions, i.e. positions that ARE within navoid half-light radii of a galaxy (or navoid*psfsize arcseconds for a PSF object) """ #ADM set up the default log from lvmutil.log import get_logger, DEBUG log = get_logger(DEBUG) start = time() #ADM if needed, determine the minimum density of sky fibers to generate if nskymin is None: nskymin = density_of_sky_fibers() log.info( 'Generating sky positions at a density of {} per sq. deg....t = {:.1f}s' .format(nskymin, time() - start)) #ADM check if input objs is a filename or the actual data if isinstance(objs, str): objs = io.read_tractor(objs) nobjs = len(objs) #ADM an avoidance separation (in arcseconds) for each #ADM object based on its half-light radius/profile log.info('Calculating avoidance zones...t = {:.1f}s'.format(time() - start)) sep = calculate_separations(objs, navoid) #ADM the maximum such separation for any object in the passed set in arcsec maxrad = max(sep) #ADM the coordinate limits and corresponding area of the passed objs ramin, ramax = np.min(objs["RA"]), np.max(objs["RA"]) #ADM guard against the wraparound bug (should never be an issue for the sweeps, anyway) if ramax - ramin > 180.: ramax -= 360. decmin, decmax = np.min(objs["DEC"]), np.max(objs["DEC"]) sindecmin, sindecmax = np.sin(np.radians(decmin)), np.sin( np.radians(decmax)) spharea = (ramax - ramin) * np.degrees(sindecmax - sindecmin) log.info('Area covered by passed objects is {:.3f} sq. deg....t = {:.1f}s'. format(spharea, time() - start)) #ADM how many sky positions we need to generate to meet the minimum density requirements nskies = int(spharea * nskymin) #ADM how many sky positions to generate, given that we'll reject objects close to bad #ADM sources. The factor of 1.2 was derived by trial-and-error...but this doesn't need #ADM to be optimal as this algorithm should become more sophisticated nchunk = int(nskies * 1.2) #ADM arrays of GOOD sky positions to populate with coordinates ragood, decgood = np.empty(nskies), np.empty(nskies) #ADM lists of BAD sky positions to populate with coordinates rabad, decbad = [], [] #ADM ngenerate will become zero when we generate enough GOOD sky positions ngenerate = nskies while ngenerate: #ADM generate random points in RA and Dec (correctly distributed on the sphere) log.info('Generated {} test positions...t = {:.1f}s'.format( nchunk, time() - start)) ra = np.random.uniform(ramin, ramax, nchunk) dec = np.degrees( np.arcsin(1. - np.random.uniform(1 - sindecmax, 1 - sindecmin, nchunk))) #ADM set up the coordinate objects cskies = SkyCoord(ra * u.degree, dec * u.degree) cobjs = SkyCoord(objs["RA"] * u.degree, objs["DEC"] * u.degree) #ADM split the objects up using a separation of just larger than psfsize*navoid #ADM arcseconds in order to speed up the coordinate matching when we have some #ADM objects with large radii sepsplit = (psfsize * navoid) + 1e-8 bigsepw = np.where(sep > sepsplit)[0] smallsepw = np.where(sep <= sepsplit)[0] #ADM set up a list of skies that don't match an object goodskies = np.ones(len(cskies), dtype=bool) #ADM guard against the case where there are no objects with small radii if len(smallsepw) > 0: #ADM match the small-separation objects and flag any skies that match such an object log.info( 'Match positions out to {:.1f} arcsec...t = {:.1f}s'.format( sepsplit, time() - start)) idskies, idobjs, d2d, _ = cobjs[smallsepw].search_around_sky( cskies, sepsplit * u.arcsec) w = np.where(d2d.arcsec < sep[smallsepw[idobjs]]) #ADM remember to guard against the case with no bad positions if len(w[0]) > 0: goodskies[idskies[w]] = False #ADM guard against the case where there are no objects with large radii if len(bigsepw) > 0: #ADM match the large-separation objects and flag any skies that match such an object log.info( '(Elliptically) Match additional positions out to {:.1f} arcsec...t = {:.1f}s' .format(maxrad, time() - start)) #ADM the transformation matrixes (shapes) for DEV and EXP objects #ADM with a factor of navoid in the half-light-radius TDEV = ellipse_matrix(objs[bigsepw]["SHAPEDEV_R"] * navoid, objs[bigsepw]["SHAPEDEV_E1"], objs[bigsepw]["SHAPEDEV_E2"]) TEXP = ellipse_matrix(objs[bigsepw]["SHAPEEXP_R"] * navoid, objs[bigsepw]["SHAPEEXP_E1"], objs[bigsepw]["SHAPEEXP_E2"]) #ADM loop through the DEV and EXP shapes, and where they are defined #ADM (radius > tiny), determine if any sky positions occupy them for i, valobj in enumerate(bigsepw): if i % 1000 == 999: log.info('Done {}/{}...t = {:.1f}s'.format( i, len(bigsepw), time() - start)) is_in = np.array(np.zeros(nchunk), dtype='bool') if objs[valobj]["SHAPEEXP_R"] > 1e-16: is_in += is_in_ellipse_matrix(cskies.ra.deg, cskies.dec.deg, cobjs[valobj].ra.deg, cobjs[valobj].dec.deg, TEXP[..., i]) if objs[valobj]["SHAPEDEV_R"] > 1e-16: is_in += is_in_ellipse_matrix(cskies.ra.deg, cskies.dec.deg, cobjs[valobj].ra.deg, cobjs[valobj].dec.deg, TDEV[..., i]) w = np.where(is_in) if len(w[0]) > 0: goodskies[w] = False #ADM good sky positions we found wgood = np.where(goodskies)[0] n1 = nskies - ngenerate ngenerate = max(0, ngenerate - len(wgood)) n2 = nskies - ngenerate ragood[n1:n2] = ra[wgood[:n2 - n1]] decgood[n1:n2] = dec[wgood[:n2 - n1]] log.info( 'Need to generate a further {} positions...t = {:.1f}s'.format( ngenerate, time() - start)) #ADM bad sky positions we found wbad = np.where(~goodskies)[0] rabad.append(list(ra[wbad])) decbad.append(list(dec[wbad])) #ADM we potentially created nested lists for the bad skies, so need to flatten #ADM also we can't need more bad sky positions than total sky positions rabad = np.array([item for sublist in rabad for item in sublist])[:nskies] decbad = np.array([item for sublist in decbad for item in sublist])[:nskies] log.info('Done...t = {:.1f}s'.format(time() - start)) return ragood, decgood, rabad, decbad
def apply_sandbox_cuts(objects, FoMthresh=None, MethodELG='XD'): """Perform target selection on objects, returning target mask arrays Args: objects: numpy structured array with UPPERCASE columns needed for target selection, OR a string tractor/sweep filename FoMthresh: If this is passed, then run apply_XD_globalerror and return the Figure of Merits calculated for the ELGs in a file "FoM.fits" in the current working directory. MethodELG: Three methods available for ELGs XD: Extreme deconvolution RF_spectro: Random Forest trained with spectro z (VIPERS and DEEP2) RF_photo: Random Forest trained with photo z (HSC) Returns: (desi_target, bgs_target, mws_target) where each element is an ndarray of target selection bitmask flags for each object If FoMthresh is passed where FoM are the Figure of Merit values calculated by apply_XD_globalerror Bugs: If objects is a astropy Table with lowercase column names, this converts them to UPPERCASE in-place, thus modifying the input table. To avoid this, pass in objects.copy() instead. See lvmtarget.targetmask for the definition of each bit """ #- Check if objects is a filename instead of the actual data if isinstance(objects, str): from lvmtarget import io objects = io.read_tractor(objects) #- ensure uppercase column names if astropy Table if isinstance(objects, (Table, Row)): for col in list(objects.columns.values()): if not col.name.isupper(): col.name = col.name.upper() #- undo Milky Way extinction flux = unextinct_fluxes(objects) gflux = flux['GFLUX'] rflux = flux['RFLUX'] zflux = flux['ZFLUX'] w1flux = flux['W1FLUX'] w2flux = flux['W2FLUX'] objtype = objects['TYPE'] gflux_ivar = objects['FLUX_IVAR_G'] rflux_ivar = objects['FLUX_IVAR_R'] zflux_ivar = objects['FLUX_IVAR_Z'] gflux_snr = objects['FLUX_G'] * np.sqrt(objects['FLUX_IVAR_G']) rflux_snr = objects['FLUX_R'] * np.sqrt(objects['FLUX_IVAR_R']) zflux_snr = objects['FLUX_Z'] * np.sqrt(objects['FLUX_IVAR_Z']) w1flux_snr = objects['FLUX_W1'] * np.sqrt(objects['FLUX_IVAR_W1']) #- DR1 has targets off the edge of the brick; trim to just this brick try: primary = objects['BRICK_PRIMARY'] except (KeyError, ValueError): if _is_row(objects): primary = True else: primary = np.ones_like(objects, dtype=bool) lrg = isLRG_2016v3(gflux=gflux, rflux=rflux, zflux=zflux, w1flux=w1flux, gflux_ivar=gflux_ivar, rflux_snr=rflux_snr, zflux_snr=zflux_snr, w1flux_snr=w1flux_snr, primary=primary) if FoMthresh is not None: if (MethodELG == 'XD'): elg, FoM = apply_XD_globalerror( objects, FoMthresh, glim=23.8, rlim=23.4, zlim=22.4, gr_ref=0.5, rz_ref=0.5, reg_r=1e-4 / (0.025**2 * 0.05), f_i=[1., 1., 0., 0.25, 0., 0.25, 0.], gmin=21., gmax=24.) elif (MethodELG == 'RF_photo'): elg, FoM = isELG_randomforest(pcut=abs(FoMthresh), primary=primary, zflux=zflux, rflux=rflux, gflux=gflux, w1flux=w1flux, w2flux=w2flux, training='photo') elif (MethodELG == 'RF_spectro'): elg, FoM = isELG_randomforest(pcut=abs(FoMthresh), primary=primary, zflux=zflux, rflux=rflux, gflux=gflux, w1flux=w1flux, w2flux=w2flux, training='spectro') #- construct the targetflag bits #- Currently our only cuts are DECam based (i.e. South) desi_target = lrg * desi_mask.LRG_SOUTH #desi_target |= elg * desi_mask.ELG_SOUTH #desi_target |= qso * desi_mask.QSO_SOUTH desi_target |= lrg * desi_mask.LRG if FoMthresh is not None: desi_target |= elg * desi_mask.ELG #desi_target |= qso * desi_mask.QSO #desi_target |= fstd * desi_mask.STD_FSTAR bgs_target = np.zeros_like(desi_target) #bgs_target = bgs_bright * bgs_mask.BGS_BRIGHT #bgs_target |= bgs_bright * bgs_mask.BGS_BRIGHT_SOUTH #bgs_target |= bgs_faint * bgs_mask.BGS_FAINT #bgs_target |= bgs_faint * bgs_mask.BGS_FAINT_SOUTH #- nothing for MWS yet; will be GAIA-based #if isinstance(bgs_target, numbers.Integral): # mws_target = 0 #else: # mws_target = np.zeros_like(bgs_target) mws_target = np.zeros_like(desi_target) #- Are any BGS or MWS bit set? Tell desi_target too. desi_target |= (bgs_target != 0) * desi_mask.BGS_ANY desi_target |= (mws_target != 0) * desi_mask.MWS_ANY if FoMthresh is not None: keep = (desi_target != 0) write_fom_targets(objects[keep], FoM[keep], desi_target[keep], bgs_target[keep], mws_target[keep]) return desi_target, bgs_target, mws_target
def _select_targets_file(filename): '''Returns targets in filename that pass the cuts''' objects = io.read_tractor(filename) desi_target, bgs_target, mws_target = apply_cuts(objects, qso_selection) return _finalize_targets(objects, desi_target, bgs_target, mws_target)
def apply_cuts(objects, qso_selection='randomforest'): """Perform target selection on objects, returning target mask arrays Args: objects: numpy structured array with UPPERCASE columns needed for target selection, OR a string tractor/sweep filename Options: qso_selection : algorithm to use for QSO selection; valid options are 'colorcuts' and 'randomforest' Returns: (desi_target, bgs_target, mws_target) where each element is an ndarray of target selection bitmask flags for each object Bugs: If objects is a astropy Table with lowercase column names, this converts them to UPPERCASE in-place, thus modifying the input table. To avoid this, pass in objects.copy() instead. See lvmtarget.targetmask for the definition of each bit """ #- Check if objects is a filename instead of the actual data if isinstance(objects, str): objects = io.read_tractor(objects) #- ensure uppercase column names if astropy Table if isinstance(objects, (Table, Row)): for col in list(objects.columns.values()): if not col.name.isupper(): col.name = col.name.upper() obs_rflux = objects['FLUX_R'] # observed r-band flux (used for F standards, below) #- undo Milky Way extinction flux = unextinct_fluxes(objects) gflux = flux['GFLUX'] rflux = flux['RFLUX'] zflux = flux['ZFLUX'] w1flux = flux['W1FLUX'] w2flux = flux['W2FLUX'] objtype = objects['TYPE'] release = objects['RELEASE'] gfluxivar = objects['FLUX_IVAR_G'] gfracflux = objects['FRACFLUX_G'].T # note transpose rfracflux = objects['FRACFLUX_R'].T # note transpose zfracflux = objects['FRACFLUX_Z'].T # note transpose gsnr = objects['FLUX_G'] * np.sqrt(objects['FLUX_IVAR_G']) rsnr = objects['FLUX_R'] * np.sqrt(objects['FLUX_IVAR_R']) zsnr = objects['FLUX_Z'] * np.sqrt(objects['FLUX_IVAR_Z']) w1snr = objects['FLUX_W1'] * np.sqrt(objects['FLUX_IVAR_W1']) w2snr = objects['FLUX_W2'] * np.sqrt(objects['FLUX_IVAR_W2']) # Delta chi2 between PSF and SIMP morphologies; note the sign.... dchisq = objects['DCHISQ'] deltaChi2 = dchisq[...,0] - dchisq[...,1] #ADM remove handful of NaN values from DCHISQ values and make them unselectable w = np.where(deltaChi2 != deltaChi2) #ADM this is to catch the single-object case for unit tests if len(w[0]) > 0: deltaChi2[w] = -1e6 #- DR1 has targets off the edge of the brick; trim to just this brick try: primary = objects['BRICK_PRIMARY'] except (KeyError, ValueError): if _is_row(objects): primary = True else: primary = np.ones_like(objects, dtype=bool) lrg, lrg1pass, lrg2pass = isLRGpass(primary=primary, gflux=gflux, rflux=rflux, zflux=zflux, w1flux=w1flux, gflux_ivar=gfluxivar, rflux_snr=rsnr, zflux_snr=zsnr, w1flux_snr=w1snr) elg = isELG(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux) bgs_bright = isBGS_bright(primary=primary, rflux=rflux, objtype=objtype) bgs_faint = isBGS_faint(primary=primary, rflux=rflux, objtype=objtype) if qso_selection=='colorcuts' : qso = isQSO_cuts(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux, w1flux=w1flux, w2flux=w2flux, deltaChi2=deltaChi2, objtype=objtype, w1snr=w1snr, w2snr=w2snr, release=release) elif qso_selection == 'randomforest': qso = isQSO_randomforest(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux, w1flux=w1flux, w2flux=w2flux, deltaChi2=deltaChi2, objtype=objtype, release=release) else: raise ValueError('Unknown qso_selection {}; valid options are {}'.format(qso_selection, qso_selection_options)) #ADM Make sure to pass all of the needed columns! At one point we stopped #ADM passing objtype, which meant no standards were being returned. fstd = isFSTD(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux, gfracflux=gfracflux, rfracflux=rfracflux, zfracflux=zfracflux, gsnr=gsnr, rsnr=rsnr, zsnr=zsnr, obs_rflux=obs_rflux, objtype=objtype) fstd_bright = isFSTD(primary=primary, zflux=zflux, rflux=rflux, gflux=gflux, gfracflux=gfracflux, rfracflux=rfracflux, zfracflux=zfracflux, gsnr=gsnr, rsnr=rsnr, zsnr=zsnr, obs_rflux=obs_rflux, objtype=objtype, bright=True) # Construct the targetflag bits; currently our only cuts are DECam based # (i.e. South). This should really be refactored into a dedicated function. desi_target = lrg * desi_mask.LRG_SOUTH desi_target |= elg * desi_mask.ELG_SOUTH desi_target |= qso * desi_mask.QSO_SOUTH desi_target |= lrg * desi_mask.LRG desi_target |= elg * desi_mask.ELG desi_target |= qso * desi_mask.QSO #ADM add the per-pass information desi_target |= lrg1pass * desi_mask.LRG_1PASS desi_target |= lrg2pass * desi_mask.LRG_2PASS # Standards; still need to set STD_WD desi_target |= fstd * desi_mask.STD_FSTAR desi_target |= fstd_bright * desi_mask.STD_BRIGHT # BGS, bright and faint bgs_target = bgs_bright * bgs_mask.BGS_BRIGHT bgs_target |= bgs_bright * bgs_mask.BGS_BRIGHT_SOUTH bgs_target |= bgs_faint * bgs_mask.BGS_FAINT bgs_target |= bgs_faint * bgs_mask.BGS_FAINT_SOUTH # Nothing for MWS yet; will be GAIA-based. if isinstance(bgs_target, numbers.Integral): mws_target = 0 else: mws_target = np.zeros_like(bgs_target) # Are any BGS or MWS bit set? Tell desi_target too. desi_target |= (bgs_target != 0) * desi_mask.BGS_ANY desi_target |= (mws_target != 0) * desi_mask.MWS_ANY return desi_target, bgs_target, mws_target
#from astropy.io import fits from lvmtarget.cuts import apply_cuts from lvmtarget.io import read_tractor import fitsio tractordir = '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/tractor/330' #tractordir = '/data/legacysurvey/dr3.1/tractor/330/' for brick in ['3301m002', '3301m007', '3303p000']: filepath = '{}/tractor-{}.fits'.format(tractordir, brick) desi_target = apply_cuts(filepath)[0] yes = np.where(desi_target != 0)[0] no = np.where(desi_target == 0)[0] keep = np.concatenate([yes[0:3], no[0:3]]) # data, hdr = fits.getdata(filepath, header=True) # fits.writeto('t/'+basename(filepath), data[keep], header=hdr) data, hdr = read_tractor(filepath, header=True) fitsio.write('t/' + basename(filepath), data[keep], header=hdr) sweepdir = '/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1' #sweepdir = '/data/legacysurvey/dr2p/sweep/' for radec in ['310m005-320p000', '320m005-330p000', '330m005-340p000']: filepath = '{}/sweep-{}.fits'.format(sweepdir, radec) desi_target = apply_cuts(filepath)[0] yes = np.where(desi_target != 0)[0] no = np.where(desi_target == 0)[0] keep = np.concatenate([yes[0:3], no[0:3]]) # data, hdr = fits.getdata(filepath, header=True) # fits.writeto('t/'+basename(filepath), data[keep], header=hdr) data, hdr = read_tractor(filepath, header=True) fitsio.write('t/' + basename(filepath), data[keep], header=hdr)
def make_bright_star_mask( bands, maglim, numproc=4, rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1', infilename=None, outfilename=None): """Make a bright star mask from a structure of bright stars drawn from the sweeps Parameters ---------- bands : :class:`str` A magnitude band from the sweeps, e.g., "G", "R", "Z". Can pass multiple bands as string, e.g. "GRZ", in which case maglim has to be a list of the same length as the string maglim : :class:`float` The upper limit in that magnitude band for which to assemble a list of bright stars. Can pass a list of magnitude limits, in which case bands has to be a string of the same length (e.g., "GRZ" for [12.3,12.7,12.6] numproc : :class:`int`, optional Number of processes over which to parallelize rootdirname : :class:`str`, optional, defaults to dr3 Root directory containing either sweeps or tractor files...e.g. for dr3 this might be /global/project/projectdirs/cosmo/data/legacysurvey/dr3/sweeps/dr3.1 infilename : :class:`str`, optional, if this exists, then the list of bright stars is read in from the file of this name if this is not passed, then code defaults to deriving the recarray of bright stars via a call to collect_bright_stars outfilename : :class:`str`, optional, defaults to not writing anything to file (FITS) File name to which to write the output bright star mask Returns ------- :class:`recarray` The bright star mask in the form RA, DEC, TARGETID, IN_RADIUS, NEAR_RADIUS (may also be written to file if "outfilename" is passed) The radii are in ARCMINUTES TARGETID is as calculated in :mod:`lvmtarget.targets.encode_targetid` Notes ----- - IN_RADIUS is a smaller radius that corresponds to the IN_BRIGHT_OBJECT bit in data/targetmask.yaml - NEAR_RADIUS is a radius that corresponds to the NEAR_BRIGHT_OBJECT bit in data/targetmask.yaml - Currently uses the radius-as-a-function-of-B-mag for Tycho stars from the BOSS mask (in every band) to set the NEAR_RADIUS: R = (0.0802B*B - 1.860B + 11.625) (see Eqn. 9 of https://arxiv.org/pdf/1203.6594.pdf) and half that radius to set the IN_RADIUS. - It's an open question as to what the correct radii are for DESI observations """ #ADM set bands to uppercase if passed as lower case bands = bands.upper() #ADM the band names and nobs columns as arrays instead of strings bandnames = np.array(["FLUX_" + band for band in bands]) nobsnames = np.array(["NOBS_" + band for band in bands]) #ADM force the input maglim to be a list (in case a single value was passed) if type(maglim) == type(16) or type(maglim) == type(16.): maglim = [maglim] if len(bandnames) != len(maglim): raise IOError( 'bands has to be the same length as maglim and {} does not equal {}' .format(len(bandnames), len(maglim))) #ADM change input magnitude(s) to a flux to test against fluxlim = 10.**((22.5 - np.array(maglim)) / 2.5) if infilename is not None: objs = io.read_tractor(infilename) else: objs = collect_bright_stars(bands, maglim, numproc, rootdirname, outfilename) #ADM write the fluxes and bands as arrays instead of named columns fluxes = objs[bandnames].view( objs[bandnames].dtype[0]).reshape(objs[bandnames].shape + (-1, )) nobs = objs[nobsnames].view( objs[nobsnames].dtype[0]).reshape(objs[nobsnames].shape + (-1, )) #ADM set any observations with NOBS = 0 to have small flux so glitches don't end up as bright star masks. w = np.where(nobs == 0) if len(w[0]) > 0: fluxes[w] = 0. #ADM limit to the passed faint limit w = np.where(np.any(fluxes > fluxlim, axis=1)) fluxes = fluxes[w] objs = objs[w] #ADM grab the (GRZ) magnitudes for observations #ADM and record only the largest flux (smallest magnitude) fluxmax = np.max(fluxes, axis=1) mags = 22.5 - 2.5 * np.log10(fluxmax) #ADM convert the largest magnitude into radii for "in" and "near" bright objects. This will require #ADM more consideration to determine the truly correct numbers for DESI near_radius = (0.0802 * mags * mags - 1.860 * mags + 11.625) in_radius = 0.5 * (0.0802 * mags * mags - 1.860 * mags + 11.625) #ADM calculate the TARGETID targetid = encode_targetid(objid=objs['OBJID'], brickid=objs['BRICKID'], release=objs['RELEASE']) #ADM create an output recarray that is just RA, Dec, TARGETID and the radius done = objs[['RA', 'DEC']].copy() done = rfn.append_fields(done, ["TARGETID", "IN_RADIUS", "NEAR_RADIUS"], [targetid, in_radius, near_radius], usemask=False, dtypes=['>i8', '<f8', '<f8']) if outfilename is not None: fitsio.write(outfilename, done, clobber=True) return done