def is_bright_star(targs, starmask): """Determine whether any of a set of targets are, themselves, a bright star mask Parameters ---------- targs : :class:`recarray` A recarray of targets as made by desitarget.cuts.select_targets starmask : :class:`recarray` A recarray containing a bright star mask as made by desitarget.brightstar.make_bright_star_mask Returns ------- is_mask : array_like. True for array entries that correspond to targets that are, themselves, a bright star mask """ #ADM initialize an array of all False (nothing yet has been shown to correspond to a star mask) is_mask = np.zeros(len(targs), dtype=bool) #ADM calculate the TARGETID for the targets targetid = encode_targetid(objid=targs['BRICK_OBJID'], brickid=targs['BRICKID'], release=targs['RELEASE']) #ADM super-fast set-based look-up of which TARGETIDs are matches between the masks and the targets matches = set(starmask["TARGETID"]).intersection(set(targetid)) #ADM determine the indexes of the targets that have a TARGETID in matches w_mask = [index for index, item in enumerate(targetid) if item in matches] #ADM w_mask now contains the target indices that match to a bright star mask on TARGETID is_mask[w_mask] = 'True' return is_mask
def is_bright_source(targs, sourcemask): """Determine whether targets are, themselves, a bright source mask. Parameters ---------- targs : :class:`recarray` Targets as made by, e.g., :func:`desitarget.cuts.select_targets()`. sourcemask : :class:`recarray` A recarray containing a bright source mask as made by, e.g., :func:`desitarget.brightmask.make_bright_star_mask()` Returns ------- is_mask : array_like ``True`` for `targs` that are, themselves, a mask. """ # ADM initialize an array of all False (nothing yet has been shown # ADM to correspond to a mask). is_mask = np.zeros(len(targs), dtype=bool) # ADM calculate the TARGETID for the targets. targetid = encode_targetid(objid=targs['BRICK_OBJID'], brickid=targs['BRICKID'], release=targs['RELEASE']) # ADM super-fast set-based look-up of which TARGETIDs are match # ADM between the masks and the targets. matches = set(sourcemask["TARGETID"]).intersection(set(targetid)) # ADM indexes of the targets that have a TARGETID in matches. w_mask = [index for index, item in enumerate(targetid) if item in matches] # ADM w_mask now holds target indices that match a mask on TARGETID. is_mask[w_mask] = True return is_mask
def gaia_gfas_from_sweep(filename, maglim=18.): """Create a set of GFAs for one sweep file. Parameters ---------- filename: :class:`str` A string corresponding to the full path to a sweep file name. maglim : :class:`float`, optional, defaults to 18 Magnitude limit for GFAs in Gaia G-band. Returns ------- :class:`~numpy.ndarray` GFA objects from Gaia, formatted according to `desitarget.gfa.gfadatamodel`. """ # ADM read in the objects. objects = fitsio.read(filename) # ADM As a mild speed up, only consider sweeps objects brighter than 3 mags # ADM fainter than the passed Gaia magnitude limit. Note that Gaia G-band # ADM approximates SDSS r-band. ii = ((objects["FLUX_G"] > 10**((22.5 - (maglim + 3)) / 2.5)) | (objects["FLUX_R"] > 10**((22.5 - (maglim + 3)) / 2.5)) | (objects["FLUX_Z"] > 10**((22.5 - (maglim + 3)) / 2.5))) objects = objects[ii] nobjs = len(objects) # ADM only retain objects with Gaia matches. # ADM It's fine to propagate an empty array if there are no matches # ADM The sweeps use 0 for objects with no REF_ID. objects = objects[objects["REF_ID"] > 0] # ADM determine a TARGETID for any objects on a brick. targetid = encode_targetid(objid=objects['OBJID'], brickid=objects['BRICKID'], release=objects['RELEASE']) # ADM format everything according to the data model. gfas = np.zeros(len(objects), dtype=gfadatamodel.dtype) # ADM make sure all columns initially have "ridiculous" numbers. gfas[...] = -99. gfas["REF_CAT"] = "" gfas["REF_EPOCH"] = 2015.5 # ADM remove the TARGETID, BRICK_OBJID, REF_CAT, REF_EPOCH columns # ADM and populate them later as they require special treatment. cols = list(gfadatamodel.dtype.names) for col in [ "TARGETID", "BRICK_OBJID", "REF_CAT", "REF_EPOCH", "URAT_ID", "URAT_SEP" ]: cols.remove(col) for col in cols: gfas[col] = objects[col] # ADM populate the TARGETID column. gfas["TARGETID"] = targetid # ADM populate the BRICK_OBJID column. gfas["BRICK_OBJID"] = objects["OBJID"] # ADM REF_CAT and REF_EPOCH didn't exist before DR8. for refcol in ["REF_CAT", "REF_EPOCH"]: if refcol in objects.dtype.names: gfas[refcol] = objects[refcol] # ADM cut the GFAs by a hard limit on magnitude. ii = gfas['GAIA_PHOT_G_MEAN_MAG'] < maglim gfas = gfas[ii] return gfas
mdata = metrics[1][mcols][:] if len(data) != len(mdata): print(np.sort(data['objid'])) print(np.sort(mdata['objid'])) print('Missing objects: {}'.format( set(data['objid']) ^ set(mdata['objid']))) data = data[:len(mdata)] ## tid = encode_targetid(objid=data['objid'], brickid=data['brickid'], release=data['release'], sky=0, mock=0) ## fluxs = np.zeros((rows, 6)) ivflux = np.zeros((rows, 6)) for j, mtype in enumerate(['psf', 'rex']): counter = 3 * j bcounter = 0 for i, isin in enumerate(grz): if isin: if len(mdata['{}_flux'.format(mtype)].shape) == 1: fluxs[:, counter +
) tiles = Table(_file[1].data)['AIRMASS', 'STAR_DENSITY', 'IMAGEFRAC_R', 'IMAGEFRAC_GRZ', 'TILEID', 'RA', 'DEC'] ## scratch = os.environ['CSCRATCH'] _sv_mtl = fits.open( scratch + '/BGS/SV-ASSIGN/mtls/MTL_ALLBGS_STDFAINT_STDBRIGHT_svresolve.0.31.0_49677629_samePRIORITY.fits' ) sv_mtl = Table(_sv_mtl[1].data) ## sv_mtl['TARGETID'] = encode_targetid(objid=sv_mtl['BRICK_OBJID'], brickid=sv_mtl['BRICKID'], release=sv_mtl['RELEASE'], sky=0, mock=0) sv_mtl.pprint() sv_mtl.write( scratch + '/BGS/SV-ASSIGN/mtls/MTL_ALLBGS_STDFAINT_STDBRIGHT_svresolve.0.31.0_49677629_samePRIORITY.fits', format='fits', overwrite=True) ## nside = 64 (indesi, tindex) = is_point_in_desi(tiles, sv_mtl['RA'].quantity.value,
def get_safe_targets(targs, sourcemask): """Get SAFE (BADSKY) locations for targs, set TARGETID/DESI_TARGET. Parameters ---------- targs : :class:`~numpy.ndarray` Targets made by, e.g. :func:`desitarget.cuts.select_targets()`. sourcemask : :class:`~numpy.ndarray` A bright source mask as made by, e.g. :func:`desitarget.brightmask.make_bright_star_mask()`. Returns ------- :class:`~numpy.ndarray` SAFE (BADSKY) locations for `targs` with the same data model as for `targs`. Notes ----- - `Tech Note 2346`_ details SAFE (BADSKY) locations. - `Tech Note 2348`_ details setting the SKY bit in TARGETID. - Hard-coded to create 1 safe location per arcsec of mask radius. The correct number (Nperradius) for DESI is an open question. """ # ADM number of safe locations per radial arcsec of each mask. Nperradius = 1 # ADM grab SAFE locations around masks at a density of Nperradius. ra, dec = generate_safe_locations(sourcemask, Nperradius) # ADM duplicate targs data model for safe locations. nrows = len(ra) safes = np.zeros(nrows, dtype=targs.dtype) # ADM return early if there are no safe locations. if nrows == 0: return safes # ADM populate the safes with the RA/Dec of the SAFE locations. safes["RA"] = ra safes["DEC"] = dec # ADM set the bit for SAFE locations in DESITARGET. safes["DESI_TARGET"] |= desi_mask.BAD_SKY # ADM add the brick information for the SAFE/BADSKY targets. b = brick.Bricks(bricksize=0.25) safes["BRICKID"] = b.brickid(safes["RA"], safes["DEC"]) safes["BRICKNAME"] = b.brickname(safes["RA"], safes["DEC"]) # ADM now add OBJIDs, counting backwards from the maximum possible # ADM OBJID to ensure no duplicateion of TARGETIDs for real targets. maxobjid = 2**targetid_mask.OBJID.nbits - 1 sortid = np.argsort(safes["BRICKID"]) _, cnts = np.unique(safes["BRICKID"], return_counts=True) brickids = np.concatenate([np.arange(i) for i in cnts]) safes["BRICK_OBJID"][sortid] = brickids # ADM finally, update the TARGETID. # ADM first, check the GAIA DR number for these skies. _, _, _, _, _, gdr = decode_targetid(targs["TARGETID"]) if len(set(gdr)) != 1: msg = "Skies are based on multiple Gaia Data Releases:".format(set(gdr)) log.critical(msg) raise ValueError(msg) safes["TARGETID"] = encode_targetid(objid=safes['BRICK_OBJID'], brickid=safes['BRICKID'], sky=1, gaiadr=gdr[0]) # ADM return the input targs with the SAFE targets appended. return safes
def finalize_secondary(scxtargs, scnd_mask, sep=1.): """Assign secondary targets a realistic TARGETID, finalize columns. Parameters ---------- scxtargs : :class:`~numpy.ndarray` An array of secondary targets, must contain the columns `RA`, `DEC` and `TARGETID`. `TARGETID` should be -1 for objects that lack a `TARGETID`. scnd_mask : :class:`desiutil.bitmask.BitMask` A mask corresponding to a set of secondary targets, e.g, could be ``from desitarget.targetmask import scnd_mask`` for the main survey mask. sep : :class:`float`, defaults to 1 arcsecond The separation at which to match secondary targets to themselves in ARCSECONDS. Returns ------- :class:`~numpy.ndarray` The array of secondary targets, with the `TARGETID` bit updated to be unique and reasonable and the `SCND_TARGET` column renamed based on the flavor of `scnd_mask`. Secondary targets that do not have `OVERRIDE` set are also matched to themselves to make sure they share a `TARGETID` and `SCND_TARGET`, and `SCND_TARGET` is updated where multiple secondary targets match a primary target. """ # ADM assign new TARGETIDs to targets without a primary match. nomatch = scxtargs["TARGETID"] == -1 # ADM get BRICKIDs, retrieve the list of unique bricks and the # ADM number of sources in each unique brick. brxid = bricks.brickid(scxtargs["RA"][nomatch], scxtargs["DEC"][nomatch]) # ADM build the OBJIDs from the number of sources per brick. # ADM the for loop doesn't seem the smartest way, but it is O(n). t0 = time() log.info("Begin assigning OBJIDs to bricks...") cntr = np.zeros(np.max(brxid) + 1, dtype=int) objid = [] for ibrx in brxid: cntr[ibrx] += 1 objid.append(cntr[ibrx]) objid = np.array(objid) log.info("Assigned OBJIDs to bricks in {:.1f}s".format(time() - t0)) # ADM assemble the TARGETID, SCND objects have RELEASE==0. targetid = encode_targetid(objid=objid, brickid=brxid) # ADM a check that the generated TARGETIDs are unique. if len(set(targetid)) != len(targetid): msg = "duplicate TARGETIDs generated for secondary targets!!!" log.critical(msg) raise ValueError(msg) # ADM assign the unique TARGETIDs to the secondary objects. scxtargs["TARGETID"][nomatch] = targetid # ADM match secondaries to themselves, to ensure duplicates # ADM share a TARGETID. Don't match special (OVERRIDE) targets # ADM or sources that have already been matched to a primary. w = np.where(~scxtargs["OVERRIDE"] & nomatch)[0] if len(w) > 0: log.info("Matching secondary targets to themselves...t={:.1f}s".format( time() - t0)) # ADM use astropy for the matching. At NERSC, astropy matches # ADM ~20M objects to themselves in about 10 minutes. c = SkyCoord(scxtargs["RA"][w] * u.deg, scxtargs["DEC"][w] * u.deg) m1, m2, _, _ = c.search_around_sky(c, sep * u.arcsec) log.info("Done with matching...t={:.1f}s".format(time() - t0)) # ADM restrict only to unique matches (and exclude self-matches). uniq = m1 > m2 m1, m2 = m1[uniq], m2[uniq] # ADM set same TARGETID for any matches. m2 must come first, here. scxtargs["TARGETID"][w[m2]] = scxtargs["TARGETID"][w[m1]] # ADM Ensure secondary targets with matching TARGETIDs have the # ADM full combination of SCND_TARGET bits set. By definition, # ADM Targets with OVERRIDE set never have matching TARGETIDs. wnoov = np.where(~scxtargs["OVERRIDE"])[0] if len(wnoov) > 0: for _, ind in duplicates(scxtargs["TARGETID"][wnoov]): bitwiseor = np.sum(np.unique(scxtargs["SCND_TARGET"][wnoov][ind])) scxtargs["SCND_TARGET"][wnoov[ind]] = bitwiseor # ADM change the data model depending on whether the mask # ADM is an SVX (X = 1, 2, etc.) mask or not. Nothing will # ADM change if the mask has no preamble. prepend = scnd_mask._name[:-9].upper() scxtargs = rfn.rename_fields(scxtargs, {'SCND_TARGET': prepend + 'SCND_TARGET'}) return scxtargs
def append_safe_targets(targs, starmask, nside=None, drbricks=None): """Append targets at SAFE (BADSKY) locations to target list, set bits in TARGETID and DESI_TARGET Parameters ---------- targs : :class:`~numpy.ndarray` A recarray of targets as made by desitarget.cuts.select_targets nside : :class:`integer` The HEALPix nside used throughout the DESI data model starmask : :class:`~numpy.ndarray` A recarray containing a bright star mask as made by desitarget.brightstar.make_bright_star_mask drbricks : :class:`~numpy.ndarray`, optional A rec array containing at least the "release", "ra", "dec" and "nobjs" columns from a survey bricks file. This is typically used for testing only. Returns ------- The original recarray of targets (targs) is returned with additional SAFE (BADSKY) targets appended to it Notes ----- - See the Tech Note at https://desi.lbl.gov/DocDB/cgi-bin/private/ShowDocument?docid=2346 for more details on the SAFE (BADSKY) locations - See the Tech Note at https://desi.lbl.gov/DocDB/cgi-bin/private/RetrieveFile?docid=2348 for more details on setting the SKY bit in TARGETID - Currently hard-coded to create an additional 10,000 safe locations per sq. deg. of mask. What is the correct number per sq. deg. (Npersqdeg) for DESI is an open question. - Perhaps we should move the default nside to a config file, somewhere? """ #ADM Number of safe locations per sq. deg. of each mask in starmask Npersqdeg = 10000 #ADM generate SAFE locations at the periphery of the star masks appropriate to a density of Npersqdeg ra, dec = generate_safe_locations(starmask, Npersqdeg) #ADM duplicate the targs rec array with a number of rows equal to the generated safe locations nrows = len(ra) safes = np.zeros(nrows, dtype=targs.dtype) #ADM populate the safes recarray with the RA/Dec of the SAFE locations safes["RA"] = ra safes["DEC"] = dec #ADM set the bit for SAFE locations in DESITARGET safes["DESI_TARGET"] |= desi_mask.BADSKY #ADM add the brick information for the SAFE/BADSKY targets b = brick.Bricks(bricksize=0.25) safes["BRICKID"] = b.brickid(safes["RA"], safes["DEC"]) safes["BRICKNAME"] = b.brickname(safes["RA"], safes["DEC"]) #ADM get the string version of the data release (to find directories for brick information) drint = np.max(targs['RELEASE'] // 1000) #ADM check the targets all have the same release checker = np.min(targs['RELEASE'] // 1000) if drint != checker: raise IOError( 'Objects from multiple data releases in same input numpy array?!') drstring = 'dr' + str(drint) #ADM now add the OBJIDs, ensuring they start higher than any other OBJID in the DR #ADM read in the Data Release bricks file if drbricks is None: rootdir = "/project/projectdirs/cosmo/data/legacysurvey/" + drstring.strip( ) + "/" drbricks = fitsio.read(rootdir + "survey-bricks-" + drstring.strip() + ".fits.gz") #ADM the BRICK IDs that are populated for this DR drbrickids = b.brickid(drbricks["ra"], drbricks["dec"]) #ADM the maximum possible BRICKID at bricksize=0.25 brickmax = 662174 #ADM create a histogram of how many SAFE/BADSKY objects are in each brick hsafes = np.histogram(safes["BRICKID"], range=[0, brickmax + 1], bins=brickmax + 1)[0] #ADM create a histogram of how many objects are in each brick in this DR hnobjs = np.zeros(len(hsafes), dtype=int) hnobjs[drbrickids] = drbricks["nobjs"] #ADM make each OBJID for a SAFE/BADSKY +1 higher than any other OBJID in the DR safes["BRICK_OBJID"] = hnobjs[safes["BRICKID"]] + 1 #ADM sort the safes array on BRICKID safes = safes[safes["BRICKID"].argsort()] #ADM remove zero entries from the histogram of BRICKIDs in safes, for speed hsafes = hsafes[np.where(hsafes > 0)] #ADM the count by which to augment each OBJID to make unique OBJIDs for safes objsadd = np.hstack([np.arange(i) for i in hsafes]) #ADM finalize the OBJID for each SAFE target safes["BRICK_OBJID"] += objsadd #ADM finally, update the TARGETID with the OBJID, the BRICKID, and the fact these are skies safes["TARGETID"] = encode_targetid(objid=safes['BRICK_OBJID'], brickid=safes['BRICKID'], sky=1) #ADM return the input targs with the SAFE targets appended return np.hstack([targs, safes])
def make_bright_star_mask( bands, maglim, numproc=4, rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1', infilename=None, outfilename=None, verbose=False): """Make a bright star mask from a structure of bright stars drawn from the sweeps Parameters ---------- bands : :class:`str` A magnitude band from the sweeps, e.g., "G", "R", "Z". Can pass multiple bands as string, e.g. "GRZ", in which case maglim has to be a list of the same length as the string maglim : :class:`float` The upper limit in that magnitude band for which to assemble a list of bright stars. Can pass a list of magnitude limits, in which case bands has to be a string of the same length (e.g., "GRZ" for [12.3,12.7,12.6] numproc : :class:`int`, optional Number of processes over which to parallelize rootdirname : :class:`str`, optional, defaults to dr3 Root directory containing either sweeps or tractor files...e.g. for dr3 this might be /global/project/projectdirs/cosmo/data/legacysurvey/dr3/sweeps/dr3.1 infilename : :class:`str`, optional, if this exists, then the list of bright stars is read in from the file of this name if this is not passed, then code defaults to deriving the recarray of bright stars via a call to collect_bright_stars outfilename : :class:`str`, optional, defaults to not writing anything to file (FITS) File name to which to write the output bright star mask verbose : :class:`bool`, optional Send to write progress to screen Returns ------- :class:`recarray` The bright star mask in the form RA, DEC, TARGETID, IN_RADIUS, NEAR_RADIUS (may also be written to file if "outfilename" is passed) The radii are in ARCMINUTES TARGETID is as calculated in :mod:`desitarget.targets.encode_targetid` Notes ----- - IN_RADIUS is a smaller radius that corresponds to the IN_BRIGHT_OBJECT bit in data/targetmask.yaml - NEAR_RADIUS is a radius that corresponds to the NEAR_BRIGHT_OBJECT bit in data/targetmask.yaml - Currently uses the radius-as-a-function-of-B-mag for Tycho stars from the BOSS mask (in every band) to set the NEAR_RADIUS: R = (0.0802B*B - 1.860B + 11.625) (see Eqn. 9 of https://arxiv.org/pdf/1203.6594.pdf) and half that radius to set the IN_RADIUS. - It's an open question as to what the correct radii are for DESI observations """ #ADM set bands to uppercase if passed as lower case bands = bands.upper() #ADM the band names and nobs columns as arrays instead of strings bandnames = np.array(["FLUX_" + band for band in bands]) nobsnames = np.array(["NOBS_" + band for band in bands]) #ADM force the input maglim to be a list (in case a single value was passed) if type(maglim) == type(16) or type(maglim) == type(16.): maglim = [maglim] if len(bandnames) != len(maglim): raise IOError( 'bands has to be the same length as maglim and {} does not equal {}' .format(len(bandnames), len(maglim))) #ADM change input magnitude(s) to a flux to test against fluxlim = 10.**((22.5 - np.array(maglim)) / 2.5) if infilename is not None: objs = io.read_tractor(infilename) else: objs = collect_bright_stars(bands, maglim, numproc, rootdirname, outfilename, verbose) #ADM write the fluxes and bands as arrays instead of named columns fluxes = objs[bandnames].view( objs[bandnames].dtype[0]).reshape(objs[bandnames].shape + (-1, )) nobs = objs[nobsnames].view( objs[nobsnames].dtype[0]).reshape(objs[nobsnames].shape + (-1, )) #ADM set any observations with NOBS = 0 to have small flux so glitches don't end up as bright star masks. w = np.where(nobs == 0) if len(w[0]) > 0: fluxes[w] = 0. #ADM limit to the passed faint limit w = np.where(np.any(fluxes > fluxlim, axis=1)) fluxes = fluxes[w] objs = objs[w] #ADM grab the (GRZ) magnitudes for observations #ADM and record only the largest flux (smallest magnitude) fluxmax = np.max(fluxes, axis=1) mags = 22.5 - 2.5 * np.log10(fluxmax) #ADM convert the largest magnitude into radii for "in" and "near" bright objects. This will require #ADM more consideration to determine the truly correct numbers for DESI near_radius = (0.0802 * mags * mags - 1.860 * mags + 11.625) in_radius = 0.5 * (0.0802 * mags * mags - 1.860 * mags + 11.625) #ADM calculate the TARGETID targetid = encode_targetid(objid=objs['OBJID'], brickid=objs['BRICKID'], release=objs['RELEASE']) #ADM create an output recarray that is just RA, Dec, TARGETID and the radius done = objs[['RA', 'DEC']].copy() done = rfn.append_fields(done, ["TARGETID", "IN_RADIUS", "NEAR_RADIUS"], [targetid, in_radius, near_radius], usemask=False, dtypes=['>i8', '<f8', '<f8']) if outfilename is not None: fitsio.write(outfilename, done, clobber=True) return done
def make_bright_source_mask( bands, maglim, numproc=4, rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr5/sweep/5.0', infilename=None, outfilename=None): """Make a mask of bright sources from a structure of bright sources drawn from the sweeps. Parameters ---------- bands : :class:`str` A magnitude band from the sweeps, e.g., "G", "R", "Z". Can pass multiple bands as string, e.g. ``"GRZ"``, in which case maglim has to be a list of the same length as the string. maglim : :class:`float` The upper limit in that magnitude band for which to assemble a list of bright sources. Can pass a list of magnitude limits, in which case bands has to be a string of the same length (e.g., ``"GRZ"`` for [12.3,12.7,12.6]). numproc : :class:`int`, optional Number of processes over which to parallelize. rootdirname : :class:`str`, optional, defaults to dr3 Root directory containing either sweeps or tractor files...e.g. for dr5 this might be ``/global/project/projectdirs/cosmo/data/legacysurvey/dr5/sweep/dr5.0``. This is only used if ``infilename`` is not passed. infilename : :class:`str`, optional, if this exists, then the list of bright sources is read in from the file of this name. if this is not passed, then code defaults to deriving the recarray of bright sources from ``rootdirname`` via a call to ``collect_bright_sources``. outfilename : :class:`str`, optional, defaults to not writing anything to file (FITS) File name to which to write the output bright source mask. Returns ------- :class:`recarray` - The bright source mask in the form ``RA`, ``DEC``, ``TARGETID``, ``IN_RADIUS``, ``NEAR_RADIUS``, ``E1``, ``E2``, ``TYPE`` (may also be written to file if ``outfilename`` is passed). - ``TARGETID`` is as calculated in :mod:`desitarget.targets.encode_targetid`. - The radii are in ARCSECONDS (they default to equivalents of half-light radii for ellipses). - ``E1`` and ``E2`` are the ellipticity components as defined at the bottom of, e.g.: http://legacysurvey.org/dr5/catalogs/. - ``TYPE`` is the ``TYPE`` from the sweeps files, see, e.g.: http://legacysurvey.org/dr5/files/#sweep-catalogs. Notes ----- - ``IN_RADIUS`` is a smaller radius that corresponds to the ``IN_BRIGHT_OBJECT`` bit in ``data/targetmask.yaml`` (and is in ARCSECONDS). - ``NEAR_RADIUS`` is a radius that corresponds to the ``NEAR_BRIGHT_OBJECT`` bit in ``data/targetmask.yaml`` (and is in ARCSECONDS). - Currently uses the radius-as-a-function-of-B-mag for Tycho stars from the BOSS mask (in every band) to set the ``NEAR_RADIUS``: R = (0.0802B*B - 1.860B + 11.625) (see Eqn. 9 of https://arxiv.org/pdf/1203.6594.pdf) and half that radius to set the ``IN_RADIUS``. We convert this from arcminutes to arcseconds. - It's an open question as to what the correct radii are for DESI observations. """ # ADM set bands to uppercase if passed as lower case. bands = bands.upper() # ADM the band names and nobs columns as arrays instead of strings. bandnames = np.array(["FLUX_" + band for band in bands]) nobsnames = np.array(["NOBS_" + band for band in bands]) # ADM force the input maglim to be a list (in case a single value was passed). if isinstance(maglim, int) or isinstance(maglim, float): maglim = [maglim] if len(bandnames) != len(maglim): msg = "bands has to be the same length as maglim and {} does not equal {}".format( len(bandnames), len(maglim)) raise IOError(msg) # ADM change input magnitude(s) to a flux to test against. fluxlim = 10.**((22.5 - np.array(maglim)) / 2.5) if infilename is not None: objs = io.read_tractor(infilename) else: objs = collect_bright_sources(bands, maglim, numproc, rootdirname, outfilename) # ADM write the fluxes and bands as arrays instead of named columns # ADM to circumvent a numpy future warning, this requires two copies of the fluxes array. fluxcheck = objs[bandnames].view( objs[bandnames].dtype[0]).reshape(objs[bandnames].shape + (-1, )) fluxes = fluxcheck.copy() nobs = objs[nobsnames].view( objs[nobsnames].dtype[0]).reshape(objs[nobsnames].shape + (-1, )) # ADM set any observations with NOBS = 0 to have small flux so glitches don't end up as bright object masks. w = np.where(nobs == 0) if len(w[0]) > 0: fluxes[w] = 0. # ADM limit to the passed faint limit. w = np.where(np.any(fluxes > fluxlim, axis=1)) fluxes = fluxes[w] objs = objs[w] # ADM grab the (GRZ) magnitudes for observations # ADM and record only the largest flux (smallest magnitude). fluxmax = np.max(fluxes, axis=1) mags = 22.5 - 2.5 * np.log10(fluxmax) # ADM each object's TYPE. objtype = objs["TYPE"] # ADM calculate the TARGETID. targetid = encode_targetid(objid=objs['OBJID'], brickid=objs['BRICKID'], release=objs['RELEASE']) # ADM first set the shape parameters assuming everything is an exponential # ADM this will correctly assign e1, e2 of 0 to things with zero shape. in_radius = objs['SHAPEEXP_R'] e1 = objs['SHAPEEXP_E1'] e2 = objs['SHAPEEXP_E2'] # ADM now to account for deVaucouleurs objects, or things that are dominated by # ADM deVaucouleurs profiles, update objects with a larger "DEV" than "EXP" radius. wdev = np.where(objs['SHAPEDEV_R'] > objs['SHAPEEXP_R']) if len(wdev[0]) > 0: in_radius[wdev] = objs[wdev]['SHAPEDEV_R'] e1[wdev] = objs[wdev]['SHAPEDEV_E1'] e2[wdev] = objs[wdev]['SHAPEDEV_E2'] # ADM finally use the Tycho radius (see the notes above) for PSF or star-like objects. # ADM More consideration will be needed to derive correct numbers for this for DESI!!! # ADM this calculation was for "near" Tycho objects and was in arcmin, so we convert # ADM it to arcsec and multiply it by infac (see the top of the module). tycho_in_radius = infac * (0.0802 * mags * mags - 1.860 * mags + 11.625) * 60. wpsf = np.where(_psflike(objtype)) in_radius[wpsf] = tycho_in_radius[wpsf] # ADM set "near" as a multiple of "in" radius using the factor at the top of the code. near_radius = in_radius * nearfac # ADM create an output recarray that is just RA, Dec, TARGETID and the radius. done = objs[['RA', 'DEC']].copy() done = rfn.append_fields( done, ["TARGETID", "IN_RADIUS", "NEAR_RADIUS", "E1", "E2", "TYPE"], [targetid, in_radius, near_radius, e1, e2, objtype], usemask=False, dtypes=['>i8', '>f4', '>f4', '>f4', '>f4', '|S4']) if outfilename is not None: fitsio.write(outfilename, done, clobber=True) return done
def finalize_secondary(scxtargs, scnd_mask, survey='main', sep=1., darkbright=False): """Assign secondary targets a realistic TARGETID, finalize columns. Parameters ---------- scxtargs : :class:`~numpy.ndarray` An array of secondary targets, must contain the columns `RA`, `DEC` and `TARGETID`. `TARGETID` should be -1 for objects that lack a `TARGETID`. scnd_mask : :class:`desiutil.bitmask.BitMask` A mask corresponding to a set of secondary targets, e.g, could be ``from desitarget.targetmask import scnd_mask`` for the main survey mask. survey : :class:`str`, optional, defaults to "main" string indicating whether we are working in the context of the Main Survey (`main`) or SV (e.g. `sv1`, `sv2` etc.). Used to set the `RELEASE` number in the `TARGETID` (see Notes). sep : :class:`float`, defaults to 1 arcsecond The separation at which to match secondary targets to themselves in ARCSECONDS. darkbright : :class:`bool`, optional, defaults to ``False`` If sent, then split `NUMOBS_INIT` and `PRIORITY_INIT` into `NUMOBS_INIT_DARK`, `NUMOBS_INIT_BRIGHT`, `PRIORITY_INIT_DARK` and `PRIORITY_INIT_BRIGHT` and calculate values appropriate to "BRIGHT" and "DARK|GRAY" observing conditions. Returns ------- :class:`~numpy.ndarray` The array of secondary targets, with the `TARGETID` bit updated to be unique and reasonable and the `SCND_TARGET` column renamed based on the flavor of `scnd_mask`. Notes ----- - Secondaries without `OVERRIDE` are also matched to themselves Such matches are given the same `TARGETID` (that of the primary if they match a primary) and the bitwise or of `SCND_TARGET` and `OBSCONDITIONS` bits across matches. The highest `PRIORITY_INIT` is retained, and others are set to -1. Only secondaries with priorities that are not -1 are written to the main file. If multiple matching secondary targets have the same (highest) priority, the first one encountered retains its `PRIORITY_INIT` - The secondary `TARGETID` is designed to be reproducible. It combines `BRICKID` based on location, `OBJID` based on the order of the targets in the secondary file (`SCND_ORDER`) and `RELEASE` from the secondary bit number (`SCND_TARGET`) and the input `survey`. `RELEASE` is set to ((X-1)*100)+np.log2(scnd_bit) with X from the `survey` string survey=svX and scnd_bit from `SCND_TARGET`. For the main survey (survey="main") X-1 is 5. """ # ADM assign new TARGETIDs to targets without a primary match. nomatch = scxtargs["TARGETID"] == -1 # ADM get the BRICKIDs for each source. brxid = bricks.brickid(scxtargs["RA"], scxtargs["DEC"]) # ADM ensure unique secondary bits for different iterations of SV # ADM and the Main Survey. if survey == 'main': Xm1 = 5 elif survey[0:2] == 'sv': # ADM the re.search just extracts the numbers in the string. Xm1 = int(re.search(r'\d+', survey).group()) - 1 # ADM we've allowed a max of up to sv5 (!). Fail if surpassed. if Xm1 >= 5: msg = "Only coded for up to 'sv5', not {}!!!".format(survey) log.critical(msg) raise ValueError(msg) else: msg = "allowed surveys: 'main', 'svX', not {}!!!".format(survey) log.critical(msg) raise ValueError(msg) # ADM the RELEASE for each source is the `SCND_TARGET` bit NUMBER. release = (Xm1 * 100) + np.log2(scxtargs["SCND_TARGET_INIT"]).astype('int') # ADM build the OBJIDs based on the values of SCND_ORDER. t0 = time() log.info("Begin assigning OBJIDs to bricks...") # ADM So as not to overwhelm the bit-limits for OBJID # ADM rank by SCND_ORDER for each brick and bit combination. # ADM First, create a unique ID based on brxid and release. scnd_order = scxtargs["SCND_ORDER"] sorter = (1000 * brxid) + release # ADM sort the unique IDs and split based on where they change. argsort = np.argsort(sorter) w = np.where(np.diff(sorter[argsort]))[0] soperbrxbit = np.split(scnd_order[argsort], w + 1) # ADM loop through each (brxid, release) and sort on scnd_order. # ADM double argsort returns the ascending ranked order of the entry # ADM (whereas a single argsort returns the indexes for ordering). sortperbrxbit = [np.argsort(np.argsort(so)) for so in soperbrxbit] # ADM finally unroll the (brxid, release) combinations... sortedobjid = np.array(list(itertools.chain.from_iterable(sortperbrxbit))) # ADM ...and reorder based on the initial argsort. objid = np.zeros_like(sortedobjid) - 1 objid[argsort] = sortedobjid log.info("Assigned OBJIDs to bricks in {:.1f}s".format(time() - t0)) # ADM check that the objid array was entirely populated. assert np.all(objid != -1) # ADM assemble the TARGETID, SCND objects have RELEASE==0. targetid = encode_targetid(objid=objid, brickid=brxid, release=release) # ADM a check that the generated TARGETIDs are unique. if len(set(targetid)) != len(targetid): msg = "duplicate TARGETIDs generated for secondary targets!!!" log.critical(msg) raise ValueError(msg) # ADM assign the unique TARGETIDs to the secondary objects. scxtargs["TARGETID"][nomatch] = targetid[nomatch] log.debug("Assigned {} targetids to unmatched secondaries".format( len(targetid[nomatch]))) # ADM match secondaries to themselves, to ensure duplicates # ADM share a TARGETID. Don't match special (OVERRIDE) targets # ADM or sources that have already been matched to a primary. w = np.where(~scxtargs["OVERRIDE"] & nomatch)[0] if len(w) > 0: log.info("Matching secondary targets to themselves...t={:.1f}s".format( time() - t0)) # ADM use astropy for the matching. At NERSC, astropy matches # ADM ~20M objects to themselves in about 10 minutes. c = SkyCoord(scxtargs["RA"][w] * u.deg, scxtargs["DEC"][w] * u.deg) m1, m2, _, _ = c.search_around_sky(c, sep * u.arcsec) log.info("Done with matching...t={:.1f}s".format(time() - t0)) # ADM restrict only to unique matches (and exclude self-matches). uniq = m1 > m2 m1, m2 = m1[uniq], m2[uniq] # ADM set same TARGETID for any matches. m2 must come first, here. scxtargs["TARGETID"][w[m2]] = scxtargs["TARGETID"][w[m1]] # ADM Ensure secondary targets with matching TARGETIDs have all the # ADM relevant SCND_TARGET bits set. By definition, targets with # ADM OVERRIDE set never have matching TARGETIDs. wnoov = np.where(~scxtargs["OVERRIDE"])[0] if len(wnoov) > 0: for _, inds in duplicates(scxtargs["TARGETID"][wnoov]): scnd_targ = 0 for ind in inds: scnd_targ |= scxtargs["SCND_TARGET"][wnoov[ind]] scxtargs["SCND_TARGET"][wnoov[inds]] = scnd_targ log.info("Done checking SCND_TARGET...t={:.1f}s".format(time() - t0)) # ADM change the data model depending on whether the mask # ADM is an SVX (X = 1, 2, etc.) mask or not. Nothing will # ADM change if the mask has no preamble. prepend = scnd_mask._name[:-9].upper() scxtargs = rfn.rename_fields(scxtargs, {'SCND_TARGET': prepend + 'SCND_TARGET'}) # APC same thing for DESI_TARGET scxtargs = rfn.rename_fields(scxtargs, {'DESI_TARGET': prepend + 'DESI_TARGET'}) # APC Remove duplicate targetids from secondary-only targets alldups = [] for _, dups in duplicates(scxtargs['TARGETID']): # Retain the duplicate with highest priority, breaking ties # on lowest index in list of duplicates dups = np.delete(dups, np.argmax(scxtargs['PRIORITY_INIT'][dups])) alldups.append(dups) alldups = np.hstack(alldups) log.debug( "Flagging {} duplicate secondary targetids with PRIORITY_INIT=-1". format(len(alldups))) # ADM and remove the INIT fields in prep for a dark/bright split. scxtargs = rfn.drop_fields(scxtargs, ["PRIORITY_INIT", "NUMOBS_INIT"]) # ADM set initial priorities, numobs and obsconditions for both # ADM BRIGHT and DARK|GRAY conditions, if requested. nscx = len(scxtargs) nodata = np.zeros(nscx, dtype='int') - 1 if darkbright: ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"] else: ender, obscon = [""], ["DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18"] cols, vals, forms = [], [], [] for edr, oc in zip(ender, obscon): cols += ["{}_INIT{}".format(pn, edr) for pn in ["PRIORITY", "NUMOBS"]] vals += [nodata, nodata] forms += ['>i8', '>i8'] # ADM write the output array. newdt = [dt for dt in zip(cols, forms)] done = np.array(np.zeros(nscx), dtype=scxtargs.dtype.descr + newdt) for col in scxtargs.dtype.names: done[col] = scxtargs[col] for col, val in zip(cols, vals): done[col] = val # ADM add the actual PRIORITY/NUMOBS values. for edr, oc in zip(ender, obscon): pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr done[pc], done[nc] = initial_priority_numobs(done, obscon=oc, scnd=True) # APC Flagged duplicates are removed in io.write_secondary done[pc][alldups] = -1 # APC add secondary flag in DESI_TARGET cols, mx, surv = main_cmx_or_sv(done, scnd=True) done[cols[0]] = mx[0]['SCND_ANY'] # ADM set the OBSCONDITIONS. done["OBSCONDITIONS"] = set_obsconditions(done, scnd=True) return done
# ADM check whether ra, dec are in the pixel list theta, phi = np.radians(90 - dec), np.radians(ra) pixnums = hp.ang2pix(nside, theta, phi, nest=True) dat['HEALPIXEL'] = pixnums upix = np.unique(pixnums) for u in upix: inpix = dat['HEALPIXEL'] == u nsky = np.count_nonzero(inpix) objid = np.arange(nsky) dat['TARGETID'][inpix] = encode_targetid(objid=objid, brickid=u, mock=1, sky=1) # Check on copy. fid = np.unique(dat['TARGETID']) fixed = np.count_nonzero(fid != bid) assert fixed == len(dat) print(dat) dat.write('../run/targets/skies_fixedid.fits', format='fits', overwrite=True)
def gaia_gfas_from_sweep(objects, maglim=18.): """Create a set of GFAs for one sweep file or sweep objects. Parameters ---------- objects: :class:`~numpy.ndarray` or `str` Numpy structured array with UPPERCASE columns needed for target selection, OR a string corresponding to a sweep filename. maglim : :class:`float`, optional, defaults to 18 Magnitude limit for GFAs in Gaia G-band. Returns ------- :class:`~numpy.ndarray` GFA objects from Gaia, formatted according to `desitarget.gfa.gfadatamodel`. """ # ADM read in objects if a filename was passed instead of the actual data. if isinstance(objects, str): objects = desitarget.io.read_tractor(objects) # ADM As a mild speed up, only consider sweeps objects brighter than 3 mags # ADM fainter than the passed Gaia magnitude limit. Note that Gaia G-band # ADM approximates SDSS r-band. w = np.where((objects["FLUX_G"] > 10**((22.5-(maglim+3))/2.5)) | (objects["FLUX_R"] > 10**((22.5-(maglim+3))/2.5)) | (objects["FLUX_Z"] > 10**((22.5-(maglim+3))/2.5)))[0] objects = objects[w] nobjs = len(objects) # ADM only retain objects with Gaia matches. # ADM It's fine to propagate an empty array if there are no matches # ADM The sweeps use 0 for objects with no REF_ID. w = np.where(objects["REF_ID"] > 0)[0] objects = objects[w] # ADM determine a TARGETID for any objects on a brick. targetid = encode_targetid(objid=objects['OBJID'], brickid=objects['BRICKID'], release=objects['RELEASE']) # ADM format everything according to the data model. gfas = np.zeros(len(objects), dtype=gfadatamodel.dtype) # ADM make sure all columns initially have "ridiculous" numbers. gfas[...] = -99. # ADM remove the TARGETID and BRICK_OBJID columns and populate them later # ADM as they require special treatment. cols = list(gfadatamodel.dtype.names) for col in ["TARGETID", "BRICK_OBJID"]: cols.remove(col) for col in cols: gfas[col] = objects[col] # ADM populate the TARGETID column. gfas["TARGETID"] = targetid # ADM populate the BRICK_OBJID column. gfas["BRICK_OBJID"] = objects["OBJID"] # ADM cut the GFAs by a hard limit on magnitude. ii = gfas['GAIA_PHOT_G_MEAN_MAG'] < maglim gfas = gfas[ii] # ADM a final clean-up to remove columns that are Nan (from # ADM Gaia-matching) or are 0 (in the sweeps). for col in ["PMRA", "PMDEC"]: ii = ~np.isnan(gfas[col]) & (gfas[col] != 0) gfas = gfas[ii] return gfas