def is_bright_star(targs, starmask):
"""Determine whether any of a set of targets are, themselves, a bright star mask
Parameters
----------
targs : :class:`recarray`
A recarray of targets as made by desitarget.cuts.select_targets
starmask : :class:`recarray`
A recarray containing a bright star mask as made by desitarget.brightstar.make_bright_star_mask
Returns
-------
is_mask : array_like.
True for array entries that correspond to targets that are, themselves, a bright star mask
"""
#ADM initialize an array of all False (nothing yet has been shown to correspond to a star mask)
is_mask = np.zeros(len(targs), dtype=bool)
#ADM calculate the TARGETID for the targets
targetid = encode_targetid(objid=targs['BRICK_OBJID'],
brickid=targs['BRICKID'],
release=targs['RELEASE'])
#ADM super-fast set-based look-up of which TARGETIDs are matches between the masks and the targets
matches = set(starmask["TARGETID"]).intersection(set(targetid))
#ADM determine the indexes of the targets that have a TARGETID in matches
w_mask = [index for index, item in enumerate(targetid) if item in matches]
#ADM w_mask now contains the target indices that match to a bright star mask on TARGETID
is_mask[w_mask] = 'True'
return is_mask
def is_bright_source(targs, sourcemask):
"""Determine whether targets are, themselves, a bright source mask.
Parameters
----------
targs : :class:`recarray`
Targets as made by, e.g., :func:`desitarget.cuts.select_targets()`.
sourcemask : :class:`recarray`
A recarray containing a bright source mask as made by, e.g.,
:func:`desitarget.brightmask.make_bright_star_mask()`
Returns
-------
is_mask : array_like
``True`` for `targs` that are, themselves, a mask.
"""
# ADM initialize an array of all False (nothing yet has been shown
# ADM to correspond to a mask).
is_mask = np.zeros(len(targs), dtype=bool)
# ADM calculate the TARGETID for the targets.
targetid = encode_targetid(objid=targs['BRICK_OBJID'],
brickid=targs['BRICKID'],
release=targs['RELEASE'])
# ADM super-fast set-based look-up of which TARGETIDs are match
# ADM between the masks and the targets.
matches = set(sourcemask["TARGETID"]).intersection(set(targetid))
# ADM indexes of the targets that have a TARGETID in matches.
w_mask = [index for index, item in enumerate(targetid) if item in matches]
# ADM w_mask now holds target indices that match a mask on TARGETID.
is_mask[w_mask] = True
return is_mask
def gaia_gfas_from_sweep(filename, maglim=18.):
"""Create a set of GFAs for one sweep file.
Parameters
----------
filename: :class:`str`
A string corresponding to the full path to a sweep file name.
maglim : :class:`float`, optional, defaults to 18
Magnitude limit for GFAs in Gaia G-band.
Returns
-------
:class:`~numpy.ndarray`
GFA objects from Gaia, formatted according to `desitarget.gfa.gfadatamodel`.
"""
# ADM read in the objects.
objects = fitsio.read(filename)
# ADM As a mild speed up, only consider sweeps objects brighter than 3 mags
# ADM fainter than the passed Gaia magnitude limit. Note that Gaia G-band
# ADM approximates SDSS r-band.
ii = ((objects["FLUX_G"] > 10**((22.5 - (maglim + 3)) / 2.5)) |
(objects["FLUX_R"] > 10**((22.5 - (maglim + 3)) / 2.5)) |
(objects["FLUX_Z"] > 10**((22.5 - (maglim + 3)) / 2.5)))
objects = objects[ii]
nobjs = len(objects)
# ADM only retain objects with Gaia matches.
# ADM It's fine to propagate an empty array if there are no matches
# ADM The sweeps use 0 for objects with no REF_ID.
objects = objects[objects["REF_ID"] > 0]
# ADM determine a TARGETID for any objects on a brick.
targetid = encode_targetid(objid=objects['OBJID'],
brickid=objects['BRICKID'],
release=objects['RELEASE'])
# ADM format everything according to the data model.
gfas = np.zeros(len(objects), dtype=gfadatamodel.dtype)
# ADM make sure all columns initially have "ridiculous" numbers.
gfas[...] = -99.
gfas["REF_CAT"] = ""
gfas["REF_EPOCH"] = 2015.5
# ADM remove the TARGETID, BRICK_OBJID, REF_CAT, REF_EPOCH columns
# ADM and populate them later as they require special treatment.
cols = list(gfadatamodel.dtype.names)
for col in [
"TARGETID", "BRICK_OBJID", "REF_CAT", "REF_EPOCH", "URAT_ID",
"URAT_SEP"
]:
cols.remove(col)
for col in cols:
gfas[col] = objects[col]
# ADM populate the TARGETID column.
gfas["TARGETID"] = targetid
# ADM populate the BRICK_OBJID column.
gfas["BRICK_OBJID"] = objects["OBJID"]
# ADM REF_CAT and REF_EPOCH didn't exist before DR8.
for refcol in ["REF_CAT", "REF_EPOCH"]:
if refcol in objects.dtype.names:
gfas[refcol] = objects[refcol]
# ADM cut the GFAs by a hard limit on magnitude.
ii = gfas['GAIA_PHOT_G_MEAN_MAG'] < maglim
gfas = gfas[ii]
return gfas
mdata = metrics[1][mcols][:]
if len(data) != len(mdata):
print(np.sort(data['objid']))
print(np.sort(mdata['objid']))
print('Missing objects: {}'.format(
set(data['objid']) ^ set(mdata['objid'])))
data = data[:len(mdata)]
##
tid = encode_targetid(objid=data['objid'],
brickid=data['brickid'],
release=data['release'],
sky=0,
mock=0)
##
fluxs = np.zeros((rows, 6))
ivflux = np.zeros((rows, 6))
for j, mtype in enumerate(['psf', 'rex']):
counter = 3 * j
bcounter = 0
for i, isin in enumerate(grz):
if isin:
if len(mdata['{}_flux'.format(mtype)].shape) == 1:
fluxs[:, counter +
)
tiles = Table(_file[1].data)['AIRMASS', 'STAR_DENSITY', 'IMAGEFRAC_R',
'IMAGEFRAC_GRZ', 'TILEID', 'RA', 'DEC']
##
scratch = os.environ['CSCRATCH']
_sv_mtl = fits.open(
scratch +
'/BGS/SV-ASSIGN/mtls/MTL_ALLBGS_STDFAINT_STDBRIGHT_svresolve.0.31.0_49677629_samePRIORITY.fits'
)
sv_mtl = Table(_sv_mtl[1].data)
##
sv_mtl['TARGETID'] = encode_targetid(objid=sv_mtl['BRICK_OBJID'],
brickid=sv_mtl['BRICKID'],
release=sv_mtl['RELEASE'],
sky=0,
mock=0)
sv_mtl.pprint()
sv_mtl.write(
scratch +
'/BGS/SV-ASSIGN/mtls/MTL_ALLBGS_STDFAINT_STDBRIGHT_svresolve.0.31.0_49677629_samePRIORITY.fits',
format='fits',
overwrite=True)
##
nside = 64
(indesi, tindex) = is_point_in_desi(tiles,
sv_mtl['RA'].quantity.value,
def get_safe_targets(targs, sourcemask):
"""Get SAFE (BADSKY) locations for targs, set TARGETID/DESI_TARGET.
Parameters
----------
targs : :class:`~numpy.ndarray`
Targets made by, e.g. :func:`desitarget.cuts.select_targets()`.
sourcemask : :class:`~numpy.ndarray`
A bright source mask as made by, e.g.
:func:`desitarget.brightmask.make_bright_star_mask()`.
Returns
-------
:class:`~numpy.ndarray`
SAFE (BADSKY) locations for `targs` with the same data model as
for `targs`.
Notes
-----
- `Tech Note 2346`_ details SAFE (BADSKY) locations.
- `Tech Note 2348`_ details setting the SKY bit in TARGETID.
- Hard-coded to create 1 safe location per arcsec of mask radius.
The correct number (Nperradius) for DESI is an open question.
"""
# ADM number of safe locations per radial arcsec of each mask.
Nperradius = 1
# ADM grab SAFE locations around masks at a density of Nperradius.
ra, dec = generate_safe_locations(sourcemask, Nperradius)
# ADM duplicate targs data model for safe locations.
nrows = len(ra)
safes = np.zeros(nrows, dtype=targs.dtype)
# ADM return early if there are no safe locations.
if nrows == 0:
return safes
# ADM populate the safes with the RA/Dec of the SAFE locations.
safes["RA"] = ra
safes["DEC"] = dec
# ADM set the bit for SAFE locations in DESITARGET.
safes["DESI_TARGET"] |= desi_mask.BAD_SKY
# ADM add the brick information for the SAFE/BADSKY targets.
b = brick.Bricks(bricksize=0.25)
safes["BRICKID"] = b.brickid(safes["RA"], safes["DEC"])
safes["BRICKNAME"] = b.brickname(safes["RA"], safes["DEC"])
# ADM now add OBJIDs, counting backwards from the maximum possible
# ADM OBJID to ensure no duplicateion of TARGETIDs for real targets.
maxobjid = 2**targetid_mask.OBJID.nbits - 1
sortid = np.argsort(safes["BRICKID"])
_, cnts = np.unique(safes["BRICKID"], return_counts=True)
brickids = np.concatenate([np.arange(i) for i in cnts])
safes["BRICK_OBJID"][sortid] = brickids
# ADM finally, update the TARGETID.
# ADM first, check the GAIA DR number for these skies.
_, _, _, _, _, gdr = decode_targetid(targs["TARGETID"])
if len(set(gdr)) != 1:
msg = "Skies are based on multiple Gaia Data Releases:".format(set(gdr))
log.critical(msg)
raise ValueError(msg)
safes["TARGETID"] = encode_targetid(objid=safes['BRICK_OBJID'],
brickid=safes['BRICKID'],
sky=1,
gaiadr=gdr[0])
# ADM return the input targs with the SAFE targets appended.
return safes
def finalize_secondary(scxtargs, scnd_mask, sep=1.):
"""Assign secondary targets a realistic TARGETID, finalize columns.
Parameters
----------
scxtargs : :class:`~numpy.ndarray`
An array of secondary targets, must contain the columns `RA`,
`DEC` and `TARGETID`. `TARGETID` should be -1 for objects
that lack a `TARGETID`.
scnd_mask : :class:`desiutil.bitmask.BitMask`
A mask corresponding to a set of secondary targets, e.g, could
be ``from desitarget.targetmask import scnd_mask`` for the
main survey mask.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match secondary targets to
themselves in ARCSECONDS.
Returns
-------
:class:`~numpy.ndarray`
The array of secondary targets, with the `TARGETID` bit updated
to be unique and reasonable and the `SCND_TARGET` column renamed
based on the flavor of `scnd_mask`. Secondary targets that do not
have `OVERRIDE` set are also matched to themselves to make sure
they share a `TARGETID` and `SCND_TARGET`, and `SCND_TARGET` is
updated where multiple secondary targets match a primary target.
"""
# ADM assign new TARGETIDs to targets without a primary match.
nomatch = scxtargs["TARGETID"] == -1
# ADM get BRICKIDs, retrieve the list of unique bricks and the
# ADM number of sources in each unique brick.
brxid = bricks.brickid(scxtargs["RA"][nomatch], scxtargs["DEC"][nomatch])
# ADM build the OBJIDs from the number of sources per brick.
# ADM the for loop doesn't seem the smartest way, but it is O(n).
t0 = time()
log.info("Begin assigning OBJIDs to bricks...")
cntr = np.zeros(np.max(brxid) + 1, dtype=int)
objid = []
for ibrx in brxid:
cntr[ibrx] += 1
objid.append(cntr[ibrx])
objid = np.array(objid)
log.info("Assigned OBJIDs to bricks in {:.1f}s".format(time() - t0))
# ADM assemble the TARGETID, SCND objects have RELEASE==0.
targetid = encode_targetid(objid=objid, brickid=brxid)
# ADM a check that the generated TARGETIDs are unique.
if len(set(targetid)) != len(targetid):
msg = "duplicate TARGETIDs generated for secondary targets!!!"
log.critical(msg)
raise ValueError(msg)
# ADM assign the unique TARGETIDs to the secondary objects.
scxtargs["TARGETID"][nomatch] = targetid
# ADM match secondaries to themselves, to ensure duplicates
# ADM share a TARGETID. Don't match special (OVERRIDE) targets
# ADM or sources that have already been matched to a primary.
w = np.where(~scxtargs["OVERRIDE"] & nomatch)[0]
if len(w) > 0:
log.info("Matching secondary targets to themselves...t={:.1f}s".format(
time() - t0))
# ADM use astropy for the matching. At NERSC, astropy matches
# ADM ~20M objects to themselves in about 10 minutes.
c = SkyCoord(scxtargs["RA"][w] * u.deg, scxtargs["DEC"][w] * u.deg)
m1, m2, _, _ = c.search_around_sky(c, sep * u.arcsec)
log.info("Done with matching...t={:.1f}s".format(time() - t0))
# ADM restrict only to unique matches (and exclude self-matches).
uniq = m1 > m2
m1, m2 = m1[uniq], m2[uniq]
# ADM set same TARGETID for any matches. m2 must come first, here.
scxtargs["TARGETID"][w[m2]] = scxtargs["TARGETID"][w[m1]]
# ADM Ensure secondary targets with matching TARGETIDs have the
# ADM full combination of SCND_TARGET bits set. By definition,
# ADM Targets with OVERRIDE set never have matching TARGETIDs.
wnoov = np.where(~scxtargs["OVERRIDE"])[0]
if len(wnoov) > 0:
for _, ind in duplicates(scxtargs["TARGETID"][wnoov]):
bitwiseor = np.sum(np.unique(scxtargs["SCND_TARGET"][wnoov][ind]))
scxtargs["SCND_TARGET"][wnoov[ind]] = bitwiseor
# ADM change the data model depending on whether the mask
# ADM is an SVX (X = 1, 2, etc.) mask or not. Nothing will
# ADM change if the mask has no preamble.
prepend = scnd_mask._name[:-9].upper()
scxtargs = rfn.rename_fields(scxtargs,
{'SCND_TARGET': prepend + 'SCND_TARGET'})
return scxtargs
def append_safe_targets(targs, starmask, nside=None, drbricks=None):
"""Append targets at SAFE (BADSKY) locations to target list, set bits in TARGETID and DESI_TARGET
Parameters
----------
targs : :class:`~numpy.ndarray`
A recarray of targets as made by desitarget.cuts.select_targets
nside : :class:`integer`
The HEALPix nside used throughout the DESI data model
starmask : :class:`~numpy.ndarray`
A recarray containing a bright star mask as made by desitarget.brightstar.make_bright_star_mask
drbricks : :class:`~numpy.ndarray`, optional
A rec array containing at least the "release", "ra", "dec" and "nobjs" columns from a survey bricks file.
This is typically used for testing only.
Returns
-------
The original recarray of targets (targs) is returned with additional SAFE (BADSKY) targets appended to it
Notes
-----
- See the Tech Note at https://desi.lbl.gov/DocDB/cgi-bin/private/ShowDocument?docid=2346 for more details
on the SAFE (BADSKY) locations
- See the Tech Note at https://desi.lbl.gov/DocDB/cgi-bin/private/RetrieveFile?docid=2348 for more details
on setting the SKY bit in TARGETID
- Currently hard-coded to create an additional 10,000 safe locations per sq. deg. of mask. What is the
correct number per sq. deg. (Npersqdeg) for DESI is an open question.
- Perhaps we should move the default nside to a config file, somewhere?
"""
#ADM Number of safe locations per sq. deg. of each mask in starmask
Npersqdeg = 10000
#ADM generate SAFE locations at the periphery of the star masks appropriate to a density of Npersqdeg
ra, dec = generate_safe_locations(starmask, Npersqdeg)
#ADM duplicate the targs rec array with a number of rows equal to the generated safe locations
nrows = len(ra)
safes = np.zeros(nrows, dtype=targs.dtype)
#ADM populate the safes recarray with the RA/Dec of the SAFE locations
safes["RA"] = ra
safes["DEC"] = dec
#ADM set the bit for SAFE locations in DESITARGET
safes["DESI_TARGET"] |= desi_mask.BADSKY
#ADM add the brick information for the SAFE/BADSKY targets
b = brick.Bricks(bricksize=0.25)
safes["BRICKID"] = b.brickid(safes["RA"], safes["DEC"])
safes["BRICKNAME"] = b.brickname(safes["RA"], safes["DEC"])
#ADM get the string version of the data release (to find directories for brick information)
drint = np.max(targs['RELEASE'] // 1000)
#ADM check the targets all have the same release
checker = np.min(targs['RELEASE'] // 1000)
if drint != checker:
raise IOError(
'Objects from multiple data releases in same input numpy array?!')
drstring = 'dr' + str(drint)
#ADM now add the OBJIDs, ensuring they start higher than any other OBJID in the DR
#ADM read in the Data Release bricks file
if drbricks is None:
rootdir = "/project/projectdirs/cosmo/data/legacysurvey/" + drstring.strip(
) + "/"
drbricks = fitsio.read(rootdir + "survey-bricks-" + drstring.strip() +
".fits.gz")
#ADM the BRICK IDs that are populated for this DR
drbrickids = b.brickid(drbricks["ra"], drbricks["dec"])
#ADM the maximum possible BRICKID at bricksize=0.25
brickmax = 662174
#ADM create a histogram of how many SAFE/BADSKY objects are in each brick
hsafes = np.histogram(safes["BRICKID"],
range=[0, brickmax + 1],
bins=brickmax + 1)[0]
#ADM create a histogram of how many objects are in each brick in this DR
hnobjs = np.zeros(len(hsafes), dtype=int)
hnobjs[drbrickids] = drbricks["nobjs"]
#ADM make each OBJID for a SAFE/BADSKY +1 higher than any other OBJID in the DR
safes["BRICK_OBJID"] = hnobjs[safes["BRICKID"]] + 1
#ADM sort the safes array on BRICKID
safes = safes[safes["BRICKID"].argsort()]
#ADM remove zero entries from the histogram of BRICKIDs in safes, for speed
hsafes = hsafes[np.where(hsafes > 0)]
#ADM the count by which to augment each OBJID to make unique OBJIDs for safes
objsadd = np.hstack([np.arange(i) for i in hsafes])
#ADM finalize the OBJID for each SAFE target
safes["BRICK_OBJID"] += objsadd
#ADM finally, update the TARGETID with the OBJID, the BRICKID, and the fact these are skies
safes["TARGETID"] = encode_targetid(objid=safes['BRICK_OBJID'],
brickid=safes['BRICKID'],
sky=1)
#ADM return the input targs with the SAFE targets appended
return np.hstack([targs, safes])
def make_bright_star_mask(
bands,
maglim,
numproc=4,
rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1',
infilename=None,
outfilename=None,
verbose=False):
"""Make a bright star mask from a structure of bright stars drawn from the sweeps
Parameters
----------
bands : :class:`str`
A magnitude band from the sweeps, e.g., "G", "R", "Z".
Can pass multiple bands as string, e.g. "GRZ", in which case maglim has to be a
list of the same length as the string
maglim : :class:`float`
The upper limit in that magnitude band for which to assemble a list of bright stars.
Can pass a list of magnitude limits, in which case bands has to be a string of the
same length (e.g., "GRZ" for [12.3,12.7,12.6]
numproc : :class:`int`, optional
Number of processes over which to parallelize
rootdirname : :class:`str`, optional, defaults to dr3
Root directory containing either sweeps or tractor files...e.g. for dr3 this might be
/global/project/projectdirs/cosmo/data/legacysurvey/dr3/sweeps/dr3.1
infilename : :class:`str`, optional,
if this exists, then the list of bright stars is read in from the file of this name
if this is not passed, then code defaults to deriving the recarray of bright stars
via a call to collect_bright_stars
outfilename : :class:`str`, optional, defaults to not writing anything to file
(FITS) File name to which to write the output bright star mask
verbose : :class:`bool`, optional
Send to write progress to screen
Returns
-------
:class:`recarray`
The bright star mask in the form RA, DEC, TARGETID, IN_RADIUS, NEAR_RADIUS (may also be written to file
if "outfilename" is passed)
The radii are in ARCMINUTES
TARGETID is as calculated in :mod:`desitarget.targets.encode_targetid`
Notes
-----
- IN_RADIUS is a smaller radius that corresponds to the IN_BRIGHT_OBJECT bit in data/targetmask.yaml
- NEAR_RADIUS is a radius that corresponds to the NEAR_BRIGHT_OBJECT bit in data/targetmask.yaml
- Currently uses the radius-as-a-function-of-B-mag for Tycho stars from the BOSS mask (in every band) to set
the NEAR_RADIUS:
R = (0.0802B*B - 1.860B + 11.625) (see Eqn. 9 of https://arxiv.org/pdf/1203.6594.pdf)
and half that radius to set the IN_RADIUS.
- It's an open question as to what the correct radii are for DESI observations
"""
#ADM set bands to uppercase if passed as lower case
bands = bands.upper()
#ADM the band names and nobs columns as arrays instead of strings
bandnames = np.array(["FLUX_" + band for band in bands])
nobsnames = np.array(["NOBS_" + band for band in bands])
#ADM force the input maglim to be a list (in case a single value was passed)
if type(maglim) == type(16) or type(maglim) == type(16.):
maglim = [maglim]
if len(bandnames) != len(maglim):
raise IOError(
'bands has to be the same length as maglim and {} does not equal {}'
.format(len(bandnames), len(maglim)))
#ADM change input magnitude(s) to a flux to test against
fluxlim = 10.**((22.5 - np.array(maglim)) / 2.5)
if infilename is not None:
objs = io.read_tractor(infilename)
else:
objs = collect_bright_stars(bands, maglim, numproc, rootdirname,
outfilename, verbose)
#ADM write the fluxes and bands as arrays instead of named columns
fluxes = objs[bandnames].view(
objs[bandnames].dtype[0]).reshape(objs[bandnames].shape + (-1, ))
nobs = objs[nobsnames].view(
objs[nobsnames].dtype[0]).reshape(objs[nobsnames].shape + (-1, ))
#ADM set any observations with NOBS = 0 to have small flux so glitches don't end up as bright star masks.
w = np.where(nobs == 0)
if len(w[0]) > 0:
fluxes[w] = 0.
#ADM limit to the passed faint limit
w = np.where(np.any(fluxes > fluxlim, axis=1))
fluxes = fluxes[w]
objs = objs[w]
#ADM grab the (GRZ) magnitudes for observations
#ADM and record only the largest flux (smallest magnitude)
fluxmax = np.max(fluxes, axis=1)
mags = 22.5 - 2.5 * np.log10(fluxmax)
#ADM convert the largest magnitude into radii for "in" and "near" bright objects. This will require
#ADM more consideration to determine the truly correct numbers for DESI
near_radius = (0.0802 * mags * mags - 1.860 * mags + 11.625)
in_radius = 0.5 * (0.0802 * mags * mags - 1.860 * mags + 11.625)
#ADM calculate the TARGETID
targetid = encode_targetid(objid=objs['OBJID'],
brickid=objs['BRICKID'],
release=objs['RELEASE'])
#ADM create an output recarray that is just RA, Dec, TARGETID and the radius
done = objs[['RA', 'DEC']].copy()
done = rfn.append_fields(done, ["TARGETID", "IN_RADIUS", "NEAR_RADIUS"],
[targetid, in_radius, near_radius],
usemask=False,
dtypes=['>i8', '<f8', '<f8'])
if outfilename is not None:
fitsio.write(outfilename, done, clobber=True)
return done
def make_bright_source_mask(
bands,
maglim,
numproc=4,
rootdirname='/global/project/projectdirs/cosmo/data/legacysurvey/dr5/sweep/5.0',
infilename=None,
outfilename=None):
"""Make a mask of bright sources from a structure of bright sources drawn from the sweeps.
Parameters
----------
bands : :class:`str`
A magnitude band from the sweeps, e.g., "G", "R", "Z".
Can pass multiple bands as string, e.g. ``"GRZ"``, in which case maglim has to be a
list of the same length as the string.
maglim : :class:`float`
The upper limit in that magnitude band for which to assemble a list of bright sources.
Can pass a list of magnitude limits, in which case bands has to be a string of the
same length (e.g., ``"GRZ"`` for [12.3,12.7,12.6]).
numproc : :class:`int`, optional
Number of processes over which to parallelize.
rootdirname : :class:`str`, optional, defaults to dr3
Root directory containing either sweeps or tractor files...e.g. for dr5 this might be
``/global/project/projectdirs/cosmo/data/legacysurvey/dr5/sweep/dr5.0``. This is only
used if ``infilename`` is not passed.
infilename : :class:`str`, optional,
if this exists, then the list of bright sources is read in from the file of this name.
if this is not passed, then code defaults to deriving the recarray of bright sources
from ``rootdirname`` via a call to ``collect_bright_sources``.
outfilename : :class:`str`, optional, defaults to not writing anything to file
(FITS) File name to which to write the output bright source mask.
Returns
-------
:class:`recarray`
- The bright source mask in the form ``RA`, ``DEC``, ``TARGETID``, ``IN_RADIUS``,
``NEAR_RADIUS``, ``E1``, ``E2``, ``TYPE``
(may also be written to file if ``outfilename`` is passed).
- ``TARGETID`` is as calculated in :mod:`desitarget.targets.encode_targetid`.
- The radii are in ARCSECONDS (they default to equivalents of half-light radii for ellipses).
- ``E1`` and ``E2`` are the ellipticity components as defined at the bottom of, e.g.:
http://legacysurvey.org/dr5/catalogs/.
- ``TYPE`` is the ``TYPE`` from the sweeps files, see, e.g.:
http://legacysurvey.org/dr5/files/#sweep-catalogs.
Notes
-----
- ``IN_RADIUS`` is a smaller radius that corresponds to the ``IN_BRIGHT_OBJECT`` bit in
``data/targetmask.yaml`` (and is in ARCSECONDS).
- ``NEAR_RADIUS`` is a radius that corresponds to the ``NEAR_BRIGHT_OBJECT`` bit in
``data/targetmask.yaml`` (and is in ARCSECONDS).
- Currently uses the radius-as-a-function-of-B-mag for Tycho stars from the BOSS mask
(in every band) to set the ``NEAR_RADIUS``:
R = (0.0802B*B - 1.860B + 11.625) (see Eqn. 9 of https://arxiv.org/pdf/1203.6594.pdf)
and half that radius to set the ``IN_RADIUS``. We convert this from arcminutes to arcseconds.
- It's an open question as to what the correct radii are for DESI observations.
"""
# ADM set bands to uppercase if passed as lower case.
bands = bands.upper()
# ADM the band names and nobs columns as arrays instead of strings.
bandnames = np.array(["FLUX_" + band for band in bands])
nobsnames = np.array(["NOBS_" + band for band in bands])
# ADM force the input maglim to be a list (in case a single value was passed).
if isinstance(maglim, int) or isinstance(maglim, float):
maglim = [maglim]
if len(bandnames) != len(maglim):
msg = "bands has to be the same length as maglim and {} does not equal {}".format(
len(bandnames), len(maglim))
raise IOError(msg)
# ADM change input magnitude(s) to a flux to test against.
fluxlim = 10.**((22.5 - np.array(maglim)) / 2.5)
if infilename is not None:
objs = io.read_tractor(infilename)
else:
objs = collect_bright_sources(bands, maglim, numproc, rootdirname,
outfilename)
# ADM write the fluxes and bands as arrays instead of named columns
# ADM to circumvent a numpy future warning, this requires two copies of the fluxes array.
fluxcheck = objs[bandnames].view(
objs[bandnames].dtype[0]).reshape(objs[bandnames].shape + (-1, ))
fluxes = fluxcheck.copy()
nobs = objs[nobsnames].view(
objs[nobsnames].dtype[0]).reshape(objs[nobsnames].shape + (-1, ))
# ADM set any observations with NOBS = 0 to have small flux so glitches don't end up as bright object masks.
w = np.where(nobs == 0)
if len(w[0]) > 0:
fluxes[w] = 0.
# ADM limit to the passed faint limit.
w = np.where(np.any(fluxes > fluxlim, axis=1))
fluxes = fluxes[w]
objs = objs[w]
# ADM grab the (GRZ) magnitudes for observations
# ADM and record only the largest flux (smallest magnitude).
fluxmax = np.max(fluxes, axis=1)
mags = 22.5 - 2.5 * np.log10(fluxmax)
# ADM each object's TYPE.
objtype = objs["TYPE"]
# ADM calculate the TARGETID.
targetid = encode_targetid(objid=objs['OBJID'],
brickid=objs['BRICKID'],
release=objs['RELEASE'])
# ADM first set the shape parameters assuming everything is an exponential
# ADM this will correctly assign e1, e2 of 0 to things with zero shape.
in_radius = objs['SHAPEEXP_R']
e1 = objs['SHAPEEXP_E1']
e2 = objs['SHAPEEXP_E2']
# ADM now to account for deVaucouleurs objects, or things that are dominated by
# ADM deVaucouleurs profiles, update objects with a larger "DEV" than "EXP" radius.
wdev = np.where(objs['SHAPEDEV_R'] > objs['SHAPEEXP_R'])
if len(wdev[0]) > 0:
in_radius[wdev] = objs[wdev]['SHAPEDEV_R']
e1[wdev] = objs[wdev]['SHAPEDEV_E1']
e2[wdev] = objs[wdev]['SHAPEDEV_E2']
# ADM finally use the Tycho radius (see the notes above) for PSF or star-like objects.
# ADM More consideration will be needed to derive correct numbers for this for DESI!!!
# ADM this calculation was for "near" Tycho objects and was in arcmin, so we convert
# ADM it to arcsec and multiply it by infac (see the top of the module).
tycho_in_radius = infac * (0.0802 * mags * mags - 1.860 * mags +
11.625) * 60.
wpsf = np.where(_psflike(objtype))
in_radius[wpsf] = tycho_in_radius[wpsf]
# ADM set "near" as a multiple of "in" radius using the factor at the top of the code.
near_radius = in_radius * nearfac
# ADM create an output recarray that is just RA, Dec, TARGETID and the radius.
done = objs[['RA', 'DEC']].copy()
done = rfn.append_fields(
done, ["TARGETID", "IN_RADIUS", "NEAR_RADIUS", "E1", "E2", "TYPE"],
[targetid, in_radius, near_radius, e1, e2, objtype],
usemask=False,
dtypes=['>i8', '>f4', '>f4', '>f4', '>f4', '|S4'])
if outfilename is not None:
fitsio.write(outfilename, done, clobber=True)
return done
def finalize_secondary(scxtargs,
scnd_mask,
survey='main',
sep=1.,
darkbright=False):
"""Assign secondary targets a realistic TARGETID, finalize columns.
Parameters
----------
scxtargs : :class:`~numpy.ndarray`
An array of secondary targets, must contain the columns `RA`,
`DEC` and `TARGETID`. `TARGETID` should be -1 for objects
that lack a `TARGETID`.
scnd_mask : :class:`desiutil.bitmask.BitMask`
A mask corresponding to a set of secondary targets, e.g, could
be ``from desitarget.targetmask import scnd_mask`` for the
main survey mask.
survey : :class:`str`, optional, defaults to "main"
string indicating whether we are working in the context of the
Main Survey (`main`) or SV (e.g. `sv1`, `sv2` etc.). Used to
set the `RELEASE` number in the `TARGETID` (see Notes).
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match secondary targets to
themselves in ARCSECONDS.
darkbright : :class:`bool`, optional, defaults to ``False``
If sent, then split `NUMOBS_INIT` and `PRIORITY_INIT` into
`NUMOBS_INIT_DARK`, `NUMOBS_INIT_BRIGHT`, `PRIORITY_INIT_DARK`
and `PRIORITY_INIT_BRIGHT` and calculate values appropriate
to "BRIGHT" and "DARK|GRAY" observing conditions.
Returns
-------
:class:`~numpy.ndarray`
The array of secondary targets, with the `TARGETID` bit updated
to be unique and reasonable and the `SCND_TARGET` column renamed
based on the flavor of `scnd_mask`.
Notes
-----
- Secondaries without `OVERRIDE` are also matched to themselves
Such matches are given the same `TARGETID` (that of the primary
if they match a primary) and the bitwise or of `SCND_TARGET` and
`OBSCONDITIONS` bits across matches. The highest `PRIORITY_INIT`
is retained, and others are set to -1. Only secondaries with
priorities that are not -1 are written to the main file. If
multiple matching secondary targets have the same (highest)
priority, the first one encountered retains its `PRIORITY_INIT`
- The secondary `TARGETID` is designed to be reproducible. It
combines `BRICKID` based on location, `OBJID` based on the
order of the targets in the secondary file (`SCND_ORDER`) and
`RELEASE` from the secondary bit number (`SCND_TARGET`) and the
input `survey`. `RELEASE` is set to ((X-1)*100)+np.log2(scnd_bit)
with X from the `survey` string survey=svX and scnd_bit from
`SCND_TARGET`. For the main survey (survey="main") X-1 is 5.
"""
# ADM assign new TARGETIDs to targets without a primary match.
nomatch = scxtargs["TARGETID"] == -1
# ADM get the BRICKIDs for each source.
brxid = bricks.brickid(scxtargs["RA"], scxtargs["DEC"])
# ADM ensure unique secondary bits for different iterations of SV
# ADM and the Main Survey.
if survey == 'main':
Xm1 = 5
elif survey[0:2] == 'sv':
# ADM the re.search just extracts the numbers in the string.
Xm1 = int(re.search(r'\d+', survey).group()) - 1
# ADM we've allowed a max of up to sv5 (!). Fail if surpassed.
if Xm1 >= 5:
msg = "Only coded for up to 'sv5', not {}!!!".format(survey)
log.critical(msg)
raise ValueError(msg)
else:
msg = "allowed surveys: 'main', 'svX', not {}!!!".format(survey)
log.critical(msg)
raise ValueError(msg)
# ADM the RELEASE for each source is the `SCND_TARGET` bit NUMBER.
release = (Xm1 * 100) + np.log2(scxtargs["SCND_TARGET_INIT"]).astype('int')
# ADM build the OBJIDs based on the values of SCND_ORDER.
t0 = time()
log.info("Begin assigning OBJIDs to bricks...")
# ADM So as not to overwhelm the bit-limits for OBJID
# ADM rank by SCND_ORDER for each brick and bit combination.
# ADM First, create a unique ID based on brxid and release.
scnd_order = scxtargs["SCND_ORDER"]
sorter = (1000 * brxid) + release
# ADM sort the unique IDs and split based on where they change.
argsort = np.argsort(sorter)
w = np.where(np.diff(sorter[argsort]))[0]
soperbrxbit = np.split(scnd_order[argsort], w + 1)
# ADM loop through each (brxid, release) and sort on scnd_order.
# ADM double argsort returns the ascending ranked order of the entry
# ADM (whereas a single argsort returns the indexes for ordering).
sortperbrxbit = [np.argsort(np.argsort(so)) for so in soperbrxbit]
# ADM finally unroll the (brxid, release) combinations...
sortedobjid = np.array(list(itertools.chain.from_iterable(sortperbrxbit)))
# ADM ...and reorder based on the initial argsort.
objid = np.zeros_like(sortedobjid) - 1
objid[argsort] = sortedobjid
log.info("Assigned OBJIDs to bricks in {:.1f}s".format(time() - t0))
# ADM check that the objid array was entirely populated.
assert np.all(objid != -1)
# ADM assemble the TARGETID, SCND objects have RELEASE==0.
targetid = encode_targetid(objid=objid, brickid=brxid, release=release)
# ADM a check that the generated TARGETIDs are unique.
if len(set(targetid)) != len(targetid):
msg = "duplicate TARGETIDs generated for secondary targets!!!"
log.critical(msg)
raise ValueError(msg)
# ADM assign the unique TARGETIDs to the secondary objects.
scxtargs["TARGETID"][nomatch] = targetid[nomatch]
log.debug("Assigned {} targetids to unmatched secondaries".format(
len(targetid[nomatch])))
# ADM match secondaries to themselves, to ensure duplicates
# ADM share a TARGETID. Don't match special (OVERRIDE) targets
# ADM or sources that have already been matched to a primary.
w = np.where(~scxtargs["OVERRIDE"] & nomatch)[0]
if len(w) > 0:
log.info("Matching secondary targets to themselves...t={:.1f}s".format(
time() - t0))
# ADM use astropy for the matching. At NERSC, astropy matches
# ADM ~20M objects to themselves in about 10 minutes.
c = SkyCoord(scxtargs["RA"][w] * u.deg, scxtargs["DEC"][w] * u.deg)
m1, m2, _, _ = c.search_around_sky(c, sep * u.arcsec)
log.info("Done with matching...t={:.1f}s".format(time() - t0))
# ADM restrict only to unique matches (and exclude self-matches).
uniq = m1 > m2
m1, m2 = m1[uniq], m2[uniq]
# ADM set same TARGETID for any matches. m2 must come first, here.
scxtargs["TARGETID"][w[m2]] = scxtargs["TARGETID"][w[m1]]
# ADM Ensure secondary targets with matching TARGETIDs have all the
# ADM relevant SCND_TARGET bits set. By definition, targets with
# ADM OVERRIDE set never have matching TARGETIDs.
wnoov = np.where(~scxtargs["OVERRIDE"])[0]
if len(wnoov) > 0:
for _, inds in duplicates(scxtargs["TARGETID"][wnoov]):
scnd_targ = 0
for ind in inds:
scnd_targ |= scxtargs["SCND_TARGET"][wnoov[ind]]
scxtargs["SCND_TARGET"][wnoov[inds]] = scnd_targ
log.info("Done checking SCND_TARGET...t={:.1f}s".format(time() - t0))
# ADM change the data model depending on whether the mask
# ADM is an SVX (X = 1, 2, etc.) mask or not. Nothing will
# ADM change if the mask has no preamble.
prepend = scnd_mask._name[:-9].upper()
scxtargs = rfn.rename_fields(scxtargs,
{'SCND_TARGET': prepend + 'SCND_TARGET'})
# APC same thing for DESI_TARGET
scxtargs = rfn.rename_fields(scxtargs,
{'DESI_TARGET': prepend + 'DESI_TARGET'})
# APC Remove duplicate targetids from secondary-only targets
alldups = []
for _, dups in duplicates(scxtargs['TARGETID']):
# Retain the duplicate with highest priority, breaking ties
# on lowest index in list of duplicates
dups = np.delete(dups, np.argmax(scxtargs['PRIORITY_INIT'][dups]))
alldups.append(dups)
alldups = np.hstack(alldups)
log.debug(
"Flagging {} duplicate secondary targetids with PRIORITY_INIT=-1".
format(len(alldups)))
# ADM and remove the INIT fields in prep for a dark/bright split.
scxtargs = rfn.drop_fields(scxtargs, ["PRIORITY_INIT", "NUMOBS_INIT"])
# ADM set initial priorities, numobs and obsconditions for both
# ADM BRIGHT and DARK|GRAY conditions, if requested.
nscx = len(scxtargs)
nodata = np.zeros(nscx, dtype='int') - 1
if darkbright:
ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"]
else:
ender, obscon = [""], ["DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18"]
cols, vals, forms = [], [], []
for edr, oc in zip(ender, obscon):
cols += ["{}_INIT{}".format(pn, edr) for pn in ["PRIORITY", "NUMOBS"]]
vals += [nodata, nodata]
forms += ['>i8', '>i8']
# ADM write the output array.
newdt = [dt for dt in zip(cols, forms)]
done = np.array(np.zeros(nscx), dtype=scxtargs.dtype.descr + newdt)
for col in scxtargs.dtype.names:
done[col] = scxtargs[col]
for col, val in zip(cols, vals):
done[col] = val
# ADM add the actual PRIORITY/NUMOBS values.
for edr, oc in zip(ender, obscon):
pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr
done[pc], done[nc] = initial_priority_numobs(done,
obscon=oc,
scnd=True)
# APC Flagged duplicates are removed in io.write_secondary
done[pc][alldups] = -1
# APC add secondary flag in DESI_TARGET
cols, mx, surv = main_cmx_or_sv(done, scnd=True)
done[cols[0]] = mx[0]['SCND_ANY']
# ADM set the OBSCONDITIONS.
done["OBSCONDITIONS"] = set_obsconditions(done, scnd=True)
return done
# ADM check whether ra, dec are in the pixel list
theta, phi = np.radians(90 - dec), np.radians(ra)
pixnums = hp.ang2pix(nside, theta, phi, nest=True)
dat['HEALPIXEL'] = pixnums
upix = np.unique(pixnums)
for u in upix:
inpix = dat['HEALPIXEL'] == u
nsky = np.count_nonzero(inpix)
objid = np.arange(nsky)
dat['TARGETID'][inpix] = encode_targetid(objid=objid,
brickid=u,
mock=1,
sky=1)
# Check on copy.
fid = np.unique(dat['TARGETID'])
fixed = np.count_nonzero(fid != bid)
assert fixed == len(dat)
print(dat)
dat.write('../run/targets/skies_fixedid.fits', format='fits', overwrite=True)
def gaia_gfas_from_sweep(objects, maglim=18.):
"""Create a set of GFAs for one sweep file or sweep objects.
Parameters
----------
objects: :class:`~numpy.ndarray` or `str`
Numpy structured array with UPPERCASE columns needed for target selection, OR
a string corresponding to a sweep filename.
maglim : :class:`float`, optional, defaults to 18
Magnitude limit for GFAs in Gaia G-band.
Returns
-------
:class:`~numpy.ndarray`
GFA objects from Gaia, formatted according to `desitarget.gfa.gfadatamodel`.
"""
# ADM read in objects if a filename was passed instead of the actual data.
if isinstance(objects, str):
objects = desitarget.io.read_tractor(objects)
# ADM As a mild speed up, only consider sweeps objects brighter than 3 mags
# ADM fainter than the passed Gaia magnitude limit. Note that Gaia G-band
# ADM approximates SDSS r-band.
w = np.where((objects["FLUX_G"] > 10**((22.5-(maglim+3))/2.5)) |
(objects["FLUX_R"] > 10**((22.5-(maglim+3))/2.5)) |
(objects["FLUX_Z"] > 10**((22.5-(maglim+3))/2.5)))[0]
objects = objects[w]
nobjs = len(objects)
# ADM only retain objects with Gaia matches.
# ADM It's fine to propagate an empty array if there are no matches
# ADM The sweeps use 0 for objects with no REF_ID.
w = np.where(objects["REF_ID"] > 0)[0]
objects = objects[w]
# ADM determine a TARGETID for any objects on a brick.
targetid = encode_targetid(objid=objects['OBJID'],
brickid=objects['BRICKID'],
release=objects['RELEASE'])
# ADM format everything according to the data model.
gfas = np.zeros(len(objects), dtype=gfadatamodel.dtype)
# ADM make sure all columns initially have "ridiculous" numbers.
gfas[...] = -99.
# ADM remove the TARGETID and BRICK_OBJID columns and populate them later
# ADM as they require special treatment.
cols = list(gfadatamodel.dtype.names)
for col in ["TARGETID", "BRICK_OBJID"]:
cols.remove(col)
for col in cols:
gfas[col] = objects[col]
# ADM populate the TARGETID column.
gfas["TARGETID"] = targetid
# ADM populate the BRICK_OBJID column.
gfas["BRICK_OBJID"] = objects["OBJID"]
# ADM cut the GFAs by a hard limit on magnitude.
ii = gfas['GAIA_PHOT_G_MEAN_MAG'] < maglim
gfas = gfas[ii]
# ADM a final clean-up to remove columns that are Nan (from
# ADM Gaia-matching) or are 0 (in the sweeps).
for col in ["PMRA", "PMDEC"]:
ii = ~np.isnan(gfas[col]) & (gfas[col] != 0)
gfas = gfas[ii]
return gfas
