import desimodel.io
from desimodel.footprint import is_point_in_desi
import desitarget.io
from desitarget.internal import sharedmem
from desitarget.gaiamatch import read_gaia_file, find_gaia_files_beyond_gal_b
from desitarget.gaiamatch import find_gaia_files_tiles, find_gaia_files_box
from desitarget.uratmatch import match_to_urat
from desitarget.targets import encode_targetid, resolve
from desitarget.geomask import is_in_gal_box, is_in_box
from desiutil import brick
from desiutil.log import get_logger
# ADM set up the Legacy Surveys bricks object.
bricks = brick.Bricks(bricksize=0.25)
# ADM set up the default DESI logger.
log = get_logger()
# ADM the current data model for columns in the GFA files.
gfadatamodel = np.array([],
dtype=[('RELEASE', '>i4'), ('TARGETID', 'i8'),
('BRICKID', 'i4'), ('BRICK_OBJID', 'i4'),
('RA', 'f8'), ('DEC', 'f8'), ('RA_IVAR', 'f4'),
('DEC_IVAR', 'f4'), ('TYPE', 'S4'),
('FLUX_G', 'f4'), ('FLUX_R', 'f4'),
('FLUX_Z', 'f4'), ('FLUX_IVAR_G', 'f4'),
('FLUX_IVAR_R', 'f4'), ('FLUX_IVAR_Z', 'f4'),
('REF_ID', 'i8'), ('REF_CAT', 'S2'),
('REF_EPOCH', 'f4'), ('PARALLAX', 'f4'),
('PARALLAX_IVAR', 'f4'), ('PMRA', 'f4'),
def select_randoms(drdir,
density=100000,
numproc=32,
nside=4,
pixlist=None,
bundlebricks=None,
brickspersec=2.5,
dustdir=None):
"""NOBS, GALDEPTH, PSFDEPTH (per-band) for random points in a DR of the Legacy Surveys
Parameters
----------
drdir : :class:`str`
The root directory pointing to a Data Release from the Legacy Surveys
e.g. /global/project/projectdirs/cosmo/data/legacysurvey/dr7.
density : :class:`int`, optional, defaults to 100,000
The number of random points to return per sq. deg. As a typical brick is
~0.25 x 0.25 sq. deg. about (0.0625*density) points will be returned
numproc : :class:`int`, optional, defaults to 32
The number of processes over which to parallelize
nside : :class:`int`, optional, defaults to nside=4 (214.86 sq. deg.)
The (NESTED) HEALPixel nside to be used with the `pixlist` and `bundlebricks` input.
pixlist : :class:`list` or `int`, optional, defaults to None
Bricks will only be processed if the CENTER of the brick lies within the bounds of
pixels that are in this list of integers, at the supplied HEALPixel `nside`.
Uses the HEALPix NESTED scheme. Useful for parallelizing. If pixlist is None
then all bricks in the passed `survey` will be processed.
bundlebricks : :class:`int`, defaults to None
If not None, then instead of selecting the skies, print, to screen, the slurm
script that will approximately balance the brick distribution at `bundlebricks`
bricks per node. So, for instance, if bundlebricks is 14000 (which as of
the latest git push works well to fit on the interactive nodes on Cori and run
in about an hour), then commands would be returned with the correct pixlist values
to pass to the code to pack at about 14000 bricks per node across all of the bricks
in `survey`.
brickspersec : :class:`float`, optional, defaults to 2.5
The rough number of bricks processed per second by the code (parallelized across
a chosen number of nodes). Used in conjunction with `bundlebricks` for the code
to estimate time to completion when parallelizing across pixels.
dustdir : :class:`str`, optional, defaults to $DUST_DIR+'maps'
The root directory pointing to SFD dust maps. If not
sent the code will try to use $DUST_DIR+'maps')
before failing.
Returns
-------
:class:`~numpy.ndarray`
a numpy structured array with the following columns:
RA: Right Ascension of a random point
DEC: Declination of a random point
BRICKNAME: Passed brick name
NOBS_G: Number of observations at this location in the g-band
NOBS_R: Number of observations at this location in the r-band
NOBS_Z: Number of observations at this location in the z-band
PSFDEPTH_G: PSF depth at this location in the g-band
PSFDEPTH_R: PSF depth at this location in the r-band
PSFDEPTH_Z: PSF depth at this location in the z-band
GALDEPTH_G: Galaxy depth at this location in the g-band
GALDEPTH_R: Galaxy depth at this location in the r-band
GALDEPTH_Z: Galaxy depth at this location in the z-band
MASKBITS: Extra mask bits info as stored in the header of e.g.,
dr7dir + 'coadd/111/1116p210/legacysurvey-1116p210-maskbits.fits.gz'
EBV: E(B-V) at this location from the SFD dust maps
"""
# ADM read in the survey bricks file, which lists the bricks of interest for this DR.
# ADM if this is pre-or-post-DR8 we need to find the correct directory or directories.
drdirs = _pre_or_post_dr8(drdir)
bricknames = []
brickinfo = []
for dd in drdirs:
sbfile = glob(dd + '/*bricks-dr*')
if len(sbfile) > 0:
sbfile = sbfile[0]
hdu = fits.open(sbfile)
brickinfo.append(hdu[1].data)
bricknames.append(hdu[1].data['BRICKNAME'])
else:
# ADM this is a hack for test bricks where we didn't always generate the
# ADM bricks file. It's probably safe to remove it at some point.
from desitarget.io import brickname_from_filename
fns = glob(os.path.join(dd, 'tractor', '*', '*fits'))
bricknames.append([brickname_from_filename(fn) for fn in fns])
brickinfo.append([])
if pixlist is not None or bundlebricks is not None:
msg = 'DR-specific bricks file not found'
msg += 'and pixlist or bundlebricks passed!!!'
log.critical(msg)
raise ValueError(msg)
bricknames = np.concatenate(bricknames)
brickinfo = np.concatenate(brickinfo)
# ADM if the pixlist or bundlebricks option was sent, we'll need the HEALPixel
# ADM information for each brick.
if pixlist is not None or bundlebricks is not None:
theta, phi = np.radians(90 - brickinfo["dec"]), np.radians(
brickinfo["ra"])
pixnum = hp.ang2pix(nside, theta, phi, nest=True)
# ADM if the bundlebricks option was sent, call the packing code.
if bundlebricks is not None:
bundle_bricks(pixnum,
bundlebricks,
nside,
brickspersec=brickspersec,
prefix='randoms',
surveydir=drdir)
return
# ADM restrict to only bricks in a set of HEALPixels, if requested.
if pixlist is not None:
# ADM if an integer was passed, turn it into a list.
if isinstance(pixlist, int):
pixlist = [pixlist]
wbricks = np.where([pix in pixlist for pix in pixnum])[0]
bricknames = bricknames[wbricks]
if len(wbricks) == 0:
log.warning('ZERO bricks in passed pixel list!!!')
log.info(
"Processing bricks in (nside={}, pixel numbers={}) HEALPixels".
format(nside, pixlist))
nbricks = len(bricknames)
log.info(
'Processing {} bricks from DR at {} at density {:.1e} per sq. deg...t = {:.1f}s'
.format(nbricks, drdir, density,
time() - start))
# ADM a little more information if we're slurming across nodes.
if os.getenv('SLURMD_NODENAME') is not None:
log.info('Running on Node {}'.format(os.getenv('SLURMD_NODENAME')))
# ADM initialize the bricks class, and retrieve the brick information look-up table
# ADM so it can be used in a common fashion.
from desiutil import brick
bricktable = brick.Bricks(bricksize=0.25).to_table()
# ADM the critical function to run on every brick.
def _get_quantities(brickname):
'''wrapper on nobs_positions_in_a_brick_from_edges() given a brick name'''
# ADM retrieve the edges for the brick that we're working on
wbrick = np.where(bricktable["BRICKNAME"] == brickname)[0]
ramin, ramax, decmin, decmax = np.array(bricktable[wbrick]["RA1",
"RA2",
"DEC1",
"DEC2"])[0]
# ADM populate the brick with random points, and retrieve the quantities
# ADM of interest at those points.
return get_quantities_in_a_brick(ramin,
ramax,
decmin,
decmax,
brickname,
drdir,
density=density,
dustdir=dustdir)
# ADM this is just to count bricks in _update_status
nbrick = np.zeros((), dtype='i8')
t0 = time()
def _update_status(result):
''' wrapper function for the critical reduction operation,
that occurs on the main parallel process '''
if nbrick % 50 == 0 and nbrick > 0:
rate = nbrick / (time() - t0)
log.info('{}/{} bricks; {:.1f} bricks/sec'.format(
nbrick, nbricks, rate))
# ADM if we're going to exceed 4 hours, warn the user
if nbricks / rate > 4 * 3600.:
log.error(
"May take > 4 hours to run. Try running with bundlebricks instead."
)
nbrick[...] += 1 # this is an in-place modification
return result
# - Parallel process input files
if numproc > 1:
pool = sharedmem.MapReduce(np=numproc)
with pool:
qinfo = pool.map(_get_quantities,
bricknames,
reduce=_update_status)
else:
qinfo = list()
for brickname in bricknames:
qinfo.append(_update_status(_get_quantities(brickname)))
# ADM concatenate the randoms into a single long list and resolve whether
# ADM they are officially in the north or the south.
qinfo = np.concatenate(qinfo)
qinfo = resolve(qinfo)
# ADM one last shuffle to randomize across brick boundaries.
np.random.seed(616)
np.random.shuffle(qinfo)
return qinfo
def get_brick_info(drdirs, counts=False, allbricks=False):
"""Retrieve brick names and coordinates from Legacy Surveys directories.
Parameters
----------
drdirs : :class:`list` or `str`
A list of strings, each of which corresponds to a directory pointing
to a Data Release from the Legacy Surveys. Can be of length one.
e.g. ['/global/project/projectdirs/cosmo/data/legacysurvey/dr7'].
or '/global/project/projectdirs/cosmo/data/legacysurvey/dr7'
Can be None if `allbricks` is passed.
counts : :class:`bool`, optional, defaults to ``False``
If ``True`` also return a count of the number of times each brick
appears ([RAcen, DECcen, RAmin, RAmax, DECmin, DECmax, CNT]).
allbricks : :class:`bool`, optional, defaults to ``False``
If ``True`` ignore `drdirs` and simply return a dictionary of ALL
of the bricks.
Returns
-------
:class:`dict`
UNIQUE bricks covered by the Data Release(s). Keys are brick names
and values are a list of the brick center and the brick corners
([RAcen, DECcen, RAmin, RAmax, DECmin, DECmax]).
Notes
-----
- Tries a few different ways in case the survey bricks files have
not yet been created.
"""
# ADM convert a single input string to a list.
if isinstance(drdirs, str):
drdirs = [
drdirs,
]
# ADM initialize the bricks class, retrieve the brick information look-up
# ADM table and turn it into a fast look-up dictionary.
from desiutil import brick
bricktable = brick.Bricks(bricksize=0.25).to_table()
brickdict = {}
for b in bricktable:
brickdict[b["BRICKNAME"]] = [
b["RA"], b["DEC"], b["RA1"], b["RA2"], b["DEC1"], b["DEC2"]
]
# ADM if requested, return the dictionary of ALL bricks.
if allbricks:
return brickdict
bricknames = []
for dd in drdirs:
# ADM in the simplest case, read in the survey bricks file, which lists
# ADM the bricks of interest for this DR.
sbfile = glob(dd + '/*bricks-dr*')
if len(sbfile) > 0:
brickinfo = fitsio.read(sbfile[0])
# ADM fitsio reads things in as bytes, so convert to unicode.
bricknames.append(brickinfo['brickname'].astype('U'))
else:
# ADM hack for test bricks where we didn't generate the bricks file.
fns = glob(os.path.join(dd, 'tractor', '*', '*fits'))
bricknames.append([brickname_from_filename(fn) for fn in fns])
# ADM don't count bricks twice, but record number of duplicate bricks.
bricknames, cnts = np.unique(np.concatenate(bricknames),
return_counts=True)
# ADM only return the subset of the dictionary with bricks in the DR.
if counts:
return {bn: brickdict[bn] + [cnt] for bn, cnt in zip(bricknames, cnts)}
return {bn: brickdict[bn] for bn in bricknames}
def get_safe_targets(targs, sourcemask):
"""Get SAFE (BADSKY) locations for targs, set TARGETID/DESI_TARGET.
Parameters
----------
targs : :class:`~numpy.ndarray`
Targets made by, e.g. :func:`desitarget.cuts.select_targets()`.
sourcemask : :class:`~numpy.ndarray`
A bright source mask as made by, e.g.
:func:`desitarget.brightmask.make_bright_star_mask()`.
Returns
-------
:class:`~numpy.ndarray`
SAFE (BADSKY) locations for `targs` with the same data model as
for `targs`.
Notes
-----
- `Tech Note 2346`_ details SAFE (BADSKY) locations.
- `Tech Note 2348`_ details setting the SKY bit in TARGETID.
- Hard-coded to create 1 safe location per arcsec of mask radius.
The correct number (Nperradius) for DESI is an open question.
"""
# ADM number of safe locations per radial arcsec of each mask.
Nperradius = 1
# ADM grab SAFE locations around masks at a density of Nperradius.
ra, dec = generate_safe_locations(sourcemask, Nperradius)
# ADM duplicate targs data model for safe locations.
nrows = len(ra)
safes = np.zeros(nrows, dtype=targs.dtype)
# ADM return early if there are no safe locations.
if nrows == 0:
return safes
# ADM populate the safes with the RA/Dec of the SAFE locations.
safes["RA"] = ra
safes["DEC"] = dec
# ADM set the bit for SAFE locations in DESITARGET.
safes["DESI_TARGET"] |= desi_mask.BAD_SKY
# ADM add the brick information for the SAFE/BADSKY targets.
b = brick.Bricks(bricksize=0.25)
safes["BRICKID"] = b.brickid(safes["RA"], safes["DEC"])
safes["BRICKNAME"] = b.brickname(safes["RA"], safes["DEC"])
# ADM now add OBJIDs, counting backwards from the maximum possible
# ADM OBJID to ensure no duplicateion of TARGETIDs for real targets.
maxobjid = 2**targetid_mask.OBJID.nbits - 1
sortid = np.argsort(safes["BRICKID"])
_, cnts = np.unique(safes["BRICKID"], return_counts=True)
brickids = np.concatenate([np.arange(i) for i in cnts])
safes["BRICK_OBJID"][sortid] = brickids
# ADM finally, update the TARGETID.
# ADM first, check the GAIA DR number for these skies.
_, _, _, _, _, gdr = decode_targetid(targs["TARGETID"])
if len(set(gdr)) != 1:
msg = "Skies are based on multiple Gaia Data Releases:".format(set(gdr))
log.critical(msg)
raise ValueError(msg)
safes["TARGETID"] = encode_targetid(objid=safes['BRICK_OBJID'],
brickid=safes['BRICKID'],
sky=1,
gaiadr=gdr[0])
# ADM return the input targs with the SAFE targets appended.
return safes
def append_safe_targets(targs, starmask, nside=None, drbricks=None):
"""Append targets at SAFE (BADSKY) locations to target list, set bits in TARGETID and DESI_TARGET
Parameters
----------
targs : :class:`~numpy.ndarray`
A recarray of targets as made by desitarget.cuts.select_targets
nside : :class:`integer`
The HEALPix nside used throughout the DESI data model
starmask : :class:`~numpy.ndarray`
A recarray containing a bright star mask as made by desitarget.brightstar.make_bright_star_mask
drbricks : :class:`~numpy.ndarray`, optional
A rec array containing at least the "release", "ra", "dec" and "nobjs" columns from a survey bricks file.
This is typically used for testing only.
Returns
-------
The original recarray of targets (targs) is returned with additional SAFE (BADSKY) targets appended to it
Notes
-----
- See the Tech Note at https://desi.lbl.gov/DocDB/cgi-bin/private/ShowDocument?docid=2346 for more details
on the SAFE (BADSKY) locations
- See the Tech Note at https://desi.lbl.gov/DocDB/cgi-bin/private/RetrieveFile?docid=2348 for more details
on setting the SKY bit in TARGETID
- Currently hard-coded to create an additional 10,000 safe locations per sq. deg. of mask. What is the
correct number per sq. deg. (Npersqdeg) for DESI is an open question.
- Perhaps we should move the default nside to a config file, somewhere?
"""
#ADM Number of safe locations per sq. deg. of each mask in starmask
Npersqdeg = 10000
#ADM generate SAFE locations at the periphery of the star masks appropriate to a density of Npersqdeg
ra, dec = generate_safe_locations(starmask, Npersqdeg)
#ADM duplicate the targs rec array with a number of rows equal to the generated safe locations
nrows = len(ra)
safes = np.zeros(nrows, dtype=targs.dtype)
#ADM populate the safes recarray with the RA/Dec of the SAFE locations
safes["RA"] = ra
safes["DEC"] = dec
#ADM set the bit for SAFE locations in DESITARGET
safes["DESI_TARGET"] |= desi_mask.BADSKY
#ADM add the brick information for the SAFE/BADSKY targets
b = brick.Bricks(bricksize=0.25)
safes["BRICKID"] = b.brickid(safes["RA"], safes["DEC"])
safes["BRICKNAME"] = b.brickname(safes["RA"], safes["DEC"])
#ADM get the string version of the data release (to find directories for brick information)
drint = np.max(targs['RELEASE'] // 1000)
#ADM check the targets all have the same release
checker = np.min(targs['RELEASE'] // 1000)
if drint != checker:
raise IOError(
'Objects from multiple data releases in same input numpy array?!')
drstring = 'dr' + str(drint)
#ADM now add the OBJIDs, ensuring they start higher than any other OBJID in the DR
#ADM read in the Data Release bricks file
if drbricks is None:
rootdir = "/project/projectdirs/cosmo/data/legacysurvey/" + drstring.strip(
) + "/"
drbricks = fitsio.read(rootdir + "survey-bricks-" + drstring.strip() +
".fits.gz")
#ADM the BRICK IDs that are populated for this DR
drbrickids = b.brickid(drbricks["ra"], drbricks["dec"])
#ADM the maximum possible BRICKID at bricksize=0.25
brickmax = 662174
#ADM create a histogram of how many SAFE/BADSKY objects are in each brick
hsafes = np.histogram(safes["BRICKID"],
range=[0, brickmax + 1],
bins=brickmax + 1)[0]
#ADM create a histogram of how many objects are in each brick in this DR
hnobjs = np.zeros(len(hsafes), dtype=int)
hnobjs[drbrickids] = drbricks["nobjs"]
#ADM make each OBJID for a SAFE/BADSKY +1 higher than any other OBJID in the DR
safes["BRICK_OBJID"] = hnobjs[safes["BRICKID"]] + 1
#ADM sort the safes array on BRICKID
safes = safes[safes["BRICKID"].argsort()]
#ADM remove zero entries from the histogram of BRICKIDs in safes, for speed
hsafes = hsafes[np.where(hsafes > 0)]
#ADM the count by which to augment each OBJID to make unique OBJIDs for safes
objsadd = np.hstack([np.arange(i) for i in hsafes])
#ADM finalize the OBJID for each SAFE target
safes["BRICK_OBJID"] += objsadd
#ADM finally, update the TARGETID with the OBJID, the BRICKID, and the fact these are skies
safes["TARGETID"] = encode_targetid(objid=safes['BRICK_OBJID'],
brickid=safes['BRICKID'],
sky=1)
#ADM return the input targs with the SAFE targets appended
return np.hstack([targs, safes])
'0436p020', '0438p020', '0441p020', '0443p020', '0446p020', '0448p020',
'0451p020', '0453p020', '0456p020', '0458p020', '0461p020', '0463p020',
'0436p022', '0438p022', '0441p022', '0443p022', '0446p022', '0448p022',
'0451p022', '0453p022', '0456p022', '0458p022', '0461p022', '0463p022',
'0436p025', '0438p025', '0441p025', '0443p025', '0446p025', '0448p025',
'0451p025', '0453p025', '0456p025', '0458p025', '0461p025', '0463p025'
]
# nside = 64.
# items = ['0443p000', '0446p000', '0448p000', '0451p000', '0453p000', '0456p000', '0443p002', '0446p002', '0448p002', '0451p002', '0453p002', '0456p002', '0443p005', '0446p005', '0448p005', '0451p005', '0453p005', '0456p005', '0443p007', '0446p007', '0448p007', '0451p007', '0453p007', '0456p007', '0443p010', '0446p010', '0448p010', '0451p010', '0453p010', '0456p010', '0443p012', '0446p012', '0448p012', '0451p012', '0453p012', '0456p012']
area = len(items) * 0.25 * 0.25
print(area, hp.pixelfunc.nside2pixarea(nside, degrees=True))
bricktable = brick.Bricks(bricksize=0.25).to_table()
bricktable['A'] = [
'{:04d}'.format(x)
for x in np.array(bricktable['RA1'] * 10.).astype(np.int)
]
bricktable['B'] = [
'{:04d}'.format(x)
for x in np.array(bricktable['DEC1'] * 10.).astype(np.int)
]
bricktable['C'] = [
'{:04d}'.format(x)
for x in np.array(bricktable['RA2'] * 10.).astype(np.int)
]
bricktable['D'] = [
