def supplement_skies(nskiespersqdeg=None,
numproc=16,
gaiadir=None,
nside=None,
pixlist=None,
mindec=-30.,
mingalb=10.,
radius=2.):
"""Generate supplemental sky locations using Gaia-G-band avoidance.
Parameters
----------
nskiespersqdeg : :class:`float`, optional
The minimum DENSITY of sky fibers to generate. Defaults to
reading from :func:`~desimodel.io` with a margin of 4x.
numproc : :class:`int`, optional, defaults to 16
The number of processes over which to parallelize.
gaiadir : :class:`str`, optional, defaults to $GAIA_DIR
The GAIA_DIR environment variable is set to this directory.
If None is passed, then it's assumed to already exist.
nside : :class:`int`, optional, defaults to `None`
(NESTED) HEALPix `nside` to use with `pixlist`.
pixlist : :class:`list` or `int`, optional, defaults to `None`
Only return targets in a set of (NESTED) HEALpixels at the
supplied `nside`. Useful for parallelizing across nodes.
The first entry sets RELEASE for TARGETIDs, and must be < 1000
(to prevent confusion with DR1 and above).
mindec : :class:`float`, optional, defaults to -30
Minimum declination (o) to include for output sky locations.
mingalb : :class:`float`, optional, defaults to 10
Closest latitude to Galactic plane for output sky locations
(e.g. send 10 to limit to areas beyond -10o <= b < 10o).
radius : :class:`float`, optional, defaults to 2
Radius at which to avoid (all) Gaia sources (arcseconds).
Returns
-------
:class:`~numpy.ndarray`
a structured array of supplemental sky positions in the DESI sky
target format within the passed `mindec` and `mingalb` limits.
Notes
-----
- The environment variable $GAIA_DIR must be set, or `gaiadir`
must be passed.
"""
log.info("running on {} processors".format(numproc))
# ADM if the GAIA directory was passed, set it.
if gaiadir is not None:
os.environ["GAIA_DIR"] = gaiadir
# ADM if needed, determine the density of sky fibers to generate.
if nskiespersqdeg is None:
nskiespersqdeg = density_of_sky_fibers(margin=4)
# ADM determine the HEALPixel nside of the standard Gaia files.
anyfiles = find_gaia_files([0, 0], radec=True)
hdr = fitsio.read_header(anyfiles[0], "GAIAHPX")
nsidegaia = hdr["HPXNSIDE"]
# ADM create a set of random locations accounting for mindec.
log.info("Generating supplemental sky locations at Dec > {}o...t={:.1f}s".
format(mindec,
time() - start))
from desitarget.randoms import randoms_in_a_brick_from_edges
ras, decs = randoms_in_a_brick_from_edges(0.,
360.,
mindec,
90.,
density=nskiespersqdeg,
wrap=False)
# ADM limit randoms by HEALPixel, if requested.
if pixlist is not None:
inhp = is_in_hp([ras, decs], nside, pixlist, radec=True)
ras, decs = ras[inhp], decs[inhp]
# ADM limit randoms by mingalb.
log.info(
"Generated {} sky locations. Limiting to |b| > {}o...t={:.1f}s".format(
len(ras), mingalb,
time() - start))
bnorth = is_in_gal_box([ras, decs], [0, 360, mingalb, 90], radec=True)
bsouth = is_in_gal_box([ras, decs], [0, 360, -90, -mingalb], radec=True)
ras, decs = ras[bnorth | bsouth], decs[bnorth | bsouth]
# ADM find which Gaia HEALPixels are occupied by the random points.
log.info(
"Cut to {} sky locations. Finding their Gaia HEALPixels...t={:.1f}s".
format(len(ras),
time() - start))
theta, phi = np.radians(90 - decs), np.radians(ras)
pixels = hp.ang2pix(nsidegaia, theta, phi, nest=True)
upixels = np.unique(pixels)
npixels = len(upixels)
log.info("Running across {} Gaia HEALPixels.".format(npixels))
# ADM parallelize across pixels. The function to run on every pixel.
def _get_supp(pix):
"""wrapper on get_supp_skies() given a HEALPixel"""
ii = (pixels == pix)
return get_supp_skies(ras[ii], decs[ii], radius=radius)
# ADM this is just to count pixels in _update_status.
npix = np.zeros((), dtype='i8')
t0 = time()
def _update_status(result):
"""wrapper function for the critical reduction operation,
that occurs on the main parallel process"""
if npix % 500 == 0 and npix > 0:
rate = npix / (time() - t0)
log.info('{}/{} HEALPixels; {:.1f} pixels/sec'.format(
npix, npixels, rate))
npix[...] += 1 # this is an in-place modification.
return result
# - Parallel process across the unique pixels.
if numproc > 1:
pool = sharedmem.MapReduce(np=numproc)
with pool:
supp = pool.map(_get_supp, upixels, reduce=_update_status)
else:
supp = []
for upix in upixels:
supp.append(_update_status(_get_supp(upix)))
# ADM Concatenate the parallelized results into one rec array.
supp = np.concatenate(supp)
# ADM build the OBJIDs from the number of sources per brick.
# ADM the for loop doesn't seem the smartest way, but it is O(n).
log.info("Begin assigning OBJIDs to bricks...t={:.1f}s".format(time() -
start))
brxid = supp["BRICKID"]
# ADM start each brick counting from zero.
cntr = np.zeros(np.max(brxid) + 1, dtype=int)
objid = []
for ibrx in brxid:
cntr[ibrx] += 1
objid.append(cntr[ibrx])
# ADM ensure the number of sky positions that were generated doesn't exceed
# ADM the largest possible OBJID (which is unlikely).
if np.any(cntr > 2**targetid_mask.OBJID.nbits):
log.fatal(
'{} sky locations requested in brick {}, but OBJID cannot exceed {}'
.format(nskies, brickname, 2**targetid_mask.OBJID.nbits))
raise ValueError
supp["OBJID"] = np.array(objid)
log.info("Assigned OBJIDs to bricks...t={:.1f}s".format(time() - start))
# ADM if splitting by HEALPixels, use the input list to
# ADM set RELEASE so we don't have duplicate TARGETIDs.
if pixlist is not None and pixlist[0] < 1000:
supp["RELEASE"] = pixlist[0]
else:
msg = "First entry in pixlist ({}) > 1000. This sets ".format(
pixlist[0])
msg += "RELEASE for SUPP_SKIES. > 1000 may duplicate TARGETID for SKIES!"
log.critical(msg)
raise IOError
# ADM add the TARGETID, DESITARGET bits etc.
nskies = len(supp)
desi_target = np.zeros(nskies, dtype='>i8')
desi_target |= desi_mask.SKY
desi_target |= desi_mask.SUPP_SKY
dum = np.zeros_like(desi_target)
supp = finalize(supp, desi_target, dum, dum, sky=1)
log.info('Done...t={:.1f}s'.format(time() - start))
return supp
def gaia_in_file(infile,
maglim=18,
mindec=-30.,
mingalb=10.,
nside=None,
pixlist=None,
addobjid=False):
"""Retrieve the Gaia objects from a HEALPixel-split Gaia file.
Parameters
----------
infile : :class:`str`
File name of a single Gaia "healpix" file.
maglim : :class:`float`, optional, defaults to 18
Magnitude limit for GFAs in Gaia G-band.
mindec : :class:`float`, optional, defaults to -30
Minimum declination (o) to include for output Gaia objects.
mingalb : :class:`float`, optional, defaults to 10
Closest latitude to Galactic plane for output Gaia objects
(e.g. send 10 to limit to areas beyond -10o <= b < 10o)"
nside : :class:`int`, optional, defaults to `None`
(NESTED) HEALPix `nside` to use with `pixlist`.
pixlist : :class:`list` or `int`, optional, defaults to `None`
Only return sources in a set of (NESTED) HEALpixels at the
supplied `nside`.
addobjid : :class:`bool`, optional, defaults to ``False``
If ``True``, include, in the output, a column "GAIA_OBJID"
that is the integer number of each row read from file.
Returns
-------
:class:`~numpy.ndarray`
Gaia objects in the passed Gaia file brighter than `maglim`,
formatted according to `desitarget.gfa.gfadatamodel`.
Notes
-----
- A "Gaia healpix file" here is as made by, e.g.
:func:`~desitarget.gaiamatch.gaia_fits_to_healpix()`
"""
# ADM read in the Gaia file and limit to the passed magnitude.
objs = read_gaia_file(infile, addobjid=addobjid)
ii = objs['GAIA_PHOT_G_MEAN_MAG'] < maglim
objs = objs[ii]
# ADM rename GAIA_RA/DEC to RA/DEC, as that's what's used for GFAs.
for radec in ["RA", "DEC"]:
objs.dtype.names = [
radec if col == "GAIA_" + radec else col
for col in objs.dtype.names
]
# ADM initiate the GFA data model.
dt = gfadatamodel.dtype.descr
if addobjid:
for tup in ('GAIA_BRICKID', '>i4'), ('GAIA_OBJID', '>i4'):
dt.append(tup)
gfas = np.zeros(len(objs), dtype=dt)
# ADM make sure all columns initially have "ridiculous" numbers
gfas[...] = -99.
for col in gfas.dtype.names:
if isinstance(gfas[col][0].item(), (bytes, str)):
gfas[col] = 'U'
if isinstance(gfas[col][0].item(), int):
gfas[col] = -1
# ADM some default special cases. Default to REF_EPOCH of Gaia DR2,
# ADM make RA/Dec very precise for Gaia measurements.
gfas["REF_EPOCH"] = 2015.5
gfas["RA_IVAR"], gfas["DEC_IVAR"] = 1e16, 1e16
# ADM populate the common columns in the Gaia/GFA data models.
cols = set(gfas.dtype.names).intersection(set(objs.dtype.names))
for col in cols:
gfas[col] = objs[col]
# ADM update the Gaia morphological type.
gfas["TYPE"] = gaia_morph(gfas)
# ADM populate the BRICKID columns.
gfas["BRICKID"] = bricks.brickid(gfas["RA"], gfas["DEC"])
# ADM limit by HEALPixel first as that's the fastest.
if pixlist is not None:
inhp = is_in_hp(gfas, nside, pixlist)
gfas = gfas[inhp]
# ADM limit by Dec first to speed transform to Galactic coordinates.
decgood = is_in_box(gfas, [0., 360., mindec, 90.])
gfas = gfas[decgood]
# ADM now limit to requesed Galactic latitude range.
if mingalb > 1e-9:
bbad = is_in_gal_box(gfas, [0., 360., -mingalb, mingalb])
gfas = gfas[~bbad]
return gfas
def select_gfas(infiles,
maglim=18,
numproc=4,
nside=None,
pixlist=None,
bundlefiles=None,
extra=None,
mindec=-30,
mingalb=10,
addurat=True):
"""Create a set of GFA locations using Gaia and matching to sweeps.
Parameters
----------
infiles : :class:`list` or `str`
A list of input filenames (sweep files) OR a single filename.
maglim : :class:`float`, optional, defaults to 18
Magnitude limit for GFAs in Gaia G-band.
numproc : :class:`int`, optional, defaults to 4
The number of parallel processes to use.
nside : :class:`int`, optional, defaults to `None`
(NESTED) HEALPix `nside` to use with `pixlist` and `bundlefiles`.
pixlist : :class:`list` or `int`, optional, defaults to `None`
Only return targets in a set of (NESTED) HEALpixels at the
supplied `nside`. Useful for parallelizing.
bundlefiles : :class:`int`, defaults to `None`
If not `None`, then, instead of selecting gfas, print the slurm
script to run in pixels at `nside`. Is an integer rather than
a boolean for historical reasons.
extra : :class:`str`, optional
Extra command line flags to be passed to the executable lines in
the output slurm script. Used in conjunction with `bundlefiles`.
mindec : :class:`float`, optional, defaults to -30
Minimum declination (o) for output sources that do NOT match
an object in the passed `infiles`.
mingalb : :class:`float`, optional, defaults to 10
Closest latitude to Galactic plane for output sources that
do NOT match an object in the passed `infiles` (e.g. send
10 to limit to regions beyond -10o <= b < 10o)".
addurat : :class:`bool`, optional, defaults to ``True``
If ``True`` then substitute proper motions from the URAT
catalog where Gaia is missing proper motions. Requires that
the :envvar:`URAT_DIR` is set and points to data downloaded and
formatted by, e.g., :func:`~desitarget.uratmatch.make_urat_files`.
Returns
-------
:class:`~numpy.ndarray`
GFA objects from Gaia with the passed geometric constraints
limited to the passed maglim and matched to the passed input
files, formatted according to `desitarget.gfa.gfadatamodel`.
Notes
-----
- If numproc==1, use the serial code instead of parallel code.
- If numproc > 4, then numproc=4 is enforced for (just those)
parts of the code that are I/O limited.
"""
# ADM the code can have memory issues for nside=2 with large numproc.
if nside is not None and nside < 4 and numproc > 8:
msg = 'Memory may be an issue near Plane for nside < 4 and numproc > 8'
log.warning(msg)
# ADM force to no more than numproc=4 for I/O limited processes.
numproc4 = numproc
if numproc4 > 4:
log.info('Forcing numproc to 4 for I/O limited parts of code')
numproc4 = 4
# ADM convert a single file, if passed to a list of files.
if isinstance(infiles, str):
infiles = [
infiles,
]
# ADM check that files exist before proceeding.
for filename in infiles:
if not os.path.exists(filename):
msg = "{} doesn't exist".format(filename)
log.critical(msg)
raise ValueError(msg)
# ADM if the pixlist option was sent, we'll need to
# ADM know which HEALPixels touch each file.
if pixlist is not None:
filesperpixel, _, _ = sweep_files_touch_hp(nside, pixlist, infiles)
# ADM if the bundlefiles option was sent, call the packing code.
if bundlefiles is not None:
# ADM were files from one or two input directories passed?
surveydirs = list(set([os.path.dirname(fn) for fn in infiles]))
bundle_bricks([0],
bundlefiles,
nside,
gather=False,
prefix='gfas',
surveydirs=surveydirs,
extra=extra)
return
# ADM restrict to input files in a set of HEALPixels, if requested.
if pixlist is not None:
infiles = list(set(np.hstack([filesperpixel[pix] for pix in pixlist])))
if len(infiles) == 0:
log.info('ZERO sweep files in passed pixel list!!!')
log.info("Processing files in (nside={}, pixel numbers={}) HEALPixels".
format(nside, pixlist))
nfiles = len(infiles)
# ADM a little more information if we're slurming across nodes.
if os.getenv('SLURMD_NODENAME') is not None:
log.info('Running on Node {}'.format(os.getenv('SLURMD_NODENAME')))
# ADM the critical function to run on every file.
def _get_gfas(fn):
'''wrapper on gaia_gfas_from_sweep() given a file name'''
return gaia_gfas_from_sweep(fn, maglim=maglim)
# ADM this is just to count sweeps files in _update_status.
t0 = time()
nfile = np.zeros((), dtype='i8')
def _update_status(result):
"""wrapper function for the critical reduction operation,
that occurs on the main parallel process"""
if nfile % 20 == 0 and nfile > 0:
elapsed = (time() - t0) / 60.
rate = nfile / elapsed / 60.
log.info('{}/{} files; {:.1f} files/sec...t = {:.1f} mins'.format(
nfile, nfiles, rate, elapsed))
nfile[...] += 1 # this is an in-place modification.
return result
# - Parallel process input files.
if len(infiles) > 0:
if numproc4 > 1:
pool = sharedmem.MapReduce(np=numproc4)
with pool:
gfas = pool.map(_get_gfas, infiles, reduce=_update_status)
else:
gfas = list()
for file in infiles:
gfas.append(_update_status(_get_gfas(file)))
gfas = np.concatenate(gfas)
# ADM resolve any duplicates between imaging data releases.
gfas = resolve(gfas)
# ADM retrieve Gaia objects in the DESI footprint or passed tiles.
log.info('Retrieving additional Gaia objects...t = {:.1f} mins'.format(
(time() - t0) / 60))
gaia = all_gaia_in_tiles(maglim=maglim,
numproc=numproc4,
allsky=True,
mindec=mindec,
mingalb=mingalb,
nside=nside,
pixlist=pixlist)
# ADM remove any duplicates. Order is important here, as np.unique
# ADM keeps the first occurence, and we want to retain sweeps
# ADM information as much as possible.
if len(infiles) > 0:
gfas = np.concatenate([gfas, gaia])
_, ind = np.unique(gfas["REF_ID"], return_index=True)
gfas = gfas[ind]
else:
gfas = gaia
# ADM for zero/NaN proper motion objects, add URAT proper motions.
if addurat:
ii = ((np.isnan(gfas["PMRA"]) | (gfas["PMRA"] == 0)) &
(np.isnan(gfas["PMDEC"]) | (gfas["PMDEC"] == 0)))
log.info(
'Adding URAT for {} objects with no PMs...t = {:.1f} mins'.format(
np.sum(ii), (time() - t0) / 60))
urat = add_urat_pms(gfas[ii], numproc=numproc)
log.info(
'Found an additional {} URAT objects...t = {:.1f} mins'.format(
np.sum(urat["URAT_ID"] != -1), (time() - t0) / 60))
for col in "PMRA", "PMDEC", "URAT_ID", "URAT_SEP":
gfas[col][ii] = urat[col]
# ADM restrict to only GFAs in a set of HEALPixels, if requested.
if pixlist is not None:
ii = is_in_hp(gfas, nside, pixlist)
gfas = gfas[ii]
return gfas
def match_secondary(infile, scxtargs, sep=1., scxdir=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
infile : :class:`str`
The full path to a file containing primary targets.
scxtargs : :class:`~numpy.ndarray`
An array of secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets. If passed,
this is written to the output primary file header as `SCNDDIR`.
Returns
-------
:class:`~numpy.ndarray`
The array of secondary targets, with the `TARGETID` bit
updated with any matching primary targets from `infile`.
Notes
-----
- The primary target `infiles` are written back to their
original path with `.fits` changed to `-wscnd.fits` and the
`SCND_TARGET` bit populated for matching targets.
"""
# ADM just the file name for logging.
fn = os.path.basename(infile)
# ADM read in the primary targets.
log.info('Reading primary targets file {}...t={:.1f}s'.format(
infile,
time() - start))
intargs, hdr = fitsio.read(infile, extension="TARGETS", header=True)
# ADM fail if file's already been matched to secondary targets.
if "SCNDDIR" in hdr:
msg = "{} already matched to secondary targets".format(fn) \
+ " (did you mean to remove {}?)!!!".format(fn)
log.critical(msg)
raise ValueError(msg)
# ADM add the SCNDDIR to the primary targets file header.
hdr["SCNDDIR"] = scxdir
# ADM add a SCND_TARGET column to the primary targets.
dt = intargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(intargs), dtype=dt)
for col in intargs.dtype.names:
targs[col] = intargs[col]
# ADM match to all secondary targets for non-custom primary files.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if 'FILEHPX' in hdr:
nside, pix = hdr['FILENSID'], hdr['FILEHPX']
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(intargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(intargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(fn, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# ADM update the secondary targets with the primary TARGETID.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM form the output primary file name and write the file.
base, ext = os.path.splitext(infile)
outfile = "{}{}{}".format(base, '-wscnd', ext)
log.info('Writing updated primary targets to {}...t={:.1f}s'.format(
outfile,
time() - start))
fitsio.write(outfile, targs, extname='TARGETS', header=hdr, clobber=True)
log.info('Done for {}...t={:.1f}s'.format(fn, time() - start))
return scxtargs
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
primtargs : :class:`~numpy.ndarray`
An array of primary targets.
scndout : :class`~numpy.ndarray`
Name of a sub-directory to which to write the information in
`desitarget.secondary.outdatamodel` with `TARGETID` and (the
highest) `PRIORITY_INIT` updated with matching primary info.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
pix : :class:`list`, optional, defaults to `None`
Limit secondary targets to (NESTED) HEALpixels that touch
pix at the supplied `nside`, as a speed-up.
nside : :class:`int`, optional, defaults to `None`
The (NESTED) HEALPixel nside to be used with `pixlist`.
Returns
-------
:class:`~numpy.ndarray`
The array of primary targets, with the `SCND_TARGET` bit
populated for matches to secondary targets
"""
# ADM add a SCND_TARGET column to the primary targets.
dt = primtargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(primtargs), dtype=dt)
for col in primtargs.dtype.names:
targs[col] = primtargs[col]
# ADM check if this is an SV or main survey file.
cols, mx, surv = main_cmx_or_sv(targs, scnd=True)
log.info('running on the {} survey...'.format(surv))
if surv != 'main':
scxdir = os.path.join(scxdir, surv)
# ADM read in non-OVERRIDE secondary targets.
scxtargs = read_files(scxdir, mx[3])
scxtargs = scxtargs[~scxtargs["OVERRIDE"]]
# ADM match primary targets to non-OVERRIDE secondary targets.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if nside is not None and pix is not None:
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(primtargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(primtargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(scndout, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
desi_mask = desimasks[0]
targs[desicols[0]][umtargs] |= desi_mask.SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# APC Secondary target bits only affect PRIORITY, NUMOBS and
# APC obsconditions for specific DESI_TARGET bits
# APC See https://github.com/desihub/desitarget/pull/530
# APC Only consider primary targets with secondary bits set
scnd_update = (targs[desicols[0]] & desi_mask['SCND_ANY']) != 0
if np.any(scnd_update):
# APC Allow changes to primaries if the DESI_TARGET bitmask has
# APC only the following bits set, in any combination.
log.info(
'Testing if secondary targets can update {} matched primaries'.
format(scnd_update.sum()))
update_from_scnd_bits = desi_mask['SCND_ANY'] | desi_mask[
'MWS_ANY'] | desi_mask['STD_BRIGHT'] | desi_mask[
'STD_FAINT'] | desi_mask['STD_WD']
scnd_update &= ((targs[desicols[0]] & ~update_from_scnd_bits) == 0)
log.info(
'Setting new priority, numobs and obsconditions from secondary for {} matched primaries'
.format(scnd_update.sum()))
# APC Primary and secondary obsconditions are or'd
scnd_obscon = set_obsconditions(targs[scnd_update], scnd=True)
targs['OBSCONDITIONS'][scnd_update] &= scnd_obscon
# APC bit of a hack here
# APC Check for _BRIGHT, _DARK split in column names
darkbright = 'NUMOBS_INIT_DARK' in targs.dtype.names
if darkbright:
ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"]
else:
ender, obscon = [""], [
"DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18"
]
# APC secondaries can increase priority and numobs
for edr, oc in zip(ender, obscon):
pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr
scnd_priority, scnd_numobs = initial_priority_numobs(
targs[scnd_update], obscon=oc, scnd=True)
targs[nc][scnd_update] = np.maximum(targs[nc][scnd_update],
scnd_numobs)
targs[pc][scnd_update] = np.maximum(targs[pc][scnd_update],
scnd_priority)
# ADM update the secondary targets with the primary information.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM the maximum priority will be used to break ties in the
# ADM unlikely event that a secondary matches two primaries.
hipri = np.maximum(targs["PRIORITY_INIT_DARK"],
targs["PRIORITY_INIT_BRIGHT"])
scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs]
# ADM write the secondary targets that have updated TARGETIDs.
ii = scxtargs["TARGETID"] != -1
nmatches = np.sum(ii)
log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format(
nmatches, scndout,
time() - start))
if nmatches > 0:
hdr = fitsio.FITSHDR()
hdr["SURVEY"] = surv
fitsio.write(scndout,
scxtargs[ii],
extname='SCND_TARG',
header=hdr,
clobber=True)
log.info('Done...t={:.1f}s'.format(time() - start))
return targs
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
primtargs : :class:`~numpy.ndarray`
An array of primary targets.
scndout : :class`~numpy.ndarray`
Name of a sub-directory to which to write the information in
`desitarget.secondary.outdatamodel` with `TARGETID` and (the
highest) `PRIORITY_INIT` updated with matching primary info.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
pix : :class:`list`, optional, defaults to `None`
Limit secondary targets to (NESTED) HEALpixels that touch
pix at the supplied `nside`, as a speed-up.
nside : :class:`int`, optional, defaults to `None`
The (NESTED) HEALPixel nside to be used with `pixlist`.
Returns
-------
:class:`~numpy.ndarray`
The array of primary targets, with the `SCND_TARGET` bit
populated for matches to secondary targets
"""
# ADM add a SCND_TARGET column to the primary targets.
dt = primtargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(primtargs), dtype=dt)
for col in primtargs.dtype.names:
targs[col] = primtargs[col]
# ADM check if this is an SV or main survey file.
cols, mx, surv = main_cmx_or_sv(targs, scnd=True)
log.info('running on the {} survey...'.format(surv))
if surv != 'main':
scxdir = os.path.join(scxdir, surv)
# ADM read in non-OVERRIDE secondary targets.
scxtargs = read_files(scxdir, mx[3])
scxtargs = scxtargs[~scxtargs["OVERRIDE"]]
# ADM match primary targets to non-OVERRIDE secondary targets.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if nside is not None and pix is not None:
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(primtargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(primtargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(scndout, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# ADM update the secondary targets with the primary information.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM the maximum priority will be used to break ties in the
# ADM unlikely event that a secondary matches two primaries.
hipri = np.maximum(targs["PRIORITY_INIT_DARK"],
targs["PRIORITY_INIT_BRIGHT"])
scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs]
# ADM write the secondary targets that have updated TARGETIDs.
ii = scxtargs["TARGETID"] != -1
nmatches = np.sum(ii)
log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format(
nmatches, scndout,
time() - start))
if nmatches > 0:
hdr = fitsio.FITSHDR()
hdr["SURVEY"] = surv
fitsio.write(scndout,
scxtargs[ii],
extname='SCND_TARG',
header=hdr,
clobber=True)
log.info('Done...t={:.1f}s'.format(time() - start))
return targs
def read_targets_in_hp(hpdirname, nside, pixlist, columns=None):
"""Read in targets in a set of HEALPixels.
Parameters
----------
hpdirname : :class:`str`
Full path to either a directory containing targets that
have been partitioned by HEALPixel (i.e. as made by
`select_targets` with the `bundle_files` option). Or the
name of a single file of targets.
nside : :class:`int`
The (NESTED) HEALPixel nside.
pixlist : :class:`list` or `int` or `~numpy.ndarray`
Return targets in these HEALPixels at the passed `nside`.
columns : :class:`list`, optional
Only read in these target columns.
Returns
-------
:class:`~numpy.ndarray`
An array of targets in the passed pixels.
"""
# ADM we'll need RA/Dec for final cuts, so ensure they're read.
addedcols = []
columnscopy = None
if columns is not None:
# ADM make a copy of columns, as it's a kwarg we'll modify.
columnscopy = columns.copy()
for radec in ["RA", "DEC"]:
if radec not in columnscopy:
columnscopy.append(radec)
addedcols.append(radec)
# ADM if a directory was passed, do fancy HEALPixel parsing...
if os.path.isdir(hpdirname):
# ADM check, and grab information from, the target directory.
filenside, filedict = check_hp_target_dir(hpdirname)
# ADM change the passed pixels to the nside of the file schema.
filepixlist = nside2nside(nside, filenside, pixlist)
# ADM only consider pixels for which we have a file.
isindict = [pix in filedict for pix in filepixlist]
filepixlist = filepixlist[isindict]
# ADM make sure each file is only read once.
infiles = set([filedict[pix] for pix in filepixlist])
# ADM read in the files and concatenate the resulting targets.
targets = []
for infile in infiles:
targets.append(fitsio.read(infile, columns=columnscopy))
targets = np.concatenate(targets)
# ADM ...otherwise just read in the targets.
else:
targets = fitsio.read(hpdirname, columns=columnscopy)
# ADM restrict the targets to the actual requested HEALPixels...
ii = is_in_hp(targets, nside, pixlist)
# ADM ...and remove RA/Dec columns if we added them.
targets = rfn.drop_fields(targets[ii], addedcols)
return targets
def mask_targets(targs, inmaskdir, nside=2, pixlist=None, bricks_are_hpx=False):
"""Add bits for if objects occupy masks, and SAFE (BADSKY) locations.
Parameters
----------
targs : :class:`str` or `~numpy.ndarray`
An array of targets/skies etc. created by, e.g.,
:func:`desitarget.cuts.select_targets()` OR the filename of a
file that contains such a set of targets/skies, etc.
inmaskdir : :class:`str`, optional
An input bright star mask file or HEALPixel-split directory as
made by :func:`desitarget.brightmask.make_bright_star_mask()`
nside : :class:`int`, optional, defaults to 2
The nside at which the targets were generated. If the mask is
a HEALPixel-split directory, then this helps to perform more
efficient masking as only the subset of masks that are in
pixels containing `targs` at this `nside` will be considered
(together with neighboring pixels to account for edge effects).
pixlist : :class:`list` or `int`, optional
A set of HEALPixels corresponding to the `targs`. Only the subset
of masks in HEALPixels in `pixlist` at `nside` will be considered
(together with neighboring pixels to account for edge effects).
If ``None``, then the pixels touched by `targs` is derived from
from `targs` itself.
bricks_are_hpx : :class:`bool`, optional, defaults to ``False``
Instead of using bricks to calculate BRICKIDs, use HEALPixels at
the "standard" size from :func:`gaiamatch.get_gaia_nside_brick()`.
Returns
-------
:class:`~numpy.ndarray`
Input targets with the `DESI_TARGET` column updated to reflect
the `BRIGHT_OBJECT` bits and SAFE (`BADSKY`) sky locations added
around the perimeter of the mask.
Notes
-----
- `Tech Note 2346`_ details SAFE (BADSKY) locations.
"""
t0 = time()
# ADM Check if targs is a file name or the structure itself.
if isinstance(targs, str):
if not os.path.exists(targs):
raise ValueError("{} doesn't exist".format(targs))
targs = fitsio.read(targs)
# ADM determine which pixels are occupied by targets.
if pixlist is None:
theta, phi = np.radians(90-targs["DEC"]), np.radians(targs["RA"])
pixlist = list(set(hp.ang2pix(nside, theta, phi, nest=True)))
else:
# ADM in case an integer was passed.
pixlist = np.atleast_1d(pixlist)
log.info("Masking using masks in {} at nside={} in HEALPixels={}".format(
inmaskdir, nside, pixlist))
pixlistwneigh = add_hp_neighbors(nside, pixlist)
# ADM read in the (potentially HEALPixel-split) mask.
sourcemask = io.read_targets_in_hp(inmaskdir, nside, pixlistwneigh)
ntargs = len(targs)
log.info('Total number of masks {}'.format(len(sourcemask)))
log.info('Total number of targets {}...t={:.1f}s'.format(ntargs, time()-t0))
# ADM update the bits depending on whether targets are in a mask.
# ADM also grab masks that contain or are near a target.
dt, mx = set_target_bits(targs, sourcemask, return_masks=True)
targs["DESI_TARGET"] = dt
inmasks, nearmasks = mx
# ADM generate SAFE locations for masks that contain a target.
safes = get_safe_targets(targs, sourcemask[inmasks],
bricks_are_hpx=bricks_are_hpx)
# ADM update the bits for the safe locations depending on whether
# ADM they're in a mask.
safes["DESI_TARGET"] = set_target_bits(safes, sourcemask)
# ADM it's possible that a safe location was generated outside of
# ADM the requested HEALPixels.
inhp = is_in_hp(safes, nside, pixlist)
safes = safes[inhp]
# ADM combine the targets and safe locations.
done = np.concatenate([targs, safes])
# ADM assert uniqueness of TARGETIDs.
stargs, ssafes = set(targs["TARGETID"]), set(safes["TARGETID"])
msg = "TARGETIDs for targets not unique"
assert len(stargs) == len(targs), msg
msg = "TARGETIDs for safes not unique"
assert len(ssafes) == len(safes), msg
msg = "TARGETIDs for safes duplicated in targets. Generating TARGETIDs"
msg += " backwards from maxobjid in get_safe_targets() has likely failed"
msg += " due to somehow generating a large number of safe locations."
assert len(stargs.intersection(ssafes)) == 0, msg
log.info('Generated {} SAFE (BADSKY) locations...t={:.1f}s'.format(
len(done)-ntargs, time()-t0))
# ADM remove any SAFE locations that are in bright masks (because they aren't really safe).
ii = (((done["DESI_TARGET"] & desi_mask.BAD_SKY) == 0) |
((done["DESI_TARGET"] & desi_mask.IN_BRIGHT_OBJECT) == 0))
done = done[ii]
log.info("...of these, {} SAFE (BADSKY) locations aren't in masks...t={:.1f}s"
.format(len(done)-ntargs, time()-t0))
log.info('Finishing up...t={:.1f}s'.format(time()-t0))
return done