def find_urat_files(objs, neighbors=True, radec=False):
"""Find full paths to URAT healpix files for objects by RA/Dec.
Parameters
----------
objs : :class:`~numpy.ndarray`
Array of objects. Must contain the columns "RA" and "DEC".
neighbors : :class:`bool`, optional, defaults to ``True``
Also return all pixels that touch the files of interest
to prevent edge effects (e.g. if a URAT source is 1 arcsec
away from a primary source and so in an adjacent pixel).
radec : :class:`bool`, optional, defaults to ``False``
If ``True`` then the passed `objs` is an [RA, Dec] list
instead of a rec array that contains "RA" and "DEC".
Returns
-------
:class:`list`
A list of all URAT files to read to account for objects at
the passed locations.
Notes
-----
- The environment variable $URAT_DIR must be set.
"""
# ADM the resolution at which the URAT HEALPix files are stored.
nside = _get_urat_nside()
# ADM check that the URAT_DIR is set and retrieve it.
uratdir = _get_urat_dir()
hpxdir = os.path.join(uratdir, 'healpix')
# ADM which flavor of RA/Dec was passed.
if radec:
ra, dec = objs
else:
ra, dec = objs["RA"], objs["DEC"]
# ADM convert RA/Dec to co-latitude and longitude in radians.
theta, phi = np.radians(90 - dec), np.radians(ra)
# ADM retrieve the pixels in which the locations lie.
pixnum = hp.ang2pix(nside, theta, phi, nest=True)
# ADM if neighbors was sent, then retrieve all pixels that touch each
# ADM pixel covered by the provided locations, to prevent edge effects...
if neighbors:
pixnum = add_hp_neighbors(nside, pixnum)
# ADM reformat file names in the URAT healpix format.
uratfiles = [
os.path.join(hpxdir, 'healpix-{:05d}.fits'.format(pn)) for pn in pixnum
]
# ADM restrict to only files/HEALPixels actually covered by URAT.
uratfiles = [fn for fn in uratfiles if os.path.exists(fn)]
return uratfiles
def find_gaia_files_tiles(tiles=None, neighbors=True):
"""
Parameters
----------
tiles : :class:`~numpy.ndarray`
Array of tiles, or ``None`` to use all DESI tiles from
:func:`desimodel.io.load_tiles`.
neighbors : :class:`bool`, optional, defaults to ``True``
Also return all neighboring pixels that touch the files of interest
in order to prevent edge effects (e.g. if a Gaia source is 1 arcsec
away from a primary source and so in an adjacent pixel).
Returns
-------
:class:`list`
A list of all Gaia files that touch the passed tiles.
Notes
-----
- The environment variables $GAIA_DIR and $DESIMODEL must be set.
"""
# ADM check that the DESIMODEL environement variable is set.
if os.environ.get('DESIMODEL') is None:
msg = "DESIMODEL environment variable must be set!!!"
log.critical(msg)
raise ValueError(msg)
# ADM the resolution at which the healpix files are stored.
nside = _get_gaia_nside()
# ADM check that the GAIA_DIR is set and retrieve it.
gaiadir = _get_gaia_dir()
hpxdir = os.path.join(gaiadir, 'healpix')
# ADM determine the pixels that touch the tiles.
from desimodel.footprint import tiles2pix
pixnum = tiles2pix(nside, tiles=tiles)
# ADM if neighbors was sent, then retrieve all pixels that touch each
# ADM pixel covered by the provided locations, to prevent edge effects...
if neighbors:
pixnum = add_hp_neighbors(nside, pixnum)
# ADM reformat in the Gaia healpix format used by desitarget.
gaiafiles = [
os.path.join(hpxdir, 'healpix-{:05d}.fits'.format(pn)) for pn in pixnum
]
return gaiafiles
def find_gaia_files_box(gaiabounds, neighbors=True):
"""Find full paths to all relevant Gaia healpix files in an RA/Dec box.
Parameters
----------
gaiabounds : :class:`list`
A region of the sky bounded by RA/Dec. Pass as a 4-entry list to
represent an area bounded by [RAmin, RAmax, DECmin, DECmax]
neighbors : :class:`bool`, optional, defaults to ``True``
Also return all neighboring pixels that touch the files of interest
in order to prevent edge effects (e.g. if a Gaia source is 1 arcsec
away from a primary source and so in an adjacent pixel)
Returns
-------
:class:`list`
A list of all Gaia files that need to be read in to account for objects
in the passed box.
Notes
-----
- Uses the `healpy` routines that rely on `fact`, so the usual
warnings about returning different pixel sets at different values
of `fact` apply. See:
https://healpy.readthedocs.io/en/latest/generated/healpy.query_polygon.html
- The environment variable $GAIA_DIR must be set.
"""
# ADM the resolution at which the healpix files are stored.
nside = _get_gaia_nside()
# ADM check that the GAIA_DIR is set and retrieve it.
gaiadir = _get_gaia_dir()
hpxdir = os.path.join(gaiadir, 'healpix')
# ADM determine the pixels that touch the box.
pixnum = hp_in_box(nside, gaiabounds, inclusive=True, fact=4)
# ADM if neighbors was sent, then retrieve all pixels that touch each
# ADM pixel covered by the provided locations, to prevent edge effects...
if neighbors:
pixnum = add_hp_neighbors(nside, pixnum)
# ADM reformat in the Gaia healpix format used by desitarget.
gaiafiles = [
os.path.join(hpxdir, 'healpix-{:05d}.fits'.format(pn)) for pn in pixnum
]
return gaiafiles
def find_gaia_files_beyond_gal_b(mingalb, neighbors=True):
"""Find full paths to Gaia healpix files beyond a Galactic b.
Parameters
----------
mingalb : :class:`float`
Closest latitude to Galactic plane to return HEALPixels
(e.g. send 10 to limit to pixels beyond -10o <= b < 10o).
neighbors : :class:`bool`, optional, defaults to ``True``
Also return files corresponding to neighboring pixels that touch
in order to prevent edge effects (e.g. if a Gaia source might be
1 arcsec beyond mingalb and so in an adjacent pixel).
Returns
-------
:class:`list`
All Gaia files that need to be read in to account for objects
further from the Galactic plane than `mingalb`.
Notes
-----
- The environment variable $GAIA_DIR must be set.
- :func:`desitarget.geomask.hp_beyond_gal_b()` is already quite
inclusive, so you may retrieve some extra files along the
`mingalb` boundary.
"""
# ADM the resolution at which the healpix files are stored.
nside = _get_gaia_nside()
# ADM check that the GAIA_DIR is set and retrieve it.
gaiadir = _get_gaia_dir()
hpxdir = os.path.join(gaiadir, 'healpix')
# ADM determine the pixels beyond mingalb.
pixnum = hp_beyond_gal_b(nside, mingalb, neighbors=True)
# ADM if neighbors was sent, retrieve all pixels that touch each
# ADM retrieved, to prevent edge effects...
if neighbors:
pixnum = add_hp_neighbors(nside, pixnum)
# ADM reformat in the Gaia healpix format used by desitarget.
gaiafiles = [
os.path.join(hpxdir, 'healpix-{:05d}.fits'.format(pn)) for pn in pixnum
]
return gaiafiles
def find_tycho_files_hp(nside, pixlist, neighbors=True):
"""Find full paths to Tycho healpix files in a set of HEALPixels.
Parameters
----------
nside : :class:`int`
(NESTED) HEALPixel nside.
pixlist : :class:`list` or `int`
A set of HEALPixels at `nside`.
neighbors : :class:`bool`, optional, defaults to ``True``
Also return files corresponding to all neighbors that touch the
pixels in `pixlist` to prevent edge effects (e.g. a Tycho source
is 1 arcsec outside of `pixlist` and so in an adjacent pixel).
Returns
-------
:class:`list`
A list of all Tycho files that need to be read in to account for
objects in the passed list of pixels.
Notes
-----
- The environment variable $TYCHO_DIR must be set.
"""
# ADM the resolution at which the healpix files are stored.
filenside = get_tycho_nside()
# ADM check that the TYCHO_DIR is set and retrieve it.
tychodir = get_tycho_dir()
hpxdir = os.path.join(tychodir, 'healpix')
# ADM work with pixlist as an array.
pixlist = np.atleast_1d(pixlist)
# ADM determine the pixels that touch the passed pixlist.
pixnum = nside2nside(nside, filenside, pixlist)
# ADM if neighbors was sent, then retrieve all pixels that touch each
# ADM pixel covered by the provided locations, to prevent edge effects...
if neighbors:
pixnum = add_hp_neighbors(filenside, pixnum)
# ADM reformat in the healpix format used by desitarget.
tychofiles = [os.path.join(hpxdir, io.hpx_filename(pn)) for pn in pixnum]
return tychofiles
def find_gaia_files(objs, neighbors=True):
"""Find full paths to all relevant Gaia healpix files for an object array.
Parameters
----------
objs : :class:`~numpy.ndarray`
Array of objects. Must contain at least the columns "RA" and "DEC".
neighbors : :class:`bool`, optional, defaults to ``True``
Also return all neighboring pixels that touch the files of interest
in order to prevent edge effects (e.g. if a Gaia source is 1 arcsec
away from a primary source and so in an adjacent pixel)
Returns
-------
:class:`list`
A list of all Gaia files that need to be read in to account for objects
at the passed locations.
Notes
-----
- The environment variable $GAIA_DIR must be set.
"""
# ADM the resolution at which the Gaia HEALPix files are stored.
nside = _get_gaia_nside()
# ADM check that the GAIA_DIR is set and retrieve it.
gaiadir = _get_gaia_dir()
hpxdir = os.path.join(gaiadir, 'healpix')
# ADM convert RA/Dec to co-latitude and longitude in radians.
theta, phi = np.radians(90 - objs["DEC"]), np.radians(objs["RA"])
# ADM retrieve the pixels in which the locations lie.
pixnum = hp.ang2pix(nside, theta, phi, nest=True)
# ADM if neighbors was sent, then retrieve all pixels that touch each
# ADM pixel covered by the provided locations, to prevent edge effects...
if neighbors:
pixnum = add_hp_neighbors(nside, pixnum)
# ADM reformat in the Gaia healpix format used by desitarget.
gaiafiles = [
os.path.join(hpxdir, 'healpix-{:05d}.fits'.format(pn)) for pn in pixnum
]
return gaiafiles
def mask_targets(targs, inmaskdir, nside=2, pixlist=None):
"""Add bits for if objects occupy masks, and SAFE (BADSKY) locations.
Parameters
----------
targs : :class:`str` or `~numpy.ndarray`
An array of targets/skies etc. created by, e.g.,
:func:`desitarget.cuts.select_targets()` OR the filename of a
file that contains such a set of targets/skies, etc.
inmaskdir : :class:`str`, optional
An input bright star mask file or HEALPixel-split directory as
made by :func:`desitarget.brightmask.make_bright_star_mask()`
nside : :class:`int`, optional, defaults to 2
The nside at which the targets were generated. If the mask is
a HEALPixel-split directory, then this helps to perform more
efficient masking as only the subset of masks that are in
pixels containing `targs` at this `nside` will be considered
(together with neighboring pixels to account for edge effects).
pixlist : :class:`list` or `int`, optional
A set of HEALPixels corresponding to the `targs`. Only the subset
of masks in HEALPixels in `pixlist` at `nside` will be considered
(together with neighboring pixels to account for edge effects).
If ``None``, then the pixels touched by `targs` is derived from
from `targs` itself.
Returns
-------
:class:`~numpy.ndarray`
Input targets with the `DESI_TARGET` column updated to reflect
the `BRIGHT_OBJECT` bits and SAFE (`BADSKY`) sky locations added
around the perimeter of the mask.
Notes
-----
- `Tech Note 2346`_ details SAFE (BADSKY) locations.
"""
t0 = time()
# ADM Check if targs is a file name or the structure itself.
if isinstance(targs, str):
if not os.path.exists(targs):
raise ValueError("{} doesn't exist".format(targs))
targs = fitsio.read(targs)
# ADM determine which pixels are occupied by targets.
if pixlist is None:
theta, phi = np.radians(90-targs["DEC"]), np.radians(targs["RA"])
pixlist = list(set(hp.ang2pix(nside, theta, phi, nest=True)))
else:
# ADM in case an integer was passed.
pixlist = np.atleast_1d(pixlist)
log.info("Masking using masks in {} at nside={} in HEALPixels={}".format(
inmaskdir, nside, pixlist))
pixlist = add_hp_neighbors(nside, pixlist)
# ADM read in the (potentially HEALPixel-split) mask.
sourcemask = io.read_targets_in_hp(inmaskdir, nside, pixlist)
ntargs = len(targs)
log.info('Total number of masks {}'.format(len(sourcemask)))
log.info('Total number of targets {}...t={:.1f}s'.format(ntargs, time()-t0))
# ADM update the bits depending on whether targets are in a mask.
# ADM also grab masks that contain or are near a target.
dt, mx = set_target_bits(targs, sourcemask, return_masks=True)
targs["DESI_TARGET"] = dt
inmasks, nearmasks = mx
# ADM generate SAFE locations for masks that contain a target.
safes = get_safe_targets(targs, sourcemask[inmasks])
# ADM update the bits for the safe locations depending on whether
# ADM they're in a mask.
safes["DESI_TARGET"] = set_target_bits(safes, sourcemask)
# ADM combine the targets and safe locations.
done = np.concatenate([targs, safes])
log.info('Generated {} SAFE (BADSKY) locations...t={:.1f}s'.format(
len(done)-ntargs, time()-t0))
# ADM remove any SAFE locations that are in bright masks (because they aren't really safe).
ii = (((done["DESI_TARGET"] & desi_mask.BAD_SKY) == 0) |
((done["DESI_TARGET"] & desi_mask.IN_BRIGHT_OBJECT) == 0))
done = done[ii]
log.info("...of these, {} SAFE (BADSKY) locations aren't in masks...t={:.1f}s"
.format(len(done)-ntargs, time()-t0))
log.info('Finishing up...t={:.1f}s'.format(time()-t0))
return done
def match_secondary(infile, scxtargs, sep=1., scxdir=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
infile : :class:`str`
The full path to a file containing primary targets.
scxtargs : :class:`~numpy.ndarray`
An array of secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets. If passed,
this is written to the output primary file header as `SCNDDIR`.
Returns
-------
:class:`~numpy.ndarray`
The array of secondary targets, with the `TARGETID` bit
updated with any matching primary targets from `infile`.
Notes
-----
- The primary target `infiles` are written back to their
original path with `.fits` changed to `-wscnd.fits` and the
`SCND_TARGET` bit populated for matching targets.
"""
# ADM just the file name for logging.
fn = os.path.basename(infile)
# ADM read in the primary targets.
log.info('Reading primary targets file {}...t={:.1f}s'.format(
infile,
time() - start))
intargs, hdr = fitsio.read(infile, extension="TARGETS", header=True)
# ADM fail if file's already been matched to secondary targets.
if "SCNDDIR" in hdr:
msg = "{} already matched to secondary targets".format(fn) \
+ " (did you mean to remove {}?)!!!".format(fn)
log.critical(msg)
raise ValueError(msg)
# ADM add the SCNDDIR to the primary targets file header.
hdr["SCNDDIR"] = scxdir
# ADM add a SCND_TARGET column to the primary targets.
dt = intargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(intargs), dtype=dt)
for col in intargs.dtype.names:
targs[col] = intargs[col]
# ADM match to all secondary targets for non-custom primary files.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if 'FILEHPX' in hdr:
nside, pix = hdr['FILENSID'], hdr['FILEHPX']
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(intargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(intargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(fn, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# ADM update the secondary targets with the primary TARGETID.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM form the output primary file name and write the file.
base, ext = os.path.splitext(infile)
outfile = "{}{}{}".format(base, '-wscnd', ext)
log.info('Writing updated primary targets to {}...t={:.1f}s'.format(
outfile,
time() - start))
fitsio.write(outfile, targs, extname='TARGETS', header=hdr, clobber=True)
log.info('Done for {}...t={:.1f}s'.format(fn, time() - start))
return scxtargs
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
primtargs : :class:`~numpy.ndarray`
An array of primary targets.
scndout : :class`~numpy.ndarray`
Name of a sub-directory to which to write the information in
`desitarget.secondary.outdatamodel` with `TARGETID` and (the
highest) `PRIORITY_INIT` updated with matching primary info.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
pix : :class:`list`, optional, defaults to `None`
Limit secondary targets to (NESTED) HEALpixels that touch
pix at the supplied `nside`, as a speed-up.
nside : :class:`int`, optional, defaults to `None`
The (NESTED) HEALPixel nside to be used with `pixlist`.
Returns
-------
:class:`~numpy.ndarray`
The array of primary targets, with the `SCND_TARGET` bit
populated for matches to secondary targets
"""
# ADM add a SCND_TARGET column to the primary targets.
dt = primtargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(primtargs), dtype=dt)
for col in primtargs.dtype.names:
targs[col] = primtargs[col]
# ADM check if this is an SV or main survey file.
cols, mx, surv = main_cmx_or_sv(targs, scnd=True)
log.info('running on the {} survey...'.format(surv))
if surv != 'main':
scxdir = os.path.join(scxdir, surv)
# ADM read in non-OVERRIDE secondary targets.
scxtargs = read_files(scxdir, mx[3])
scxtargs = scxtargs[~scxtargs["OVERRIDE"]]
# ADM match primary targets to non-OVERRIDE secondary targets.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if nside is not None and pix is not None:
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(primtargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(primtargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(scndout, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
desi_mask = desimasks[0]
targs[desicols[0]][umtargs] |= desi_mask.SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# APC Secondary target bits only affect PRIORITY, NUMOBS and
# APC obsconditions for specific DESI_TARGET bits
# APC See https://github.com/desihub/desitarget/pull/530
# APC Only consider primary targets with secondary bits set
scnd_update = (targs[desicols[0]] & desi_mask['SCND_ANY']) != 0
if np.any(scnd_update):
# APC Allow changes to primaries if the DESI_TARGET bitmask has
# APC only the following bits set, in any combination.
log.info(
'Testing if secondary targets can update {} matched primaries'.
format(scnd_update.sum()))
update_from_scnd_bits = desi_mask['SCND_ANY'] | desi_mask[
'MWS_ANY'] | desi_mask['STD_BRIGHT'] | desi_mask[
'STD_FAINT'] | desi_mask['STD_WD']
scnd_update &= ((targs[desicols[0]] & ~update_from_scnd_bits) == 0)
log.info(
'Setting new priority, numobs and obsconditions from secondary for {} matched primaries'
.format(scnd_update.sum()))
# APC Primary and secondary obsconditions are or'd
scnd_obscon = set_obsconditions(targs[scnd_update], scnd=True)
targs['OBSCONDITIONS'][scnd_update] &= scnd_obscon
# APC bit of a hack here
# APC Check for _BRIGHT, _DARK split in column names
darkbright = 'NUMOBS_INIT_DARK' in targs.dtype.names
if darkbright:
ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"]
else:
ender, obscon = [""], [
"DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18"
]
# APC secondaries can increase priority and numobs
for edr, oc in zip(ender, obscon):
pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr
scnd_priority, scnd_numobs = initial_priority_numobs(
targs[scnd_update], obscon=oc, scnd=True)
targs[nc][scnd_update] = np.maximum(targs[nc][scnd_update],
scnd_numobs)
targs[pc][scnd_update] = np.maximum(targs[pc][scnd_update],
scnd_priority)
# ADM update the secondary targets with the primary information.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM the maximum priority will be used to break ties in the
# ADM unlikely event that a secondary matches two primaries.
hipri = np.maximum(targs["PRIORITY_INIT_DARK"],
targs["PRIORITY_INIT_BRIGHT"])
scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs]
# ADM write the secondary targets that have updated TARGETIDs.
ii = scxtargs["TARGETID"] != -1
nmatches = np.sum(ii)
log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format(
nmatches, scndout,
time() - start))
if nmatches > 0:
hdr = fitsio.FITSHDR()
hdr["SURVEY"] = surv
fitsio.write(scndout,
scxtargs[ii],
extname='SCND_TARG',
header=hdr,
clobber=True)
log.info('Done...t={:.1f}s'.format(time() - start))
return targs
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
primtargs : :class:`~numpy.ndarray`
An array of primary targets.
scndout : :class`~numpy.ndarray`
Name of a sub-directory to which to write the information in
`desitarget.secondary.outdatamodel` with `TARGETID` and (the
highest) `PRIORITY_INIT` updated with matching primary info.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
pix : :class:`list`, optional, defaults to `None`
Limit secondary targets to (NESTED) HEALpixels that touch
pix at the supplied `nside`, as a speed-up.
nside : :class:`int`, optional, defaults to `None`
The (NESTED) HEALPixel nside to be used with `pixlist`.
Returns
-------
:class:`~numpy.ndarray`
The array of primary targets, with the `SCND_TARGET` bit
populated for matches to secondary targets
"""
# ADM add a SCND_TARGET column to the primary targets.
dt = primtargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(primtargs), dtype=dt)
for col in primtargs.dtype.names:
targs[col] = primtargs[col]
# ADM check if this is an SV or main survey file.
cols, mx, surv = main_cmx_or_sv(targs, scnd=True)
log.info('running on the {} survey...'.format(surv))
if surv != 'main':
scxdir = os.path.join(scxdir, surv)
# ADM read in non-OVERRIDE secondary targets.
scxtargs = read_files(scxdir, mx[3])
scxtargs = scxtargs[~scxtargs["OVERRIDE"]]
# ADM match primary targets to non-OVERRIDE secondary targets.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if nside is not None and pix is not None:
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(primtargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(primtargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(scndout, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# ADM update the secondary targets with the primary information.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM the maximum priority will be used to break ties in the
# ADM unlikely event that a secondary matches two primaries.
hipri = np.maximum(targs["PRIORITY_INIT_DARK"],
targs["PRIORITY_INIT_BRIGHT"])
scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs]
# ADM write the secondary targets that have updated TARGETIDs.
ii = scxtargs["TARGETID"] != -1
nmatches = np.sum(ii)
log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format(
nmatches, scndout,
time() - start))
if nmatches > 0:
hdr = fitsio.FITSHDR()
hdr["SURVEY"] = surv
fitsio.write(scndout,
scxtargs[ii],
extname='SCND_TARG',
header=hdr,
clobber=True)
log.info('Done...t={:.1f}s'.format(time() - start))
return targs
