def match_to_tycho(objs, matchrad=1., radec=False):
"""Match objects to Tycho healpixel files.
Parameters
----------
objs : :class:`~numpy.ndarray`
Must contain at least "RA" and "DEC".
matchrad : :class:`float`, optional, defaults to 1 arcsec
The radius at which to match in arcseconds.
radec : :class:`bool`, optional, defaults to ``False``
If ``True`` then the passed `objs` is an [RA, Dec] list instead of
a rec array.
Returns
-------
:class:`~numpy.ndarray`
The matching Tycho information for each object. The returned
format is as for desitarget.tychomatch.tychodatamodel with
an extra column "TYCHO_SEP" which is the matching distance
in ARCSECONDS.
Notes
-----
- For objects with NO match in Tycho, the "TYC1", "TYC2" and
"TYCHO_SEP" columns are -1, and other columns are zero.
- Retrieves the CLOSEST match to Tycho for each passed object.
- Because this reads in HEALPixel split files, it's (far) faster
for objects that are clumped rather than widely distributed.
"""
# ADM parse whether a structure or coordinate list was passed.
if radec:
ra, dec = objs
else:
ra, dec = objs["RA"], objs["DEC"]
# ADM set up an array of Tycho information for the output.
nobjs = len(ra)
done = np.zeros(nobjs, dtype=tychodatamodel.dtype)
# ADM objects without matches should have TYC1/2/3, TYCHO_SEP of -1.
for col in "TYC1", "TYC2":
done[col] = -1
tycho_sep = np.zeros(nobjs) - 1
# ADM determine which Tycho files need to be scraped.
tychofiles = find_tycho_files([ra, dec], radec=True)
nfiles = len(tychofiles)
# ADM catch the case of no matches to Tycho.
if nfiles > 0:
# ADM loop through the Tycho files and find matches.
for ifn, fn in enumerate(tychofiles):
if ifn % 500 == 0 and ifn > 0:
log.info('{}/{} files; {:.1f} total mins elapsed'
.format(ifn, nfiles, (time()-start)/60.))
tycho = fitsio.read(fn)
idtycho, idobjs, dist = radec_match_to(
[tycho["RA"], tycho["DEC"]], [ra, dec],
sep=matchrad, radec=True, return_sep=True)
# ADM update matches whenever we have a CLOSER match.
ii = (tycho_sep[idobjs] == -1) | (tycho_sep[idobjs] > dist)
done[idobjs[ii]] = tycho[idtycho[ii]]
tycho_sep[idobjs[ii]] = dist[ii]
# ADM add the separation distances to the output array.
dt = tychodatamodel.dtype.descr + [("TYCHO_SEP", ">f4")]
output = np.zeros(nobjs, dtype=dt)
for col in tychodatamodel.dtype.names:
output[col] = done[col]
output["TYCHO_SEP"] = tycho_sep
return output
def make_bright_star_mask_in_hp(nside, pixnum, verbose=True, gaiaepoch=2015.5,
maglim=12., matchrad=1., maskepoch=2023.0):
"""Make a bright star mask in a HEALPixel using Tycho, Gaia and URAT.
Parameters
----------
nside : :class:`int`
(NESTED) HEALPixel nside.
pixnum : :class:`int`
A single HEALPixel number.
verbose : :class:`bool`
If ``True`` then log informational messages.
Returns
-------
:class:`recarray`
The bright star mask in the form of `maskdatamodel.dtype`.
Notes
-----
- Runs in a a minute or so for a typical nside=4 pixel.
- See :func:`~desitarget.brightmask.make_bright_star_mask` for
descriptions of the output mask and the other input parameters.
"""
# ADM start the clock.
t0 = time()
# ADM read in the Tycho files.
tychofns = find_tycho_files_hp(nside, pixnum, neighbors=False)
tychoobjs = []
for fn in tychofns:
tychoobjs.append(fitsio.read(fn, ext='TYCHOHPX'))
tychoobjs = np.concatenate(tychoobjs)
# ADM create the Tycho reference magnitude, which is VT then HP
# ADM then BT in order of preference.
tychomag = tychoobjs["MAG_VT"].copy()
tychomag[tychomag == 0] = tychoobjs["MAG_HP"][tychomag == 0]
tychomag[tychomag == 0] = tychoobjs["MAG_BT"][tychomag == 0]
# ADM discard any Tycho objects below the input magnitude limit
# ADM and outside of the HEALPixels of interest.
theta, phi = np.radians(90-tychoobjs["DEC"]), np.radians(tychoobjs["RA"])
tychohpx = hp.ang2pix(nside, theta, phi, nest=True)
ii = (tychohpx == pixnum) & (tychomag < maglim)
tychomag, tychoobjs = tychomag[ii], tychoobjs[ii]
if verbose:
log.info('Read {} (mag < {}) Tycho objects (pix={})...t={:.1f} mins'.
format(np.sum(ii), maglim, pixnum, (time()-t0)/60))
# ADM read in the associated Gaia files. Also grab
# ADM neighboring pixels to prevent edge effects.
gaiafns = find_gaia_files(tychoobjs, neighbors=True)
gaiaobjs = []
cols = 'SOURCE_ID', 'RA', 'DEC', 'PHOT_G_MEAN_MAG', 'PMRA', 'PMDEC'
for fn in gaiafns:
if os.path.exists(fn):
gaiaobjs.append(fitsio.read(fn, ext='GAIAHPX', columns=cols))
gaiaobjs = np.concatenate(gaiaobjs)
gaiaobjs = rfn.rename_fields(gaiaobjs, {"SOURCE_ID": "REF_ID"})
# ADM limit Gaia objects to 3 magnitudes fainter than the passed
# ADM limit. This leaves some (!) leeway when matching to Tycho.
gaiaobjs = gaiaobjs[gaiaobjs['PHOT_G_MEAN_MAG'] < maglim + 3]
if verbose:
log.info('Read {} (G < {}) Gaia sources (pix={})...t={:.1f} mins'.format(
len(gaiaobjs), maglim+3, pixnum, (time()-t0)/60))
# ADM substitute URAT where Gaia proper motions don't exist.
ii = ((np.isnan(gaiaobjs["PMRA"]) | (gaiaobjs["PMRA"] == 0)) &
(np.isnan(gaiaobjs["PMDEC"]) | (gaiaobjs["PMDEC"] == 0)))
if verbose:
log.info('Add URAT for {} Gaia objs with no PMs (pix={})...t={:.1f} mins'
.format(np.sum(ii), pixnum, (time()-t0)/60))
urat = add_urat_pms(gaiaobjs[ii], numproc=1)
if verbose:
log.info('Found an additional {} URAT objects (pix={})...t={:.1f} mins'
.format(np.sum(urat["URAT_ID"] != -1), pixnum, (time()-t0)/60))
for col in "PMRA", "PMDEC":
gaiaobjs[col][ii] = urat[col]
# ADM need to track the URATID to track which objects have
# ADM substituted proper motions.
uratid = np.zeros_like(gaiaobjs["REF_ID"])-1
uratid[ii] = urat["URAT_ID"]
# ADM match to remove Tycho objects already in Gaia. Prefer the more
# ADM accurate Gaia proper motions. Note, however, that Tycho epochs
# ADM can differ from the mean (1991.5) by as as much as 0.86 years,
# ADM so a star with a proper motion as large as Barnard's Star
# ADM (10.3 arcsec) can be off by a significant margin (~10").
margin = 10.
ra, dec = rewind_coords(gaiaobjs["RA"], gaiaobjs["DEC"],
gaiaobjs["PMRA"], gaiaobjs["PMDEC"],
epochnow=gaiaepoch)
# ADM match Gaia to Tycho with a suitable margin.
if verbose:
log.info('Match Gaia to Tycho with margin={}" (pix={})...t={:.1f} mins'
.format(margin, pixnum, (time()-t0)/60))
igaia, itycho = radec_match_to([ra, dec],
[tychoobjs["RA"], tychoobjs["DEC"]],
sep=margin, radec=True)
if verbose:
log.info('{} matches. Refining at 1" (pix={})...t={:.1f} mins'.format(
len(itycho), pixnum, (time()-t0)/60))
# ADM match Gaia to Tycho at the more exact reference epoch.
epoch_ra = tychoobjs[itycho]["EPOCH_RA"]
epoch_dec = tychoobjs[itycho]["EPOCH_DEC"]
# ADM some of the Tycho epochs aren't populated.
epoch_ra[epoch_ra == 0], epoch_dec[epoch_dec == 0] = 1991.5, 1991.5
ra, dec = rewind_coords(gaiaobjs["RA"][igaia], gaiaobjs["DEC"][igaia],
gaiaobjs["PMRA"][igaia], gaiaobjs["PMDEC"][igaia],
epochnow=gaiaepoch,
epochpast=epoch_ra, epochpastdec=epoch_dec)
# ADM catch the corner case where there are no initial matches.
if ra.size > 0:
_, refined = radec_match_to([ra, dec], [tychoobjs["RA"][itycho],
tychoobjs["DEC"][itycho]], radec=True)
else:
refined = np.array([], dtype='int')
# ADM retain Tycho objects that DON'T match Gaia.
keep = np.ones(len(tychoobjs), dtype='bool')
keep[itycho[refined]] = False
tychokeep, tychomag = tychoobjs[keep], tychomag[keep]
if verbose:
log.info('Kept {} Tychos with no Gaia match (pix={})...t={:.1f} mins'
.format(len(tychokeep), pixnum, (time()-t0)/60))
# ADM now we're done matching to Gaia, limit Gaia to the passed
# ADM magnitude limit and to the HEALPixel boundary of interest.
theta, phi = np.radians(90-gaiaobjs["DEC"]), np.radians(gaiaobjs["RA"])
gaiahpx = hp.ang2pix(nside, theta, phi, nest=True)
ii = (gaiahpx == pixnum) & (gaiaobjs['PHOT_G_MEAN_MAG'] < maglim)
gaiakeep, uratid = gaiaobjs[ii], uratid[ii]
if verbose:
log.info('Mask also comprises {} Gaia sources (pix={})...t={:.1f} mins'
.format(len(gaiakeep), pixnum, (time()-t0)/60))
# ADM move the coordinates forwards to the input mask epoch.
epoch_ra, epoch_dec = tychokeep["EPOCH_RA"], tychokeep["EPOCH_DEC"]
# ADM some of the Tycho epochs aren't populated.
epoch_ra[epoch_ra == 0], epoch_dec[epoch_dec == 0] = 1991.5, 1991.5
ra, dec = rewind_coords(
tychokeep["RA"], tychokeep["DEC"], tychokeep["PM_RA"], tychokeep["PM_DEC"],
epochnow=epoch_ra, epochnowdec=epoch_dec, epochpast=maskepoch)
tychokeep["RA"], tychokeep["DEC"] = ra, dec
ra, dec = rewind_coords(
gaiakeep["RA"], gaiakeep["DEC"], gaiakeep["PMRA"], gaiakeep["PMDEC"],
epochnow=gaiaepoch, epochpast=maskepoch)
gaiakeep["RA"], gaiakeep["DEC"] = ra, dec
# ADM finally, format according to the mask data model...
gaiamask = np.zeros(len(gaiakeep), dtype=maskdatamodel.dtype)
tychomask = np.zeros(len(tychokeep), dtype=maskdatamodel.dtype)
for col in "RA", "DEC":
gaiamask[col] = gaiakeep[col]
gaiamask["PM"+col] = gaiakeep["PM"+col]
tychomask[col] = tychokeep[col]
tychomask["PM"+col] = tychokeep["PM_"+col]
gaiamask["REF_ID"] = gaiakeep["REF_ID"]
# ADM take care to rigorously convert to int64 for Tycho.
tychomask["REF_ID"] = tychokeep["TYC1"].astype('int64')*int(1e6) + \
tychokeep["TYC2"].astype('int64')*10 + tychokeep["TYC3"]
gaiamask["REF_CAT"], tychomask["REF_CAT"] = 'G2', 'T2'
gaiamask["REF_MAG"] = gaiakeep['PHOT_G_MEAN_MAG']
tychomask["REF_MAG"] = tychomag
gaiamask["URAT_ID"], tychomask["URAT_ID"] = uratid, -1
gaiamask["TYPE"], tychomask["TYPE"] = 'PSF', 'PSF'
mask = np.concatenate([gaiamask, tychomask])
# ADM ...and add the mask radii.
mask["IN_RADIUS"], mask["NEAR_RADIUS"] = radii(mask["REF_MAG"])
if verbose:
log.info("Done making mask...(pix={})...t={:.1f} mins".format(
pixnum, (time()-t0)/60.))
return mask
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
primtargs : :class:`~numpy.ndarray`
An array of primary targets.
scndout : :class`~numpy.ndarray`
Name of a sub-directory to which to write the information in
`desitarget.secondary.outdatamodel` with `TARGETID` and (the
highest) `PRIORITY_INIT` updated with matching primary info.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
pix : :class:`list`, optional, defaults to `None`
Limit secondary targets to (NESTED) HEALpixels that touch
pix at the supplied `nside`, as a speed-up.
nside : :class:`int`, optional, defaults to `None`
The (NESTED) HEALPixel nside to be used with `pixlist`.
Returns
-------
:class:`~numpy.ndarray`
The array of primary targets, with the `SCND_TARGET` bit
populated for matches to secondary targets
"""
# ADM add a SCND_TARGET column to the primary targets.
dt = primtargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(primtargs), dtype=dt)
for col in primtargs.dtype.names:
targs[col] = primtargs[col]
# ADM check if this is an SV or main survey file.
cols, mx, surv = main_cmx_or_sv(targs, scnd=True)
log.info('running on the {} survey...'.format(surv))
if surv != 'main':
scxdir = os.path.join(scxdir, surv)
# ADM read in non-OVERRIDE secondary targets.
scxtargs = read_files(scxdir, mx[3])
scxtargs = scxtargs[~scxtargs["OVERRIDE"]]
# ADM match primary targets to non-OVERRIDE secondary targets.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if nside is not None and pix is not None:
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(primtargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(primtargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(scndout, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
desi_mask = desimasks[0]
targs[desicols[0]][umtargs] |= desi_mask.SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# APC Secondary target bits only affect PRIORITY, NUMOBS and
# APC obsconditions for specific DESI_TARGET bits
# APC See https://github.com/desihub/desitarget/pull/530
# APC Only consider primary targets with secondary bits set
scnd_update = (targs[desicols[0]] & desi_mask['SCND_ANY']) != 0
if np.any(scnd_update):
# APC Allow changes to primaries if the DESI_TARGET bitmask has
# APC only the following bits set, in any combination.
log.info(
'Testing if secondary targets can update {} matched primaries'.
format(scnd_update.sum()))
update_from_scnd_bits = desi_mask['SCND_ANY'] | desi_mask[
'MWS_ANY'] | desi_mask['STD_BRIGHT'] | desi_mask[
'STD_FAINT'] | desi_mask['STD_WD']
scnd_update &= ((targs[desicols[0]] & ~update_from_scnd_bits) == 0)
log.info(
'Setting new priority, numobs and obsconditions from secondary for {} matched primaries'
.format(scnd_update.sum()))
# APC Primary and secondary obsconditions are or'd
scnd_obscon = set_obsconditions(targs[scnd_update], scnd=True)
targs['OBSCONDITIONS'][scnd_update] &= scnd_obscon
# APC bit of a hack here
# APC Check for _BRIGHT, _DARK split in column names
darkbright = 'NUMOBS_INIT_DARK' in targs.dtype.names
if darkbright:
ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"]
else:
ender, obscon = [""], [
"DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18"
]
# APC secondaries can increase priority and numobs
for edr, oc in zip(ender, obscon):
pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr
scnd_priority, scnd_numobs = initial_priority_numobs(
targs[scnd_update], obscon=oc, scnd=True)
targs[nc][scnd_update] = np.maximum(targs[nc][scnd_update],
scnd_numobs)
targs[pc][scnd_update] = np.maximum(targs[pc][scnd_update],
scnd_priority)
# ADM update the secondary targets with the primary information.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM the maximum priority will be used to break ties in the
# ADM unlikely event that a secondary matches two primaries.
hipri = np.maximum(targs["PRIORITY_INIT_DARK"],
targs["PRIORITY_INIT_BRIGHT"])
scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs]
# ADM write the secondary targets that have updated TARGETIDs.
ii = scxtargs["TARGETID"] != -1
nmatches = np.sum(ii)
log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format(
nmatches, scndout,
time() - start))
if nmatches > 0:
hdr = fitsio.FITSHDR()
hdr["SURVEY"] = surv
fitsio.write(scndout,
scxtargs[ii],
extname='SCND_TARG',
header=hdr,
clobber=True)
log.info('Done...t={:.1f}s'.format(time() - start))
return targs
def match_secondary(infile, scxtargs, sep=1., scxdir=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
infile : :class:`str`
The full path to a file containing primary targets.
scxtargs : :class:`~numpy.ndarray`
An array of secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets. If passed,
this is written to the output primary file header as `SCNDDIR`.
Returns
-------
:class:`~numpy.ndarray`
The array of secondary targets, with the `TARGETID` bit
updated with any matching primary targets from `infile`.
Notes
-----
- The primary target `infiles` are written back to their
original path with `.fits` changed to `-wscnd.fits` and the
`SCND_TARGET` bit populated for matching targets.
"""
# ADM just the file name for logging.
fn = os.path.basename(infile)
# ADM read in the primary targets.
log.info('Reading primary targets file {}...t={:.1f}s'.format(
infile,
time() - start))
intargs, hdr = fitsio.read(infile, extension="TARGETS", header=True)
# ADM fail if file's already been matched to secondary targets.
if "SCNDDIR" in hdr:
msg = "{} already matched to secondary targets".format(fn) \
+ " (did you mean to remove {}?)!!!".format(fn)
log.critical(msg)
raise ValueError(msg)
# ADM add the SCNDDIR to the primary targets file header.
hdr["SCNDDIR"] = scxdir
# ADM add a SCND_TARGET column to the primary targets.
dt = intargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(intargs), dtype=dt)
for col in intargs.dtype.names:
targs[col] = intargs[col]
# ADM match to all secondary targets for non-custom primary files.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if 'FILEHPX' in hdr:
nside, pix = hdr['FILENSID'], hdr['FILEHPX']
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(intargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(intargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(fn, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# ADM update the secondary targets with the primary TARGETID.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM form the output primary file name and write the file.
base, ext = os.path.splitext(infile)
outfile = "{}{}{}".format(base, '-wscnd', ext)
log.info('Writing updated primary targets to {}...t={:.1f}s'.format(
outfile,
time() - start))
fitsio.write(outfile, targs, extname='TARGETS', header=hdr, clobber=True)
log.info('Done for {}...t={:.1f}s'.format(fn, time() - start))
return scxtargs
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None):
"""Match secondary targets to primary targets and update bits.
Parameters
----------
primtargs : :class:`~numpy.ndarray`
An array of primary targets.
scndout : :class`~numpy.ndarray`
Name of a sub-directory to which to write the information in
`desitarget.secondary.outdatamodel` with `TARGETID` and (the
highest) `PRIORITY_INIT` updated with matching primary info.
scxdir : :class:`str`, optional, defaults to `None`
Name of the directory that hosts secondary targets.
sep : :class:`float`, defaults to 1 arcsecond
The separation at which to match in ARCSECONDS.
pix : :class:`list`, optional, defaults to `None`
Limit secondary targets to (NESTED) HEALpixels that touch
pix at the supplied `nside`, as a speed-up.
nside : :class:`int`, optional, defaults to `None`
The (NESTED) HEALPixel nside to be used with `pixlist`.
Returns
-------
:class:`~numpy.ndarray`
The array of primary targets, with the `SCND_TARGET` bit
populated for matches to secondary targets
"""
# ADM add a SCND_TARGET column to the primary targets.
dt = primtargs.dtype.descr
dt.append(('SCND_TARGET', '>i8'))
targs = np.zeros(len(primtargs), dtype=dt)
for col in primtargs.dtype.names:
targs[col] = primtargs[col]
# ADM check if this is an SV or main survey file.
cols, mx, surv = main_cmx_or_sv(targs, scnd=True)
log.info('running on the {} survey...'.format(surv))
if surv != 'main':
scxdir = os.path.join(scxdir, surv)
# ADM read in non-OVERRIDE secondary targets.
scxtargs = read_files(scxdir, mx[3])
scxtargs = scxtargs[~scxtargs["OVERRIDE"]]
# ADM match primary targets to non-OVERRIDE secondary targets.
inhp = np.ones(len(scxtargs), dtype="?")
# ADM as a speed-up, save memory by limiting the secondary targets
# ADM to just HEALPixels that could touch the primary targets.
if nside is not None and pix is not None:
# ADM remember to grab adjacent pixels in case of edge effects.
allpix = add_hp_neighbors(nside, pix)
inhp = is_in_hp(scxtargs, nside, allpix)
# ADM it's unlikely that the matching separation is comparable
# ADM to the HEALPixel resolution, but guard against that anyway.
halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
if sep > halfpix:
msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
sep, halfpix)
log.critical(msg)
raise ValueError(msg)
# ADM warn the user if the secondary and primary samples are "large".
big = 500000
if np.sum(inhp) > big and len(primtargs) > big:
log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
np.sum(inhp), len(primtargs)))
log.warning('The code may run slowly')
# ADM for each secondary target, determine if there is a match
# ADM with a primary target. Note that sense is important, here
# ADM (the primary targets must be passed first).
log.info(
'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
format(scndout, sep,
time() - start))
mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
# ADM recast the indices to the full set of secondary targets,
# ADM instead of just those that were in the relevant HEALPixels.
mscx = np.where(inhp)[0][mscx]
# ADM loop through the matches and update the SCND_TARGET
# ADM column in the primary target list. The np.unique is a
# ADM speed-up to assign singular matches first.
umtargs, inv, cnt = np.unique(mtargs,
return_inverse=True,
return_counts=True)
# ADM number of times each primary target was matched, ordered
# ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
nmtargs = cnt[inv]
# ADM assign anything with nmtargs = 1 directly.
singular = nmtargs == 1
targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
mscx[singular]]
# ADM loop through things with nmtargs > 1 and combine the bits.
for i in range(len((mtargs[~singular]))):
targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
mscx[~singular][i]]
# ADM also assign the SCND_ANY bit to the primary targets.
desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY
# ADM rename the SCND_TARGET column, in case this is an SV file.
targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})
# ADM update the secondary targets with the primary information.
scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
# ADM the maximum priority will be used to break ties in the
# ADM unlikely event that a secondary matches two primaries.
hipri = np.maximum(targs["PRIORITY_INIT_DARK"],
targs["PRIORITY_INIT_BRIGHT"])
scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs]
# ADM write the secondary targets that have updated TARGETIDs.
ii = scxtargs["TARGETID"] != -1
nmatches = np.sum(ii)
log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format(
nmatches, scndout,
time() - start))
if nmatches > 0:
hdr = fitsio.FITSHDR()
hdr["SURVEY"] = surv
fitsio.write(scndout,
scxtargs[ii],
extname='SCND_TARG',
header=hdr,
clobber=True)
log.info('Done...t={:.1f}s'.format(time() - start))
return targs