def match_to_tycho(objs, matchrad=1., radec=False): """Match objects to Tycho healpixel files. Parameters ---------- objs : :class:`~numpy.ndarray` Must contain at least "RA" and "DEC". matchrad : :class:`float`, optional, defaults to 1 arcsec The radius at which to match in arcseconds. radec : :class:`bool`, optional, defaults to ``False`` If ``True`` then the passed `objs` is an [RA, Dec] list instead of a rec array. Returns ------- :class:`~numpy.ndarray` The matching Tycho information for each object. The returned format is as for desitarget.tychomatch.tychodatamodel with an extra column "TYCHO_SEP" which is the matching distance in ARCSECONDS. Notes ----- - For objects with NO match in Tycho, the "TYC1", "TYC2" and "TYCHO_SEP" columns are -1, and other columns are zero. - Retrieves the CLOSEST match to Tycho for each passed object. - Because this reads in HEALPixel split files, it's (far) faster for objects that are clumped rather than widely distributed. """ # ADM parse whether a structure or coordinate list was passed. if radec: ra, dec = objs else: ra, dec = objs["RA"], objs["DEC"] # ADM set up an array of Tycho information for the output. nobjs = len(ra) done = np.zeros(nobjs, dtype=tychodatamodel.dtype) # ADM objects without matches should have TYC1/2/3, TYCHO_SEP of -1. for col in "TYC1", "TYC2": done[col] = -1 tycho_sep = np.zeros(nobjs) - 1 # ADM determine which Tycho files need to be scraped. tychofiles = find_tycho_files([ra, dec], radec=True) nfiles = len(tychofiles) # ADM catch the case of no matches to Tycho. if nfiles > 0: # ADM loop through the Tycho files and find matches. for ifn, fn in enumerate(tychofiles): if ifn % 500 == 0 and ifn > 0: log.info('{}/{} files; {:.1f} total mins elapsed' .format(ifn, nfiles, (time()-start)/60.)) tycho = fitsio.read(fn) idtycho, idobjs, dist = radec_match_to( [tycho["RA"], tycho["DEC"]], [ra, dec], sep=matchrad, radec=True, return_sep=True) # ADM update matches whenever we have a CLOSER match. ii = (tycho_sep[idobjs] == -1) | (tycho_sep[idobjs] > dist) done[idobjs[ii]] = tycho[idtycho[ii]] tycho_sep[idobjs[ii]] = dist[ii] # ADM add the separation distances to the output array. dt = tychodatamodel.dtype.descr + [("TYCHO_SEP", ">f4")] output = np.zeros(nobjs, dtype=dt) for col in tychodatamodel.dtype.names: output[col] = done[col] output["TYCHO_SEP"] = tycho_sep return output
def make_bright_star_mask_in_hp(nside, pixnum, verbose=True, gaiaepoch=2015.5, maglim=12., matchrad=1., maskepoch=2023.0): """Make a bright star mask in a HEALPixel using Tycho, Gaia and URAT. Parameters ---------- nside : :class:`int` (NESTED) HEALPixel nside. pixnum : :class:`int` A single HEALPixel number. verbose : :class:`bool` If ``True`` then log informational messages. Returns ------- :class:`recarray` The bright star mask in the form of `maskdatamodel.dtype`. Notes ----- - Runs in a a minute or so for a typical nside=4 pixel. - See :func:`~desitarget.brightmask.make_bright_star_mask` for descriptions of the output mask and the other input parameters. """ # ADM start the clock. t0 = time() # ADM read in the Tycho files. tychofns = find_tycho_files_hp(nside, pixnum, neighbors=False) tychoobjs = [] for fn in tychofns: tychoobjs.append(fitsio.read(fn, ext='TYCHOHPX')) tychoobjs = np.concatenate(tychoobjs) # ADM create the Tycho reference magnitude, which is VT then HP # ADM then BT in order of preference. tychomag = tychoobjs["MAG_VT"].copy() tychomag[tychomag == 0] = tychoobjs["MAG_HP"][tychomag == 0] tychomag[tychomag == 0] = tychoobjs["MAG_BT"][tychomag == 0] # ADM discard any Tycho objects below the input magnitude limit # ADM and outside of the HEALPixels of interest. theta, phi = np.radians(90-tychoobjs["DEC"]), np.radians(tychoobjs["RA"]) tychohpx = hp.ang2pix(nside, theta, phi, nest=True) ii = (tychohpx == pixnum) & (tychomag < maglim) tychomag, tychoobjs = tychomag[ii], tychoobjs[ii] if verbose: log.info('Read {} (mag < {}) Tycho objects (pix={})...t={:.1f} mins'. format(np.sum(ii), maglim, pixnum, (time()-t0)/60)) # ADM read in the associated Gaia files. Also grab # ADM neighboring pixels to prevent edge effects. gaiafns = find_gaia_files(tychoobjs, neighbors=True) gaiaobjs = [] cols = 'SOURCE_ID', 'RA', 'DEC', 'PHOT_G_MEAN_MAG', 'PMRA', 'PMDEC' for fn in gaiafns: if os.path.exists(fn): gaiaobjs.append(fitsio.read(fn, ext='GAIAHPX', columns=cols)) gaiaobjs = np.concatenate(gaiaobjs) gaiaobjs = rfn.rename_fields(gaiaobjs, {"SOURCE_ID": "REF_ID"}) # ADM limit Gaia objects to 3 magnitudes fainter than the passed # ADM limit. This leaves some (!) leeway when matching to Tycho. gaiaobjs = gaiaobjs[gaiaobjs['PHOT_G_MEAN_MAG'] < maglim + 3] if verbose: log.info('Read {} (G < {}) Gaia sources (pix={})...t={:.1f} mins'.format( len(gaiaobjs), maglim+3, pixnum, (time()-t0)/60)) # ADM substitute URAT where Gaia proper motions don't exist. ii = ((np.isnan(gaiaobjs["PMRA"]) | (gaiaobjs["PMRA"] == 0)) & (np.isnan(gaiaobjs["PMDEC"]) | (gaiaobjs["PMDEC"] == 0))) if verbose: log.info('Add URAT for {} Gaia objs with no PMs (pix={})...t={:.1f} mins' .format(np.sum(ii), pixnum, (time()-t0)/60)) urat = add_urat_pms(gaiaobjs[ii], numproc=1) if verbose: log.info('Found an additional {} URAT objects (pix={})...t={:.1f} mins' .format(np.sum(urat["URAT_ID"] != -1), pixnum, (time()-t0)/60)) for col in "PMRA", "PMDEC": gaiaobjs[col][ii] = urat[col] # ADM need to track the URATID to track which objects have # ADM substituted proper motions. uratid = np.zeros_like(gaiaobjs["REF_ID"])-1 uratid[ii] = urat["URAT_ID"] # ADM match to remove Tycho objects already in Gaia. Prefer the more # ADM accurate Gaia proper motions. Note, however, that Tycho epochs # ADM can differ from the mean (1991.5) by as as much as 0.86 years, # ADM so a star with a proper motion as large as Barnard's Star # ADM (10.3 arcsec) can be off by a significant margin (~10"). margin = 10. ra, dec = rewind_coords(gaiaobjs["RA"], gaiaobjs["DEC"], gaiaobjs["PMRA"], gaiaobjs["PMDEC"], epochnow=gaiaepoch) # ADM match Gaia to Tycho with a suitable margin. if verbose: log.info('Match Gaia to Tycho with margin={}" (pix={})...t={:.1f} mins' .format(margin, pixnum, (time()-t0)/60)) igaia, itycho = radec_match_to([ra, dec], [tychoobjs["RA"], tychoobjs["DEC"]], sep=margin, radec=True) if verbose: log.info('{} matches. Refining at 1" (pix={})...t={:.1f} mins'.format( len(itycho), pixnum, (time()-t0)/60)) # ADM match Gaia to Tycho at the more exact reference epoch. epoch_ra = tychoobjs[itycho]["EPOCH_RA"] epoch_dec = tychoobjs[itycho]["EPOCH_DEC"] # ADM some of the Tycho epochs aren't populated. epoch_ra[epoch_ra == 0], epoch_dec[epoch_dec == 0] = 1991.5, 1991.5 ra, dec = rewind_coords(gaiaobjs["RA"][igaia], gaiaobjs["DEC"][igaia], gaiaobjs["PMRA"][igaia], gaiaobjs["PMDEC"][igaia], epochnow=gaiaepoch, epochpast=epoch_ra, epochpastdec=epoch_dec) # ADM catch the corner case where there are no initial matches. if ra.size > 0: _, refined = radec_match_to([ra, dec], [tychoobjs["RA"][itycho], tychoobjs["DEC"][itycho]], radec=True) else: refined = np.array([], dtype='int') # ADM retain Tycho objects that DON'T match Gaia. keep = np.ones(len(tychoobjs), dtype='bool') keep[itycho[refined]] = False tychokeep, tychomag = tychoobjs[keep], tychomag[keep] if verbose: log.info('Kept {} Tychos with no Gaia match (pix={})...t={:.1f} mins' .format(len(tychokeep), pixnum, (time()-t0)/60)) # ADM now we're done matching to Gaia, limit Gaia to the passed # ADM magnitude limit and to the HEALPixel boundary of interest. theta, phi = np.radians(90-gaiaobjs["DEC"]), np.radians(gaiaobjs["RA"]) gaiahpx = hp.ang2pix(nside, theta, phi, nest=True) ii = (gaiahpx == pixnum) & (gaiaobjs['PHOT_G_MEAN_MAG'] < maglim) gaiakeep, uratid = gaiaobjs[ii], uratid[ii] if verbose: log.info('Mask also comprises {} Gaia sources (pix={})...t={:.1f} mins' .format(len(gaiakeep), pixnum, (time()-t0)/60)) # ADM move the coordinates forwards to the input mask epoch. epoch_ra, epoch_dec = tychokeep["EPOCH_RA"], tychokeep["EPOCH_DEC"] # ADM some of the Tycho epochs aren't populated. epoch_ra[epoch_ra == 0], epoch_dec[epoch_dec == 0] = 1991.5, 1991.5 ra, dec = rewind_coords( tychokeep["RA"], tychokeep["DEC"], tychokeep["PM_RA"], tychokeep["PM_DEC"], epochnow=epoch_ra, epochnowdec=epoch_dec, epochpast=maskepoch) tychokeep["RA"], tychokeep["DEC"] = ra, dec ra, dec = rewind_coords( gaiakeep["RA"], gaiakeep["DEC"], gaiakeep["PMRA"], gaiakeep["PMDEC"], epochnow=gaiaepoch, epochpast=maskepoch) gaiakeep["RA"], gaiakeep["DEC"] = ra, dec # ADM finally, format according to the mask data model... gaiamask = np.zeros(len(gaiakeep), dtype=maskdatamodel.dtype) tychomask = np.zeros(len(tychokeep), dtype=maskdatamodel.dtype) for col in "RA", "DEC": gaiamask[col] = gaiakeep[col] gaiamask["PM"+col] = gaiakeep["PM"+col] tychomask[col] = tychokeep[col] tychomask["PM"+col] = tychokeep["PM_"+col] gaiamask["REF_ID"] = gaiakeep["REF_ID"] # ADM take care to rigorously convert to int64 for Tycho. tychomask["REF_ID"] = tychokeep["TYC1"].astype('int64')*int(1e6) + \ tychokeep["TYC2"].astype('int64')*10 + tychokeep["TYC3"] gaiamask["REF_CAT"], tychomask["REF_CAT"] = 'G2', 'T2' gaiamask["REF_MAG"] = gaiakeep['PHOT_G_MEAN_MAG'] tychomask["REF_MAG"] = tychomag gaiamask["URAT_ID"], tychomask["URAT_ID"] = uratid, -1 gaiamask["TYPE"], tychomask["TYPE"] = 'PSF', 'PSF' mask = np.concatenate([gaiamask, tychomask]) # ADM ...and add the mask radii. mask["IN_RADIUS"], mask["NEAR_RADIUS"] = radii(mask["REF_MAG"]) if verbose: log.info("Done making mask...(pix={})...t={:.1f} mins".format( pixnum, (time()-t0)/60.)) return mask
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None): """Match secondary targets to primary targets and update bits. Parameters ---------- primtargs : :class:`~numpy.ndarray` An array of primary targets. scndout : :class`~numpy.ndarray` Name of a sub-directory to which to write the information in `desitarget.secondary.outdatamodel` with `TARGETID` and (the highest) `PRIORITY_INIT` updated with matching primary info. scxdir : :class:`str`, optional, defaults to `None` Name of the directory that hosts secondary targets. sep : :class:`float`, defaults to 1 arcsecond The separation at which to match in ARCSECONDS. pix : :class:`list`, optional, defaults to `None` Limit secondary targets to (NESTED) HEALpixels that touch pix at the supplied `nside`, as a speed-up. nside : :class:`int`, optional, defaults to `None` The (NESTED) HEALPixel nside to be used with `pixlist`. Returns ------- :class:`~numpy.ndarray` The array of primary targets, with the `SCND_TARGET` bit populated for matches to secondary targets """ # ADM add a SCND_TARGET column to the primary targets. dt = primtargs.dtype.descr dt.append(('SCND_TARGET', '>i8')) targs = np.zeros(len(primtargs), dtype=dt) for col in primtargs.dtype.names: targs[col] = primtargs[col] # ADM check if this is an SV or main survey file. cols, mx, surv = main_cmx_or_sv(targs, scnd=True) log.info('running on the {} survey...'.format(surv)) if surv != 'main': scxdir = os.path.join(scxdir, surv) # ADM read in non-OVERRIDE secondary targets. scxtargs = read_files(scxdir, mx[3]) scxtargs = scxtargs[~scxtargs["OVERRIDE"]] # ADM match primary targets to non-OVERRIDE secondary targets. inhp = np.ones(len(scxtargs), dtype="?") # ADM as a speed-up, save memory by limiting the secondary targets # ADM to just HEALPixels that could touch the primary targets. if nside is not None and pix is not None: # ADM remember to grab adjacent pixels in case of edge effects. allpix = add_hp_neighbors(nside, pix) inhp = is_in_hp(scxtargs, nside, allpix) # ADM it's unlikely that the matching separation is comparable # ADM to the HEALPixel resolution, but guard against that anyway. halfpix = np.degrees(hp.max_pixrad(nside)) * 3600. if sep > halfpix: msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format( sep, halfpix) log.critical(msg) raise ValueError(msg) # ADM warn the user if the secondary and primary samples are "large". big = 500000 if np.sum(inhp) > big and len(primtargs) > big: log.warning('Large secondary (N={}) and primary (N={}) samples'.format( np.sum(inhp), len(primtargs))) log.warning('The code may run slowly') # ADM for each secondary target, determine if there is a match # ADM with a primary target. Note that sense is important, here # ADM (the primary targets must be passed first). log.info( 'Matching primary and secondary targets for {} at {}"...t={:.1f}s'. format(scndout, sep, time() - start)) mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep) # ADM recast the indices to the full set of secondary targets, # ADM instead of just those that were in the relevant HEALPixels. mscx = np.where(inhp)[0][mscx] # ADM loop through the matches and update the SCND_TARGET # ADM column in the primary target list. The np.unique is a # ADM speed-up to assign singular matches first. umtargs, inv, cnt = np.unique(mtargs, return_inverse=True, return_counts=True) # ADM number of times each primary target was matched, ordered # ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs. nmtargs = cnt[inv] # ADM assign anything with nmtargs = 1 directly. singular = nmtargs == 1 targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][ mscx[singular]] # ADM loop through things with nmtargs > 1 and combine the bits. for i in range(len((mtargs[~singular]))): targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][ mscx[~singular][i]] # ADM also assign the SCND_ANY bit to the primary targets. desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True) desi_mask = desimasks[0] targs[desicols[0]][umtargs] |= desi_mask.SCND_ANY # ADM rename the SCND_TARGET column, in case this is an SV file. targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]}) # APC Secondary target bits only affect PRIORITY, NUMOBS and # APC obsconditions for specific DESI_TARGET bits # APC See https://github.com/desihub/desitarget/pull/530 # APC Only consider primary targets with secondary bits set scnd_update = (targs[desicols[0]] & desi_mask['SCND_ANY']) != 0 if np.any(scnd_update): # APC Allow changes to primaries if the DESI_TARGET bitmask has # APC only the following bits set, in any combination. log.info( 'Testing if secondary targets can update {} matched primaries'. format(scnd_update.sum())) update_from_scnd_bits = desi_mask['SCND_ANY'] | desi_mask[ 'MWS_ANY'] | desi_mask['STD_BRIGHT'] | desi_mask[ 'STD_FAINT'] | desi_mask['STD_WD'] scnd_update &= ((targs[desicols[0]] & ~update_from_scnd_bits) == 0) log.info( 'Setting new priority, numobs and obsconditions from secondary for {} matched primaries' .format(scnd_update.sum())) # APC Primary and secondary obsconditions are or'd scnd_obscon = set_obsconditions(targs[scnd_update], scnd=True) targs['OBSCONDITIONS'][scnd_update] &= scnd_obscon # APC bit of a hack here # APC Check for _BRIGHT, _DARK split in column names darkbright = 'NUMOBS_INIT_DARK' in targs.dtype.names if darkbright: ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"] else: ender, obscon = [""], [ "DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18" ] # APC secondaries can increase priority and numobs for edr, oc in zip(ender, obscon): pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr scnd_priority, scnd_numobs = initial_priority_numobs( targs[scnd_update], obscon=oc, scnd=True) targs[nc][scnd_update] = np.maximum(targs[nc][scnd_update], scnd_numobs) targs[pc][scnd_update] = np.maximum(targs[pc][scnd_update], scnd_priority) # ADM update the secondary targets with the primary information. scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs] # ADM the maximum priority will be used to break ties in the # ADM unlikely event that a secondary matches two primaries. hipri = np.maximum(targs["PRIORITY_INIT_DARK"], targs["PRIORITY_INIT_BRIGHT"]) scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs] # ADM write the secondary targets that have updated TARGETIDs. ii = scxtargs["TARGETID"] != -1 nmatches = np.sum(ii) log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format( nmatches, scndout, time() - start)) if nmatches > 0: hdr = fitsio.FITSHDR() hdr["SURVEY"] = surv fitsio.write(scndout, scxtargs[ii], extname='SCND_TARG', header=hdr, clobber=True) log.info('Done...t={:.1f}s'.format(time() - start)) return targs
def match_secondary(infile, scxtargs, sep=1., scxdir=None): """Match secondary targets to primary targets and update bits. Parameters ---------- infile : :class:`str` The full path to a file containing primary targets. scxtargs : :class:`~numpy.ndarray` An array of secondary targets. sep : :class:`float`, defaults to 1 arcsecond The separation at which to match in ARCSECONDS. scxdir : :class:`str`, optional, defaults to `None` Name of the directory that hosts secondary targets. If passed, this is written to the output primary file header as `SCNDDIR`. Returns ------- :class:`~numpy.ndarray` The array of secondary targets, with the `TARGETID` bit updated with any matching primary targets from `infile`. Notes ----- - The primary target `infiles` are written back to their original path with `.fits` changed to `-wscnd.fits` and the `SCND_TARGET` bit populated for matching targets. """ # ADM just the file name for logging. fn = os.path.basename(infile) # ADM read in the primary targets. log.info('Reading primary targets file {}...t={:.1f}s'.format( infile, time() - start)) intargs, hdr = fitsio.read(infile, extension="TARGETS", header=True) # ADM fail if file's already been matched to secondary targets. if "SCNDDIR" in hdr: msg = "{} already matched to secondary targets".format(fn) \ + " (did you mean to remove {}?)!!!".format(fn) log.critical(msg) raise ValueError(msg) # ADM add the SCNDDIR to the primary targets file header. hdr["SCNDDIR"] = scxdir # ADM add a SCND_TARGET column to the primary targets. dt = intargs.dtype.descr dt.append(('SCND_TARGET', '>i8')) targs = np.zeros(len(intargs), dtype=dt) for col in intargs.dtype.names: targs[col] = intargs[col] # ADM match to all secondary targets for non-custom primary files. inhp = np.ones(len(scxtargs), dtype="?") # ADM as a speed-up, save memory by limiting the secondary targets # ADM to just HEALPixels that could touch the primary targets. if 'FILEHPX' in hdr: nside, pix = hdr['FILENSID'], hdr['FILEHPX'] # ADM remember to grab adjacent pixels in case of edge effects. allpix = add_hp_neighbors(nside, pix) inhp = is_in_hp(scxtargs, nside, allpix) # ADM it's unlikely that the matching separation is comparable # ADM to the HEALPixel resolution, but guard against that anyway. halfpix = np.degrees(hp.max_pixrad(nside)) * 3600. if sep > halfpix: msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format( sep, halfpix) log.critical(msg) raise ValueError(msg) # ADM warn the user if the secondary and primary samples are "large". big = 500000 if np.sum(inhp) > big and len(intargs) > big: log.warning('Large secondary (N={}) and primary (N={}) samples'.format( np.sum(inhp), len(intargs))) log.warning('The code may run slowly') # ADM for each secondary target, determine if there is a match # ADM with a primary target. Note that sense is important, here # ADM (the primary targets must be passed first). log.info( 'Matching primary and secondary targets for {} at {}"...t={:.1f}s'. format(fn, sep, time() - start)) mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep) # ADM recast the indices to the full set of secondary targets, # ADM instead of just those that were in the relevant HEALPixels. mscx = np.where(inhp)[0][mscx] # ADM loop through the matches and update the SCND_TARGET # ADM column in the primary target list. The np.unique is a # ADM speed-up to assign singular matches first. umtargs, inv, cnt = np.unique(mtargs, return_inverse=True, return_counts=True) # ADM number of times each primary target was matched, ordered # ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs. nmtargs = cnt[inv] # ADM assign anything with nmtargs = 1 directly. singular = nmtargs == 1 targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][ mscx[singular]] # ADM loop through things with nmtargs > 1 and combine the bits. for i in range(len((mtargs[~singular]))): targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][ mscx[~singular][i]] # ADM also assign the SCND_ANY bit to the primary targets. desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True) targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY # ADM rename the SCND_TARGET column, in case this is an SV file. targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]}) # ADM update the secondary targets with the primary TARGETID. scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs] # ADM form the output primary file name and write the file. base, ext = os.path.splitext(infile) outfile = "{}{}{}".format(base, '-wscnd', ext) log.info('Writing updated primary targets to {}...t={:.1f}s'.format( outfile, time() - start)) fitsio.write(outfile, targs, extname='TARGETS', header=hdr, clobber=True) log.info('Done for {}...t={:.1f}s'.format(fn, time() - start)) return scxtargs
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None): """Match secondary targets to primary targets and update bits. Parameters ---------- primtargs : :class:`~numpy.ndarray` An array of primary targets. scndout : :class`~numpy.ndarray` Name of a sub-directory to which to write the information in `desitarget.secondary.outdatamodel` with `TARGETID` and (the highest) `PRIORITY_INIT` updated with matching primary info. scxdir : :class:`str`, optional, defaults to `None` Name of the directory that hosts secondary targets. sep : :class:`float`, defaults to 1 arcsecond The separation at which to match in ARCSECONDS. pix : :class:`list`, optional, defaults to `None` Limit secondary targets to (NESTED) HEALpixels that touch pix at the supplied `nside`, as a speed-up. nside : :class:`int`, optional, defaults to `None` The (NESTED) HEALPixel nside to be used with `pixlist`. Returns ------- :class:`~numpy.ndarray` The array of primary targets, with the `SCND_TARGET` bit populated for matches to secondary targets """ # ADM add a SCND_TARGET column to the primary targets. dt = primtargs.dtype.descr dt.append(('SCND_TARGET', '>i8')) targs = np.zeros(len(primtargs), dtype=dt) for col in primtargs.dtype.names: targs[col] = primtargs[col] # ADM check if this is an SV or main survey file. cols, mx, surv = main_cmx_or_sv(targs, scnd=True) log.info('running on the {} survey...'.format(surv)) if surv != 'main': scxdir = os.path.join(scxdir, surv) # ADM read in non-OVERRIDE secondary targets. scxtargs = read_files(scxdir, mx[3]) scxtargs = scxtargs[~scxtargs["OVERRIDE"]] # ADM match primary targets to non-OVERRIDE secondary targets. inhp = np.ones(len(scxtargs), dtype="?") # ADM as a speed-up, save memory by limiting the secondary targets # ADM to just HEALPixels that could touch the primary targets. if nside is not None and pix is not None: # ADM remember to grab adjacent pixels in case of edge effects. allpix = add_hp_neighbors(nside, pix) inhp = is_in_hp(scxtargs, nside, allpix) # ADM it's unlikely that the matching separation is comparable # ADM to the HEALPixel resolution, but guard against that anyway. halfpix = np.degrees(hp.max_pixrad(nside)) * 3600. if sep > halfpix: msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format( sep, halfpix) log.critical(msg) raise ValueError(msg) # ADM warn the user if the secondary and primary samples are "large". big = 500000 if np.sum(inhp) > big and len(primtargs) > big: log.warning('Large secondary (N={}) and primary (N={}) samples'.format( np.sum(inhp), len(primtargs))) log.warning('The code may run slowly') # ADM for each secondary target, determine if there is a match # ADM with a primary target. Note that sense is important, here # ADM (the primary targets must be passed first). log.info( 'Matching primary and secondary targets for {} at {}"...t={:.1f}s'. format(scndout, sep, time() - start)) mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep) # ADM recast the indices to the full set of secondary targets, # ADM instead of just those that were in the relevant HEALPixels. mscx = np.where(inhp)[0][mscx] # ADM loop through the matches and update the SCND_TARGET # ADM column in the primary target list. The np.unique is a # ADM speed-up to assign singular matches first. umtargs, inv, cnt = np.unique(mtargs, return_inverse=True, return_counts=True) # ADM number of times each primary target was matched, ordered # ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs. nmtargs = cnt[inv] # ADM assign anything with nmtargs = 1 directly. singular = nmtargs == 1 targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][ mscx[singular]] # ADM loop through things with nmtargs > 1 and combine the bits. for i in range(len((mtargs[~singular]))): targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][ mscx[~singular][i]] # ADM also assign the SCND_ANY bit to the primary targets. desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True) targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY # ADM rename the SCND_TARGET column, in case this is an SV file. targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]}) # ADM update the secondary targets with the primary information. scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs] # ADM the maximum priority will be used to break ties in the # ADM unlikely event that a secondary matches two primaries. hipri = np.maximum(targs["PRIORITY_INIT_DARK"], targs["PRIORITY_INIT_BRIGHT"]) scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs] # ADM write the secondary targets that have updated TARGETIDs. ii = scxtargs["TARGETID"] != -1 nmatches = np.sum(ii) log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format( nmatches, scndout, time() - start)) if nmatches > 0: hdr = fitsio.FITSHDR() hdr["SURVEY"] = surv fitsio.write(scndout, scxtargs[ii], extname='SCND_TARG', header=hdr, clobber=True) log.info('Done...t={:.1f}s'.format(time() - start)) return targs