Exemple #1
0
def supplement_skies(nskiespersqdeg=None,
                     numproc=16,
                     gaiadir=None,
                     nside=None,
                     pixlist=None,
                     mindec=-30.,
                     mingalb=10.,
                     radius=2.):
    """Generate supplemental sky locations using Gaia-G-band avoidance.

    Parameters
    ----------
    nskiespersqdeg : :class:`float`, optional
        The minimum DENSITY of sky fibers to generate. Defaults to
        reading from :func:`~desimodel.io` with a margin of 4x.
    numproc : :class:`int`, optional, defaults to 16
        The number of processes over which to parallelize.
    gaiadir : :class:`str`, optional, defaults to $GAIA_DIR
        The GAIA_DIR environment variable is set to this directory.
        If None is passed, then it's assumed to already exist.
    nside : :class:`int`, optional, defaults to `None`
        (NESTED) HEALPix `nside` to use with `pixlist`.
    pixlist : :class:`list` or `int`, optional, defaults to `None`
        Only return targets in a set of (NESTED) HEALpixels at the
        supplied `nside`. Useful for parallelizing across nodes.
        The first entry sets RELEASE for TARGETIDs, and must be < 1000
        (to prevent confusion with DR1 and above).
    mindec : :class:`float`, optional, defaults to -30
        Minimum declination (o) to include for output sky locations.
    mingalb : :class:`float`, optional, defaults to 10
        Closest latitude to Galactic plane for output sky locations
        (e.g. send 10 to limit to areas beyond -10o <= b < 10o).
    radius : :class:`float`, optional, defaults to 2
        Radius at which to avoid (all) Gaia sources (arcseconds).

    Returns
    -------
    :class:`~numpy.ndarray`
        a structured array of supplemental sky positions in the DESI sky
        target format within the passed `mindec` and `mingalb` limits.

    Notes
    -----
        - The environment variable $GAIA_DIR must be set, or `gaiadir`
          must be passed.
    """
    log.info("running on {} processors".format(numproc))

    # ADM if the GAIA directory was passed, set it.
    if gaiadir is not None:
        os.environ["GAIA_DIR"] = gaiadir

    # ADM if needed, determine the density of sky fibers to generate.
    if nskiespersqdeg is None:
        nskiespersqdeg = density_of_sky_fibers(margin=4)

    # ADM determine the HEALPixel nside of the standard Gaia files.
    anyfiles = find_gaia_files([0, 0], radec=True)
    hdr = fitsio.read_header(anyfiles[0], "GAIAHPX")
    nsidegaia = hdr["HPXNSIDE"]

    # ADM create a set of random locations accounting for mindec.
    log.info("Generating supplemental sky locations at Dec > {}o...t={:.1f}s".
             format(mindec,
                    time() - start))
    from desitarget.randoms import randoms_in_a_brick_from_edges
    ras, decs = randoms_in_a_brick_from_edges(0.,
                                              360.,
                                              mindec,
                                              90.,
                                              density=nskiespersqdeg,
                                              wrap=False)

    # ADM limit randoms by HEALPixel, if requested.
    if pixlist is not None:
        inhp = is_in_hp([ras, decs], nside, pixlist, radec=True)
        ras, decs = ras[inhp], decs[inhp]

    # ADM limit randoms by mingalb.
    log.info(
        "Generated {} sky locations. Limiting to |b| > {}o...t={:.1f}s".format(
            len(ras), mingalb,
            time() - start))
    bnorth = is_in_gal_box([ras, decs], [0, 360, mingalb, 90], radec=True)
    bsouth = is_in_gal_box([ras, decs], [0, 360, -90, -mingalb], radec=True)
    ras, decs = ras[bnorth | bsouth], decs[bnorth | bsouth]

    # ADM find which Gaia HEALPixels are occupied by the random points.
    log.info(
        "Cut to {} sky locations. Finding their Gaia HEALPixels...t={:.1f}s".
        format(len(ras),
               time() - start))
    theta, phi = np.radians(90 - decs), np.radians(ras)
    pixels = hp.ang2pix(nsidegaia, theta, phi, nest=True)
    upixels = np.unique(pixels)
    npixels = len(upixels)
    log.info("Running across {} Gaia HEALPixels.".format(npixels))

    # ADM parallelize across pixels. The function to run on every pixel.
    def _get_supp(pix):
        """wrapper on get_supp_skies() given a HEALPixel"""
        ii = (pixels == pix)
        return get_supp_skies(ras[ii], decs[ii], radius=radius)

    # ADM this is just to count pixels in _update_status.
    npix = np.zeros((), dtype='i8')
    t0 = time()

    def _update_status(result):
        """wrapper function for the critical reduction operation,
        that occurs on the main parallel process"""
        if npix % 500 == 0 and npix > 0:
            rate = npix / (time() - t0)
            log.info('{}/{} HEALPixels; {:.1f} pixels/sec'.format(
                npix, npixels, rate))
        npix[...] += 1  # this is an in-place modification.
        return result

    # - Parallel process across the unique pixels.
    if numproc > 1:
        pool = sharedmem.MapReduce(np=numproc)
        with pool:
            supp = pool.map(_get_supp, upixels, reduce=_update_status)
    else:
        supp = []
        for upix in upixels:
            supp.append(_update_status(_get_supp(upix)))

    # ADM Concatenate the parallelized results into one rec array.
    supp = np.concatenate(supp)

    # ADM build the OBJIDs from the number of sources per brick.
    # ADM the for loop doesn't seem the smartest way, but it is O(n).
    log.info("Begin assigning OBJIDs to bricks...t={:.1f}s".format(time() -
                                                                   start))
    brxid = supp["BRICKID"]
    # ADM start each brick counting from zero.
    cntr = np.zeros(np.max(brxid) + 1, dtype=int)
    objid = []
    for ibrx in brxid:
        cntr[ibrx] += 1
        objid.append(cntr[ibrx])
    # ADM ensure the number of sky positions that were generated doesn't exceed
    # ADM the largest possible OBJID (which is unlikely).
    if np.any(cntr > 2**targetid_mask.OBJID.nbits):
        log.fatal(
            '{} sky locations requested in brick {}, but OBJID cannot exceed {}'
            .format(nskies, brickname, 2**targetid_mask.OBJID.nbits))
        raise ValueError
    supp["OBJID"] = np.array(objid)
    log.info("Assigned OBJIDs to bricks...t={:.1f}s".format(time() - start))

    # ADM if splitting by HEALPixels, use the input list to
    # ADM set RELEASE so we don't have duplicate TARGETIDs.
    if pixlist is not None and pixlist[0] < 1000:
        supp["RELEASE"] = pixlist[0]
    else:
        msg = "First entry in pixlist ({}) > 1000. This sets ".format(
            pixlist[0])
        msg += "RELEASE for SUPP_SKIES. > 1000 may duplicate TARGETID for SKIES!"
        log.critical(msg)
        raise IOError

    # ADM add the TARGETID, DESITARGET bits etc.
    nskies = len(supp)
    desi_target = np.zeros(nskies, dtype='>i8')
    desi_target |= desi_mask.SKY
    desi_target |= desi_mask.SUPP_SKY
    dum = np.zeros_like(desi_target)
    supp = finalize(supp, desi_target, dum, dum, sky=1)

    log.info('Done...t={:.1f}s'.format(time() - start))

    return supp
Exemple #2
0
def gaia_in_file(infile,
                 maglim=18,
                 mindec=-30.,
                 mingalb=10.,
                 nside=None,
                 pixlist=None,
                 addobjid=False):
    """Retrieve the Gaia objects from a HEALPixel-split Gaia file.

    Parameters
    ----------
    infile : :class:`str`
        File name of a single Gaia "healpix" file.
    maglim : :class:`float`, optional, defaults to 18
        Magnitude limit for GFAs in Gaia G-band.
    mindec : :class:`float`, optional, defaults to -30
        Minimum declination (o) to include for output Gaia objects.
    mingalb : :class:`float`, optional, defaults to 10
        Closest latitude to Galactic plane for output Gaia objects
        (e.g. send 10 to limit to areas beyond -10o <= b < 10o)"
    nside : :class:`int`, optional, defaults to `None`
        (NESTED) HEALPix `nside` to use with `pixlist`.
    pixlist : :class:`list` or `int`, optional, defaults to `None`
        Only return sources in a set of (NESTED) HEALpixels at the
        supplied `nside`.
    addobjid : :class:`bool`, optional, defaults to ``False``
        If ``True``, include, in the output, a column "GAIA_OBJID"
        that is the integer number of each row read from file.

    Returns
    -------
    :class:`~numpy.ndarray`
        Gaia objects in the passed Gaia file brighter than `maglim`,
        formatted according to `desitarget.gfa.gfadatamodel`.

    Notes
    -----
       - A "Gaia healpix file" here is as made by, e.g.
         :func:`~desitarget.gaiamatch.gaia_fits_to_healpix()`
    """
    # ADM read in the Gaia file and limit to the passed magnitude.
    objs = read_gaia_file(infile, addobjid=addobjid)
    ii = objs['GAIA_PHOT_G_MEAN_MAG'] < maglim
    objs = objs[ii]

    # ADM rename GAIA_RA/DEC to RA/DEC, as that's what's used for GFAs.
    for radec in ["RA", "DEC"]:
        objs.dtype.names = [
            radec if col == "GAIA_" + radec else col
            for col in objs.dtype.names
        ]

    # ADM initiate the GFA data model.
    dt = gfadatamodel.dtype.descr
    if addobjid:
        for tup in ('GAIA_BRICKID', '>i4'), ('GAIA_OBJID', '>i4'):
            dt.append(tup)

    gfas = np.zeros(len(objs), dtype=dt)
    # ADM make sure all columns initially have "ridiculous" numbers
    gfas[...] = -99.
    for col in gfas.dtype.names:
        if isinstance(gfas[col][0].item(), (bytes, str)):
            gfas[col] = 'U'
        if isinstance(gfas[col][0].item(), int):
            gfas[col] = -1
    # ADM some default special cases. Default to REF_EPOCH of Gaia DR2,
    # ADM make RA/Dec very precise for Gaia measurements.
    gfas["REF_EPOCH"] = 2015.5
    gfas["RA_IVAR"], gfas["DEC_IVAR"] = 1e16, 1e16

    # ADM populate the common columns in the Gaia/GFA data models.
    cols = set(gfas.dtype.names).intersection(set(objs.dtype.names))
    for col in cols:
        gfas[col] = objs[col]

    # ADM update the Gaia morphological type.
    gfas["TYPE"] = gaia_morph(gfas)

    # ADM populate the BRICKID columns.
    gfas["BRICKID"] = bricks.brickid(gfas["RA"], gfas["DEC"])

    # ADM limit by HEALPixel first as that's the fastest.
    if pixlist is not None:
        inhp = is_in_hp(gfas, nside, pixlist)
        gfas = gfas[inhp]
    # ADM limit by Dec first to speed transform to Galactic coordinates.
    decgood = is_in_box(gfas, [0., 360., mindec, 90.])
    gfas = gfas[decgood]
    # ADM now limit to requesed Galactic latitude range.
    if mingalb > 1e-9:
        bbad = is_in_gal_box(gfas, [0., 360., -mingalb, mingalb])
        gfas = gfas[~bbad]

    return gfas
Exemple #3
0
def select_gfas(infiles,
                maglim=18,
                numproc=4,
                nside=None,
                pixlist=None,
                bundlefiles=None,
                extra=None,
                mindec=-30,
                mingalb=10,
                addurat=True):
    """Create a set of GFA locations using Gaia and matching to sweeps.

    Parameters
    ----------
    infiles : :class:`list` or `str`
        A list of input filenames (sweep files) OR a single filename.
    maglim : :class:`float`, optional, defaults to 18
        Magnitude limit for GFAs in Gaia G-band.
    numproc : :class:`int`, optional, defaults to 4
        The number of parallel processes to use.
    nside : :class:`int`, optional, defaults to `None`
        (NESTED) HEALPix `nside` to use with `pixlist` and `bundlefiles`.
    pixlist : :class:`list` or `int`, optional, defaults to `None`
        Only return targets in a set of (NESTED) HEALpixels at the
        supplied `nside`. Useful for parallelizing.
    bundlefiles : :class:`int`, defaults to `None`
        If not `None`, then, instead of selecting gfas, print the slurm
        script to run in pixels at `nside`. Is an integer rather than
        a boolean for historical reasons.
    extra : :class:`str`, optional
        Extra command line flags to be passed to the executable lines in
        the output slurm script. Used in conjunction with `bundlefiles`.
    mindec : :class:`float`, optional, defaults to -30
        Minimum declination (o) for output sources that do NOT match
        an object in the passed `infiles`.
    mingalb : :class:`float`, optional, defaults to 10
        Closest latitude to Galactic plane for output sources that
        do NOT match an object in the passed `infiles` (e.g. send
        10 to limit to regions beyond -10o <= b < 10o)".
    addurat : :class:`bool`, optional, defaults to ``True``
        If ``True`` then substitute proper motions from the URAT
        catalog where Gaia is missing proper motions. Requires that
        the :envvar:`URAT_DIR` is set and points to data downloaded and
        formatted by, e.g., :func:`~desitarget.uratmatch.make_urat_files`.

    Returns
    -------
    :class:`~numpy.ndarray`
        GFA objects from Gaia with the passed geometric constraints
        limited to the passed maglim and matched to the passed input
        files, formatted according to `desitarget.gfa.gfadatamodel`.

    Notes
    -----
        - If numproc==1, use the serial code instead of parallel code.
        - If numproc > 4, then numproc=4 is enforced for (just those)
          parts of the code that are I/O limited.
    """
    # ADM the code can have memory issues for nside=2 with large numproc.
    if nside is not None and nside < 4 and numproc > 8:
        msg = 'Memory may be an issue near Plane for nside < 4 and numproc > 8'
        log.warning(msg)

    # ADM force to no more than numproc=4 for I/O limited processes.
    numproc4 = numproc
    if numproc4 > 4:
        log.info('Forcing numproc to 4 for I/O limited parts of code')
        numproc4 = 4

    # ADM convert a single file, if passed to a list of files.
    if isinstance(infiles, str):
        infiles = [
            infiles,
        ]

    # ADM check that files exist before proceeding.
    for filename in infiles:
        if not os.path.exists(filename):
            msg = "{} doesn't exist".format(filename)
            log.critical(msg)
            raise ValueError(msg)

    # ADM if the pixlist option was sent, we'll need to
    # ADM know which HEALPixels touch each file.
    if pixlist is not None:
        filesperpixel, _, _ = sweep_files_touch_hp(nside, pixlist, infiles)

    # ADM if the bundlefiles option was sent, call the packing code.
    if bundlefiles is not None:
        # ADM were files from one or two input directories passed?
        surveydirs = list(set([os.path.dirname(fn) for fn in infiles]))
        bundle_bricks([0],
                      bundlefiles,
                      nside,
                      gather=False,
                      prefix='gfas',
                      surveydirs=surveydirs,
                      extra=extra)
        return

    # ADM restrict to input files in a set of HEALPixels, if requested.
    if pixlist is not None:
        infiles = list(set(np.hstack([filesperpixel[pix] for pix in pixlist])))
        if len(infiles) == 0:
            log.info('ZERO sweep files in passed pixel list!!!')
        log.info("Processing files in (nside={}, pixel numbers={}) HEALPixels".
                 format(nside, pixlist))
    nfiles = len(infiles)

    # ADM a little more information if we're slurming across nodes.
    if os.getenv('SLURMD_NODENAME') is not None:
        log.info('Running on Node {}'.format(os.getenv('SLURMD_NODENAME')))

    # ADM the critical function to run on every file.
    def _get_gfas(fn):
        '''wrapper on gaia_gfas_from_sweep() given a file name'''
        return gaia_gfas_from_sweep(fn, maglim=maglim)

    # ADM this is just to count sweeps files in _update_status.
    t0 = time()
    nfile = np.zeros((), dtype='i8')

    def _update_status(result):
        """wrapper function for the critical reduction operation,
        that occurs on the main parallel process"""
        if nfile % 20 == 0 and nfile > 0:
            elapsed = (time() - t0) / 60.
            rate = nfile / elapsed / 60.
            log.info('{}/{} files; {:.1f} files/sec...t = {:.1f} mins'.format(
                nfile, nfiles, rate, elapsed))
        nfile[...] += 1  # this is an in-place modification.
        return result

    # - Parallel process input files.
    if len(infiles) > 0:
        if numproc4 > 1:
            pool = sharedmem.MapReduce(np=numproc4)
            with pool:
                gfas = pool.map(_get_gfas, infiles, reduce=_update_status)
        else:
            gfas = list()
            for file in infiles:
                gfas.append(_update_status(_get_gfas(file)))
        gfas = np.concatenate(gfas)
        # ADM resolve any duplicates between imaging data releases.
        gfas = resolve(gfas)

    # ADM retrieve Gaia objects in the DESI footprint or passed tiles.
    log.info('Retrieving additional Gaia objects...t = {:.1f} mins'.format(
        (time() - t0) / 60))
    gaia = all_gaia_in_tiles(maglim=maglim,
                             numproc=numproc4,
                             allsky=True,
                             mindec=mindec,
                             mingalb=mingalb,
                             nside=nside,
                             pixlist=pixlist)

    # ADM remove any duplicates. Order is important here, as np.unique
    # ADM keeps the first occurence, and we want to retain sweeps
    # ADM information as much as possible.
    if len(infiles) > 0:
        gfas = np.concatenate([gfas, gaia])
        _, ind = np.unique(gfas["REF_ID"], return_index=True)
        gfas = gfas[ind]
    else:
        gfas = gaia

    # ADM for zero/NaN proper motion objects, add URAT proper motions.
    if addurat:
        ii = ((np.isnan(gfas["PMRA"]) | (gfas["PMRA"] == 0)) &
              (np.isnan(gfas["PMDEC"]) | (gfas["PMDEC"] == 0)))
        log.info(
            'Adding URAT for {} objects with no PMs...t = {:.1f} mins'.format(
                np.sum(ii), (time() - t0) / 60))
        urat = add_urat_pms(gfas[ii], numproc=numproc)
        log.info(
            'Found an additional {} URAT objects...t = {:.1f} mins'.format(
                np.sum(urat["URAT_ID"] != -1), (time() - t0) / 60))
        for col in "PMRA", "PMDEC", "URAT_ID", "URAT_SEP":
            gfas[col][ii] = urat[col]

    # ADM restrict to only GFAs in a set of HEALPixels, if requested.
    if pixlist is not None:
        ii = is_in_hp(gfas, nside, pixlist)
        gfas = gfas[ii]

    return gfas
Exemple #4
0
def match_secondary(infile, scxtargs, sep=1., scxdir=None):
    """Match secondary targets to primary targets and update bits.

    Parameters
    ----------
    infile : :class:`str`
        The full path to a file containing primary targets.
    scxtargs : :class:`~numpy.ndarray`
        An array of secondary targets.
    sep : :class:`float`, defaults to 1 arcsecond
        The separation at which to match in ARCSECONDS.
    scxdir : :class:`str`, optional, defaults to `None`
        Name of the directory that hosts secondary targets. If passed,
        this is written to the output primary file header as `SCNDDIR`.

    Returns
    -------
    :class:`~numpy.ndarray`
        The array of secondary targets, with the `TARGETID` bit
        updated with any matching primary targets from `infile`.

    Notes
    -----
        - The primary target `infiles` are written back to their
          original path with `.fits` changed to `-wscnd.fits` and the
          `SCND_TARGET` bit populated for matching targets.
    """
    # ADM just the file name for logging.
    fn = os.path.basename(infile)
    # ADM read in the primary targets.
    log.info('Reading primary targets file {}...t={:.1f}s'.format(
        infile,
        time() - start))
    intargs, hdr = fitsio.read(infile, extension="TARGETS", header=True)

    # ADM fail if file's already been matched to secondary targets.
    if "SCNDDIR" in hdr:
        msg = "{} already matched to secondary targets".format(fn) \
              + " (did you mean to remove {}?)!!!".format(fn)
        log.critical(msg)
        raise ValueError(msg)
    # ADM add the SCNDDIR to the primary targets file header.
    hdr["SCNDDIR"] = scxdir
    # ADM add a SCND_TARGET column to the primary targets.
    dt = intargs.dtype.descr
    dt.append(('SCND_TARGET', '>i8'))
    targs = np.zeros(len(intargs), dtype=dt)
    for col in intargs.dtype.names:
        targs[col] = intargs[col]

    # ADM match to all secondary targets for non-custom primary files.
    inhp = np.ones(len(scxtargs), dtype="?")
    # ADM as a speed-up, save memory by limiting the secondary targets
    # ADM to just HEALPixels that could touch the primary targets.
    if 'FILEHPX' in hdr:
        nside, pix = hdr['FILENSID'], hdr['FILEHPX']
        # ADM remember to grab adjacent pixels in case of edge effects.
        allpix = add_hp_neighbors(nside, pix)
        inhp = is_in_hp(scxtargs, nside, allpix)
        # ADM it's unlikely that the matching separation is comparable
        # ADM to the HEALPixel resolution, but guard against that anyway.
        halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
        if sep > halfpix:
            msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
                sep, halfpix)
            log.critical(msg)
            raise ValueError(msg)
    # ADM warn the user if the secondary and primary samples are "large".
    big = 500000
    if np.sum(inhp) > big and len(intargs) > big:
        log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
            np.sum(inhp), len(intargs)))
        log.warning('The code may run slowly')

    # ADM for each secondary target, determine if there is a match
    # ADM with a primary target. Note that sense is important, here
    # ADM (the primary targets must be passed first).
    log.info(
        'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
        format(fn, sep,
               time() - start))
    mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
    # ADM recast the indices to the full set of secondary targets,
    # ADM instead of just those that were in the relevant HEALPixels.
    mscx = np.where(inhp)[0][mscx]

    # ADM loop through the matches and update the SCND_TARGET
    # ADM column in the primary target list. The np.unique is a
    # ADM speed-up to assign singular matches first.
    umtargs, inv, cnt = np.unique(mtargs,
                                  return_inverse=True,
                                  return_counts=True)
    # ADM number of times each primary target was matched, ordered
    # ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
    nmtargs = cnt[inv]
    # ADM assign anything with nmtargs = 1 directly.
    singular = nmtargs == 1
    targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
        mscx[singular]]
    # ADM loop through things with nmtargs > 1 and combine the bits.
    for i in range(len((mtargs[~singular]))):
        targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
            mscx[~singular][i]]
    # ADM also assign the SCND_ANY bit to the primary targets.
    desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
    targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY

    # ADM rename the SCND_TARGET column, in case this is an SV file.
    targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})

    # ADM update the secondary targets with the primary TARGETID.
    scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]

    # ADM form the output primary file name and write the file.
    base, ext = os.path.splitext(infile)
    outfile = "{}{}{}".format(base, '-wscnd', ext)
    log.info('Writing updated primary targets to {}...t={:.1f}s'.format(
        outfile,
        time() - start))
    fitsio.write(outfile, targs, extname='TARGETS', header=hdr, clobber=True)

    log.info('Done for {}...t={:.1f}s'.format(fn, time() - start))

    return scxtargs
Exemple #5
0
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None):
    """Match secondary targets to primary targets and update bits.

    Parameters
    ----------
    primtargs : :class:`~numpy.ndarray`
        An array of primary targets.
    scndout : :class`~numpy.ndarray`
        Name of a sub-directory to which to write the information in
        `desitarget.secondary.outdatamodel` with `TARGETID` and (the
        highest) `PRIORITY_INIT` updated with matching primary info.
    scxdir : :class:`str`, optional, defaults to `None`
        Name of the directory that hosts secondary targets.
    sep : :class:`float`, defaults to 1 arcsecond
        The separation at which to match in ARCSECONDS.
    pix : :class:`list`, optional, defaults to `None`
        Limit secondary targets to (NESTED) HEALpixels that touch
        pix at the supplied `nside`, as a speed-up.
    nside : :class:`int`, optional, defaults to `None`
        The (NESTED) HEALPixel nside to be used with `pixlist`.

    Returns
    -------
    :class:`~numpy.ndarray`
        The array of primary targets, with the `SCND_TARGET` bit
        populated for matches to secondary targets
    """
    # ADM add a SCND_TARGET column to the primary targets.
    dt = primtargs.dtype.descr
    dt.append(('SCND_TARGET', '>i8'))
    targs = np.zeros(len(primtargs), dtype=dt)
    for col in primtargs.dtype.names:
        targs[col] = primtargs[col]

    # ADM check if this is an SV or main survey file.
    cols, mx, surv = main_cmx_or_sv(targs, scnd=True)
    log.info('running on the {} survey...'.format(surv))
    if surv != 'main':
        scxdir = os.path.join(scxdir, surv)

    # ADM read in non-OVERRIDE secondary targets.
    scxtargs = read_files(scxdir, mx[3])
    scxtargs = scxtargs[~scxtargs["OVERRIDE"]]

    # ADM match primary targets to non-OVERRIDE secondary targets.
    inhp = np.ones(len(scxtargs), dtype="?")
    # ADM as a speed-up, save memory by limiting the secondary targets
    # ADM to just HEALPixels that could touch the primary targets.
    if nside is not None and pix is not None:
        # ADM remember to grab adjacent pixels in case of edge effects.
        allpix = add_hp_neighbors(nside, pix)
        inhp = is_in_hp(scxtargs, nside, allpix)
        # ADM it's unlikely that the matching separation is comparable
        # ADM to the HEALPixel resolution, but guard against that anyway.
        halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
        if sep > halfpix:
            msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
                sep, halfpix)
            log.critical(msg)
            raise ValueError(msg)

    # ADM warn the user if the secondary and primary samples are "large".
    big = 500000
    if np.sum(inhp) > big and len(primtargs) > big:
        log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
            np.sum(inhp), len(primtargs)))
        log.warning('The code may run slowly')

    # ADM for each secondary target, determine if there is a match
    # ADM with a primary target. Note that sense is important, here
    # ADM (the primary targets must be passed first).
    log.info(
        'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
        format(scndout, sep,
               time() - start))
    mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
    # ADM recast the indices to the full set of secondary targets,
    # ADM instead of just those that were in the relevant HEALPixels.
    mscx = np.where(inhp)[0][mscx]

    # ADM loop through the matches and update the SCND_TARGET
    # ADM column in the primary target list. The np.unique is a
    # ADM speed-up to assign singular matches first.
    umtargs, inv, cnt = np.unique(mtargs,
                                  return_inverse=True,
                                  return_counts=True)
    # ADM number of times each primary target was matched, ordered
    # ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
    nmtargs = cnt[inv]
    # ADM assign anything with nmtargs = 1 directly.
    singular = nmtargs == 1
    targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
        mscx[singular]]
    # ADM loop through things with nmtargs > 1 and combine the bits.
    for i in range(len((mtargs[~singular]))):
        targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
            mscx[~singular][i]]
    # ADM also assign the SCND_ANY bit to the primary targets.
    desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
    desi_mask = desimasks[0]

    targs[desicols[0]][umtargs] |= desi_mask.SCND_ANY

    # ADM rename the SCND_TARGET column, in case this is an SV file.
    targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})

    # APC Secondary target bits only affect PRIORITY, NUMOBS and
    # APC obsconditions for specific DESI_TARGET bits
    # APC See https://github.com/desihub/desitarget/pull/530

    # APC Only consider primary targets with secondary bits set
    scnd_update = (targs[desicols[0]] & desi_mask['SCND_ANY']) != 0
    if np.any(scnd_update):
        # APC Allow changes to primaries if the DESI_TARGET bitmask has
        # APC only the following bits set, in any combination.
        log.info(
            'Testing if secondary targets can update {} matched primaries'.
            format(scnd_update.sum()))
        update_from_scnd_bits = desi_mask['SCND_ANY'] | desi_mask[
            'MWS_ANY'] | desi_mask['STD_BRIGHT'] | desi_mask[
                'STD_FAINT'] | desi_mask['STD_WD']
        scnd_update &= ((targs[desicols[0]] & ~update_from_scnd_bits) == 0)
        log.info(
            'Setting new priority, numobs and obsconditions from secondary for {} matched primaries'
            .format(scnd_update.sum()))

        # APC Primary and secondary obsconditions are or'd
        scnd_obscon = set_obsconditions(targs[scnd_update], scnd=True)
        targs['OBSCONDITIONS'][scnd_update] &= scnd_obscon

        # APC bit of a hack here
        # APC Check for _BRIGHT, _DARK split in column names
        darkbright = 'NUMOBS_INIT_DARK' in targs.dtype.names
        if darkbright:
            ender, obscon = ["_DARK", "_BRIGHT"], ["DARK|GRAY", "BRIGHT"]
        else:
            ender, obscon = [""], [
                "DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18"
            ]

        # APC secondaries can increase priority and numobs
        for edr, oc in zip(ender, obscon):
            pc, nc = "PRIORITY_INIT" + edr, "NUMOBS_INIT" + edr
            scnd_priority, scnd_numobs = initial_priority_numobs(
                targs[scnd_update], obscon=oc, scnd=True)
            targs[nc][scnd_update] = np.maximum(targs[nc][scnd_update],
                                                scnd_numobs)
            targs[pc][scnd_update] = np.maximum(targs[pc][scnd_update],
                                                scnd_priority)

    # ADM update the secondary targets with the primary information.
    scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
    # ADM the maximum priority will be used to break ties in the
    # ADM unlikely event that a secondary matches two primaries.
    hipri = np.maximum(targs["PRIORITY_INIT_DARK"],
                       targs["PRIORITY_INIT_BRIGHT"])
    scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs]

    # ADM write the secondary targets that have updated TARGETIDs.
    ii = scxtargs["TARGETID"] != -1
    nmatches = np.sum(ii)
    log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format(
        nmatches, scndout,
        time() - start))
    if nmatches > 0:
        hdr = fitsio.FITSHDR()
        hdr["SURVEY"] = surv
        fitsio.write(scndout,
                     scxtargs[ii],
                     extname='SCND_TARG',
                     header=hdr,
                     clobber=True)

    log.info('Done...t={:.1f}s'.format(time() - start))

    return targs
Exemple #6
0
def match_secondary(primtargs, scxdir, scndout, sep=1., pix=None, nside=None):
    """Match secondary targets to primary targets and update bits.

    Parameters
    ----------
    primtargs : :class:`~numpy.ndarray`
        An array of primary targets.
    scndout : :class`~numpy.ndarray`
        Name of a sub-directory to which to write the information in
        `desitarget.secondary.outdatamodel` with `TARGETID` and (the
        highest) `PRIORITY_INIT` updated with matching primary info.
    scxdir : :class:`str`, optional, defaults to `None`
        Name of the directory that hosts secondary targets.
    sep : :class:`float`, defaults to 1 arcsecond
        The separation at which to match in ARCSECONDS.
    pix : :class:`list`, optional, defaults to `None`
        Limit secondary targets to (NESTED) HEALpixels that touch
        pix at the supplied `nside`, as a speed-up.
    nside : :class:`int`, optional, defaults to `None`
        The (NESTED) HEALPixel nside to be used with `pixlist`.

    Returns
    -------
    :class:`~numpy.ndarray`
        The array of primary targets, with the `SCND_TARGET` bit
        populated for matches to secondary targets
    """
    # ADM add a SCND_TARGET column to the primary targets.
    dt = primtargs.dtype.descr
    dt.append(('SCND_TARGET', '>i8'))
    targs = np.zeros(len(primtargs), dtype=dt)
    for col in primtargs.dtype.names:
        targs[col] = primtargs[col]

    # ADM check if this is an SV or main survey file.
    cols, mx, surv = main_cmx_or_sv(targs, scnd=True)
    log.info('running on the {} survey...'.format(surv))
    if surv != 'main':
        scxdir = os.path.join(scxdir, surv)

    # ADM read in non-OVERRIDE secondary targets.
    scxtargs = read_files(scxdir, mx[3])
    scxtargs = scxtargs[~scxtargs["OVERRIDE"]]

    # ADM match primary targets to non-OVERRIDE secondary targets.
    inhp = np.ones(len(scxtargs), dtype="?")
    # ADM as a speed-up, save memory by limiting the secondary targets
    # ADM to just HEALPixels that could touch the primary targets.
    if nside is not None and pix is not None:
        # ADM remember to grab adjacent pixels in case of edge effects.
        allpix = add_hp_neighbors(nside, pix)
        inhp = is_in_hp(scxtargs, nside, allpix)
        # ADM it's unlikely that the matching separation is comparable
        # ADM to the HEALPixel resolution, but guard against that anyway.
        halfpix = np.degrees(hp.max_pixrad(nside)) * 3600.
        if sep > halfpix:
            msg = 'sep ({}") exceeds (half) HEALPixel size ({}")'.format(
                sep, halfpix)
            log.critical(msg)
            raise ValueError(msg)
    # ADM warn the user if the secondary and primary samples are "large".
    big = 500000
    if np.sum(inhp) > big and len(primtargs) > big:
        log.warning('Large secondary (N={}) and primary (N={}) samples'.format(
            np.sum(inhp), len(primtargs)))
        log.warning('The code may run slowly')

    # ADM for each secondary target, determine if there is a match
    # ADM with a primary target. Note that sense is important, here
    # ADM (the primary targets must be passed first).
    log.info(
        'Matching primary and secondary targets for {} at {}"...t={:.1f}s'.
        format(scndout, sep,
               time() - start))
    mtargs, mscx = radec_match_to(targs, scxtargs[inhp], sep=sep)
    # ADM recast the indices to the full set of secondary targets,
    # ADM instead of just those that were in the relevant HEALPixels.
    mscx = np.where(inhp)[0][mscx]

    # ADM loop through the matches and update the SCND_TARGET
    # ADM column in the primary target list. The np.unique is a
    # ADM speed-up to assign singular matches first.
    umtargs, inv, cnt = np.unique(mtargs,
                                  return_inverse=True,
                                  return_counts=True)
    # ADM number of times each primary target was matched, ordered
    # ADM the same as mtargs, i.e. n(mtargs) for each entry in mtargs.
    nmtargs = cnt[inv]
    # ADM assign anything with nmtargs = 1 directly.
    singular = nmtargs == 1
    targs["SCND_TARGET"][mtargs[singular]] = scxtargs["SCND_TARGET"][
        mscx[singular]]
    # ADM loop through things with nmtargs > 1 and combine the bits.
    for i in range(len((mtargs[~singular]))):
        targs["SCND_TARGET"][mtargs[~singular][i]] |= scxtargs["SCND_TARGET"][
            mscx[~singular][i]]
    # ADM also assign the SCND_ANY bit to the primary targets.
    desicols, desimasks, _ = main_cmx_or_sv(targs, scnd=True)
    targs[desicols[0]][umtargs] |= desimasks[0].SCND_ANY

    # ADM rename the SCND_TARGET column, in case this is an SV file.
    targs = rfn.rename_fields(targs, {'SCND_TARGET': desicols[3]})

    # ADM update the secondary targets with the primary information.
    scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs]
    # ADM the maximum priority will be used to break ties in the
    # ADM unlikely event that a secondary matches two primaries.
    hipri = np.maximum(targs["PRIORITY_INIT_DARK"],
                       targs["PRIORITY_INIT_BRIGHT"])
    scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs]

    # ADM write the secondary targets that have updated TARGETIDs.
    ii = scxtargs["TARGETID"] != -1
    nmatches = np.sum(ii)
    log.info('Writing {} secondary target matches to {}...t={:.1f}s'.format(
        nmatches, scndout,
        time() - start))
    if nmatches > 0:
        hdr = fitsio.FITSHDR()
        hdr["SURVEY"] = surv
        fitsio.write(scndout,
                     scxtargs[ii],
                     extname='SCND_TARG',
                     header=hdr,
                     clobber=True)

    log.info('Done...t={:.1f}s'.format(time() - start))

    return targs
Exemple #7
0
def read_targets_in_hp(hpdirname, nside, pixlist, columns=None):
    """Read in targets in a set of HEALPixels.

    Parameters
    ----------
    hpdirname : :class:`str`
        Full path to either a directory containing targets that
        have been partitioned by HEALPixel (i.e. as made by
        `select_targets` with the `bundle_files` option). Or the
        name of a single file of targets.
    nside : :class:`int`
        The (NESTED) HEALPixel nside.
    pixlist : :class:`list` or `int` or `~numpy.ndarray`
        Return targets in these HEALPixels at the passed `nside`.
    columns : :class:`list`, optional
        Only read in these target columns.

    Returns
    -------
    :class:`~numpy.ndarray`
        An array of targets in the passed pixels.
    """
    # ADM we'll need RA/Dec for final cuts, so ensure they're read.
    addedcols = []
    columnscopy = None
    if columns is not None:
        # ADM make a copy of columns, as it's a kwarg we'll modify.
        columnscopy = columns.copy()
        for radec in ["RA", "DEC"]:
            if radec not in columnscopy:
                columnscopy.append(radec)
                addedcols.append(radec)

    # ADM if a directory was passed, do fancy HEALPixel parsing...
    if os.path.isdir(hpdirname):
        # ADM check, and grab information from, the target directory.
        filenside, filedict = check_hp_target_dir(hpdirname)

        # ADM change the passed pixels to the nside of the file schema.
        filepixlist = nside2nside(nside, filenside, pixlist)

        # ADM only consider pixels for which we have a file.
        isindict = [pix in filedict for pix in filepixlist]
        filepixlist = filepixlist[isindict]

        # ADM make sure each file is only read once.
        infiles = set([filedict[pix] for pix in filepixlist])

        # ADM read in the files and concatenate the resulting targets.
        targets = []
        for infile in infiles:
            targets.append(fitsio.read(infile, columns=columnscopy))
        targets = np.concatenate(targets)
    # ADM ...otherwise just read in the targets.
    else:
        targets = fitsio.read(hpdirname, columns=columnscopy)

    # ADM restrict the targets to the actual requested HEALPixels...
    ii = is_in_hp(targets, nside, pixlist)
    # ADM ...and remove RA/Dec columns if we added them.
    targets = rfn.drop_fields(targets[ii], addedcols)

    return targets
Exemple #8
0
def mask_targets(targs, inmaskdir, nside=2, pixlist=None, bricks_are_hpx=False):
    """Add bits for if objects occupy masks, and SAFE (BADSKY) locations.

    Parameters
    ----------
    targs : :class:`str` or `~numpy.ndarray`
        An array of targets/skies etc. created by, e.g.,
        :func:`desitarget.cuts.select_targets()` OR the filename of a
        file that contains such a set of targets/skies, etc.
    inmaskdir : :class:`str`, optional
        An input bright star mask file or HEALPixel-split directory as
        made by :func:`desitarget.brightmask.make_bright_star_mask()`
    nside : :class:`int`, optional, defaults to 2
        The nside at which the targets were generated. If the mask is
        a HEALPixel-split directory, then this helps to perform more
        efficient masking as only the subset of masks that are in
        pixels containing `targs` at this `nside` will be considered
        (together with neighboring pixels to account for edge effects).
    pixlist : :class:`list` or `int`, optional
        A set of HEALPixels corresponding to the `targs`. Only the subset
        of masks in HEALPixels in `pixlist` at `nside` will be considered
        (together with neighboring pixels to account for edge effects).
        If ``None``, then the pixels touched by `targs` is derived from
        from `targs` itself.
    bricks_are_hpx : :class:`bool`, optional, defaults to ``False``
        Instead of using bricks to calculate BRICKIDs, use HEALPixels at
        the "standard" size from :func:`gaiamatch.get_gaia_nside_brick()`.

    Returns
    -------
    :class:`~numpy.ndarray`
        Input targets with the `DESI_TARGET` column updated to reflect
        the `BRIGHT_OBJECT` bits and SAFE (`BADSKY`) sky locations added
        around the perimeter of the mask.

    Notes
    -----
        - `Tech Note 2346`_ details SAFE (BADSKY) locations.
    """
    t0 = time()

    # ADM Check if targs is a file name or the structure itself.
    if isinstance(targs, str):
        if not os.path.exists(targs):
            raise ValueError("{} doesn't exist".format(targs))
        targs = fitsio.read(targs)

    # ADM determine which pixels are occupied by targets.
    if pixlist is None:
        theta, phi = np.radians(90-targs["DEC"]), np.radians(targs["RA"])
        pixlist = list(set(hp.ang2pix(nside, theta, phi, nest=True)))
    else:
        # ADM in case an integer was passed.
        pixlist = np.atleast_1d(pixlist)
    log.info("Masking using masks in {} at nside={} in HEALPixels={}".format(
        inmaskdir, nside, pixlist))
    pixlistwneigh = add_hp_neighbors(nside, pixlist)

    # ADM read in the (potentially HEALPixel-split) mask.
    sourcemask = io.read_targets_in_hp(inmaskdir, nside, pixlistwneigh)

    ntargs = len(targs)
    log.info('Total number of masks {}'.format(len(sourcemask)))
    log.info('Total number of targets {}...t={:.1f}s'.format(ntargs, time()-t0))

    # ADM update the bits depending on whether targets are in a mask.
    # ADM also grab masks that contain or are near a target.
    dt, mx = set_target_bits(targs, sourcemask, return_masks=True)
    targs["DESI_TARGET"] = dt
    inmasks, nearmasks = mx

    # ADM generate SAFE locations for masks that contain a target.
    safes = get_safe_targets(targs, sourcemask[inmasks],
                             bricks_are_hpx=bricks_are_hpx)

    # ADM update the bits for the safe locations depending on whether
    # ADM they're in a mask.
    safes["DESI_TARGET"] = set_target_bits(safes, sourcemask)
    # ADM it's possible that a safe location was generated outside of
    # ADM the requested HEALPixels.
    inhp = is_in_hp(safes, nside, pixlist)
    safes = safes[inhp]

    # ADM combine the targets and safe locations.
    done = np.concatenate([targs, safes])

    # ADM assert uniqueness of TARGETIDs.
    stargs, ssafes = set(targs["TARGETID"]), set(safes["TARGETID"])
    msg = "TARGETIDs for targets not unique"
    assert len(stargs) == len(targs), msg
    msg = "TARGETIDs for safes not unique"
    assert len(ssafes) == len(safes), msg
    msg = "TARGETIDs for safes duplicated in targets. Generating TARGETIDs"
    msg += " backwards from maxobjid in get_safe_targets() has likely failed"
    msg += " due to somehow generating a large number of safe locations."
    assert len(stargs.intersection(ssafes)) == 0, msg

    log.info('Generated {} SAFE (BADSKY) locations...t={:.1f}s'.format(
        len(done)-ntargs, time()-t0))

    # ADM remove any SAFE locations that are in bright masks (because they aren't really safe).
    ii = (((done["DESI_TARGET"] & desi_mask.BAD_SKY) == 0) |
          ((done["DESI_TARGET"] & desi_mask.IN_BRIGHT_OBJECT) == 0))
    done = done[ii]

    log.info("...of these, {} SAFE (BADSKY) locations aren't in masks...t={:.1f}s"
             .format(len(done)-ntargs, time()-t0))

    log.info('Finishing up...t={:.1f}s'.format(time()-t0))

    return done