Exemplo n.º 1
0
def add_brick_data(T, north):
    B = fits_table('survey-bricks.fits.gz')
    print('Looking up brick bounds')
    ibrick = dict([(n, i) for i, n in enumerate(B.brickname)])
    bi = np.array([ibrick[n] for n in T.brickname])
    T.brickid = B.brickid[bi]
    T.ra1 = B.ra1[bi]
    T.ra2 = B.ra2[bi]
    T.dec1 = B.dec1[bi]
    T.dec2 = B.dec2[bi]
    assert (np.all(T.ra2 > T.ra1))
    T.area = ((T.ra2 - T.ra1) * (T.dec2 - T.dec1) *
              np.cos(np.deg2rad((T.dec1 + T.dec2) / 2.)))

    print('Resolving north/south split')
    from astrometry.util.starutil_numpy import radectolb
    ll, bb = radectolb(T.ra, T.dec)
    from desitarget.io import desitarget_resolve_dec
    decsplit = desitarget_resolve_dec()
    if north:
        T.survey_primary = (bb > 0) * (T.dec >= decsplit)
    else:
        T.survey_primary = np.logical_not((bb > 0) * (T.dec >= decsplit))

    print('Looking up in_desi')
    from desimodel.io import load_tiles
    from desimodel.footprint import is_point_in_desi
    desitiles = load_tiles()
    T.in_desi = is_point_in_desi(desitiles, T.ra, T.dec)
Exemplo n.º 2
0
def _get_healpixels_in_footprint(nside=64):
    """Obtain a list of HEALPix pixels in the DESI footprint.

    Parameters
    ----------
    nside : int
        HEALPix nside parameter (in form nside=2**k, k=[1,2,3,...]).

    Returns
    -------
    healpixels : ndarray
        List of HEALPix pixels within the DESI footprint.
    """
    from desimodel import footprint
    from desimodel.io import load_tiles

    # Load DESI tiles.
    tile_tab = load_tiles()

    npix = hp.nside2npix(nside)
    pix_ids = np.arange(npix)
    ra, dec = hp.pix2ang(nside, pix_ids, lonlat=True)

    # Get a list of pixel IDs inside the DESI footprint.
    in_desi = footprint.is_point_in_desi(tile_tab, ra, dec)
    healpixels = pix_ids[in_desi]

    return healpixels
def apply_footprint(ra, dec, zz, zz_rsd, nzz):
    """ apply desi footprint """
    tiles = desimodel.io.load_tiles()
    point = foot.is_point_in_desi(tiles, ra, dec)
    idx = np.where(point)

    print("Selected {} out of {} galaxies.".format(len(idx[0]), len(ra)))

    return ra[idx], dec[idx], zz[idx], zz_rsd[idx], nzz[idx]
Exemplo n.º 4
0
def generate_rnd(factor=3, out_path= None, method='random_choice'):
    """
    Routine to generate a random catalog in 3D following
    certain N(z) distribution
    
    Args:
    ----
    rad: z-values of the data catalog
    factor: Size of the generated catalog (before masking)
    out_path: Name of output file where randoms will be saved (default: None)
    method: Method to generate the random catalog (default: 'random_choice')
    """
    #Creating random that follows N(z)
    tiles = load_tiles()
    nz_file = astropy.table.Table.read('example_data/Nz_qso_2_highZ.txt',format='ascii')
    zvec = np.linspace(np.min(nz_file['col1']),np.max(nz_file['col1']),5000)
    spl_z = interp1d(nz_file['col1'],nz_file['col2'])
    dndz = spl_z(zvec)
    ntot = 4*180**2/np.pi*np.sum(nz_file['col2'])*(nz_file['col1'][1]-nz_file['col1'][0])
    ntot = int(factor*ntot)  
    if method=='rnd_choice':
        z_rnd = np.random.choice(zvec[zvec>1.8],p=dndz[zvec>1.8]/np.sum(dndz[zvec>1.8]),size=ntot)+2*(z[1]-z[0])*np.random.normal(size=ntot)
    if method=='cdf':
        cdf = np.cumsum(dndz[zvec>=1.8])/np.sum(dndz[zvec>=1.8])
        icdf = interp1d(cdf,zvec[zvec>=1.8],fill_value='extrapolate',bounds_error=False)
        z_rnd = icdf(np.random.random(size=ntot))
    ra_rnd = 360.*np.random.random(size=len(z_rnd))
    cth_rnd = -1+2.*np.random.random(size=len(z_rnd))
    dec_rnd = np.arcsin(cth_rnd)*180/np.pi
    good = is_point_in_desi(tiles,ra_rnd,dec_rnd)
    ra_rnd = ra_rnd[good]
    dec_rnd = dec_rnd[good]
    z_rnd = z_rnd[good]
    if out_path is not None:
        tab_out = astropy.table.Table([ra_rnd,dec_rnd,z_rnd],names=('RA','DEC','Z'))
        tab_out.write(out_path,overwrite=True)
    return None
Exemplo n.º 5
0
def select_gfas(infiles,
                maglim=18,
                numproc=4,
                tilesfile=None,
                cmx=False,
                mindec=-30,
                mingalb=10,
                addurat=True):
    """Create a set of GFA locations using Gaia and matching to sweeps.

    Parameters
    ----------
    infiles : :class:`list` or `str`
        A list of input filenames (sweep files) OR a single filename.
    maglim : :class:`float`, optional, defaults to 18
        Magnitude limit for GFAs in Gaia G-band.
    numproc : :class:`int`, optional, defaults to 4
        The number of parallel processes to use.
    tilesfile : :class:`str`, optional, defaults to ``None``
        Name of tiles file to load. For full details, see
        :func:`~desimodel.io.load_tiles`.
    cmx : :class:`bool`,  defaults to ``False``
        If ``True``, do not limit output to DESI tiling footprint.
        Used for selecting wider-ranging commissioning targets.
    mindec : :class:`float`, optional, defaults to -30
        Minimum declination (o) for output sources that do NOT match
        an object in the passed `infiles`.
    mingalb : :class:`float`, optional, defaults to 10
        Closest latitude to Galactic plane for output sources that
        do NOT match an object in the passed `infiles` (e.g. send
        10 to limit to regions beyond -10o <= b < 10o)".
    addurat : :class:`bool`, optional, defaults to ``True``
        If ``True`` then substitute proper motions from the URAT
        catalog where Gaia is missing proper motions. Requires that
        the :envvar:`URAT_DIR` is set and points to data downloaded and
        formatted by, e.g., :func:`~desitarget.uratmatch.make_urat_files`.

    Returns
    -------
    :class:`~numpy.ndarray`
        GFA objects from Gaia with the passed geometric constraints
        limited to the passed maglim and matched to the passed input
        files, formatted according to `desitarget.gfa.gfadatamodel`.

    Notes
    -----
        - If numproc==1, use the serial code instead of parallel code.
        - If numproc > 4, then numproc=4 is enforced for (just those)
          parts of the code that are I/O limited.
        - The tiles loaded from `tilesfile` will only be those in DESI.
          So, for custom tilings, set IN_DESI==1 in your tiles file.
    """
    # ADM force to no more than numproc=4 for I/O limited processes.
    numproc4 = numproc
    if numproc4 > 4:
        log.info('Forcing numproc to 4 for I/O limited parts of code')
        numproc4 = 4

    # ADM convert a single file, if passed to a list of files.
    if isinstance(infiles, str):
        infiles = [
            infiles,
        ]
    nfiles = len(infiles)

    # ADM check that files exist before proceeding.
    for filename in infiles:
        if not os.path.exists(filename):
            msg = "{} doesn't exist".format(filename)
            log.critical(msg)
            raise ValueError(msg)

    # ADM load the tiles file.
    tiles = desimodel.io.load_tiles(tilesfile=tilesfile)
    # ADM check some files loaded.
    if len(tiles) == 0:
        msg = "no tiles found in {}".format(tilesfile)
        log.critical(msg)
        raise ValueError(msg)

    # ADM the critical function to run on every file.
    def _get_gfas(fn):
        '''wrapper on gaia_gfas_from_sweep() given a file name'''
        return gaia_gfas_from_sweep(fn, maglim=maglim)

    # ADM this is just to count sweeps files in _update_status.
    t0 = time()
    nfile = np.zeros((), dtype='i8')

    def _update_status(result):
        """wrapper function for the critical reduction operation,
        that occurs on the main parallel process"""
        if nfile % 50 == 0 and nfile > 0:
            elapsed = (time() - t0) / 60.
            rate = nfile / elapsed / 60.
            log.info('{}/{} files; {:.1f} files/sec...t = {:.1f} mins'.format(
                nfile, nfiles, rate, elapsed))
        nfile[...] += 1  # this is an in-place modification.
        return result

    # - Parallel process input files.
    if numproc4 > 1:
        pool = sharedmem.MapReduce(np=numproc4)
        with pool:
            gfas = pool.map(_get_gfas, infiles, reduce=_update_status)
    else:
        gfas = list()
        for file in infiles:
            gfas.append(_update_status(_get_gfas(file)))

    gfas = np.concatenate(gfas)

    # ADM resolve any duplicates between imaging data releases.
    gfas = resolve(gfas)

    # ADM retrieve Gaia objects in the DESI footprint or passed tiles.
    log.info('Retrieving additional Gaia objects...t = {:.1f} mins'.format(
        (time() - t0) / 60))
    gaia = all_gaia_in_tiles(maglim=maglim,
                             numproc=numproc4,
                             allsky=cmx,
                             tiles=tiles,
                             mindec=mindec,
                             mingalb=mingalb)

    # ADM remove any duplicates. Order is important here, as np.unique
    # ADM keeps the first occurence, and we want to retain sweeps
    # ADM information as much as possible.
    gfas = np.concatenate([gfas, gaia])
    _, ind = np.unique(gfas["REF_ID"], return_index=True)
    gfas = gfas[ind]

    # ADM for zero/NaN proper motion objects, add in URAT proper motions.
    if addurat:
        ii = ((np.isnan(gfas["PMRA"]) | (gfas["PMRA"] == 0)) &
              (np.isnan(gfas["PMDEC"]) | (gfas["PMDEC"] == 0)))
        log.info(
            'Adding URAT for {} objects with no PMs...t = {:.1f} mins'.format(
                np.sum(ii), (time() - t0) / 60))
        urat = add_urat_pms(gfas[ii], numproc=numproc)
        log.info(
            'Found an additional {} URAT objects...t = {:.1f} mins'.format(
                np.sum(urat["URAT_ID"] != -1), (time() - t0) / 60))
        for col in "PMRA", "PMDEC", "URAT_ID", "URAT_SEP":
            gfas[col][ii] = urat[col]

    # ADM a final clean-up to remove columns that are NaN (from
    # ADM Gaia-matching) or that are exactly 0 (in the sweeps).
    ii = ((np.isnan(gfas["PMRA"]) | (gfas["PMRA"] == 0)) &
          (np.isnan(gfas["PMDEC"]) | (gfas["PMDEC"] == 0)))
    gfas = gfas[~ii]

    # ADM limit to DESI footprint or passed tiles, if not cmx'ing.
    if not cmx:
        ii = is_point_in_desi(tiles, gfas["RA"], gfas["DEC"])
        gfas = gfas[ii]

    return gfas
Exemplo n.º 6
0
files = glob.glob(
    '/global/cfs/cdirs/desi/spectro/redux/andes/tiles/*/*/cframe-b0-*.fits')
plates = []

for p in files:
    p = fits.open(p)[0]

    plates.append(
        [p.header['TILEID'], p.header['TILERA'], p.header['TILEDEC']])

plates = np.array(plates)

plates = Table(plates, names=['TILEID', 'RA', 'DEC'])

# tiles = load_tiles(extra=True, onlydesi=False)
isin, indx = is_point_in_desi(plates,
                              tsz['RA'],
                              tsz['DEC'],
                              return_tile_index=True)

indx = np.unique(indx).astype(np.int)

plates = plates[indx]

desitsz = tsz[isin]

print(plates)

# andes_tsz = dat[isin]
# andes_tsz.to_csv('andes_tsz.txt')
Exemplo n.º 7
0
dec = dat.iloc[:, 6]

tsz = Table(np.c_[ra, dec], names=['RA', 'DEC'])

##
files = glob.glob(
    '/global/cfs/cdirs/desi/spectro/redux/andes/tiles/*/*/cframe-b0-*.fits')
plates = []

for p in files:
    p = fits.open(p)[0]

    plates.append(
        [p.header['TILEID'], p.header['TILERA'], p.header['TILEDEC']])

plates = np.array(plates)

plates = Table(plates, names=['TILEID', 'RA', 'DEC'])

# tiles    = load_tiles(extra=True, onlydesi=False)
isin = is_point_in_desi(plates, tsz['RA'], tsz['DEC'], return_tile_index=False)

# indx     = np.unique(indx).astype(np.int)
# plates   = plates[indx]

desitsz = tsz[isin]

print(plates)

desitsz.write('andes_tsz.txt', format='ascii', overwrite=True)
Exemplo n.º 8
0
import numpy               as     np
import astropy.io.fits     as     fits
import astropy.units       as     u

from astropy.table         import Table, vstack
from desimodel.footprint   import is_point_in_desi, tiles2pix
from desimodel.io          import load_tiles 
from desitarget.targetmask import desi_mask


gcs            = Table(fits.open('/global/cscratch1/sd/mjwilson/BGS/SV-ASSIGN/masks/NGC-star-clusters.fits')[1].data)

tiles          = np.array([list(x) for x in load_tiles(onlydesi=True)])
tiles          = Table([tiles[:, 0].astype(np.int), tiles[:, 1].astype(np.float32), tiles[:,2].astype(np.float32)], names=['TILEID', 'RA', 'DEC'])

isin, index    = is_point_in_desi(tiles, gcs['ra'], gcs['dec'], return_tile_index=True)

gcs['TILEID']       = -99
gcs['TILEID'][isin] = tiles['TILEID'][index[isin]]

gcs            = gcs[gcs['name'] == 'NGC5904']
gcs['RA']      = gcs['ra']
gcs['DEC']     = gcs['dec']
gcs            = gcs['RA', 'DEC', 'name', 'TILEID']

tiles          = tiles[tiles['TILEID'] == gcs['TILEID']]

gcs.pprint()
tiles.pprint()

pix            = tiles2pix(nside=2, tiles=tiles)
Exemplo n.º 9
0
                                     release=sv_mtl['RELEASE'],
                                     sky=0,
                                     mock=0)
sv_mtl.pprint()

sv_mtl.write(
    scratch +
    '/BGS/SV-ASSIGN/mtls/MTL_ALLBGS_STDFAINT_STDBRIGHT_svresolve.0.31.0_49677629_samePRIORITY.fits',
    format='fits',
    overwrite=True)

##
nside = 64

(indesi, tindex) = is_point_in_desi(tiles,
                                    sv_mtl['RA'].quantity.value,
                                    sv_mtl['DEC'].quantity.value,
                                    return_tile_index=True)

##  Prevent 'duplication' of e.g. RA cols after join.
del tiles['RA']
del tiles['DEC']

##  Some objects in the .mtl are not in the tiles;  rough cut on creation I guess.
sv_mtl['TILEID'] = -99 * np.ones_like(sv_mtl['RA'].quantity.value,
                                      dtype=np.int)
sv_mtl['TILEID'][indesi] = tiles['TILEID'].quantity[tindex[indesi]]
sv_mtl.sort('TILEID')

##  Not associated to a tile?
##  notin = sv_mtl[sv_mtl['TILEID'] < 0]
##  notin.sort('DEC')
Exemplo n.º 10
0
 def QSO_filter(RA, DEC):
     return is_point_in_desi(tiles, RA, DEC)
Exemplo n.º 11
0
##  ['tycho', 'gaia']     
survey     = 'gaia'

##
scratch    = os.environ['CSCRATCH']

gaia       = fits.open(scratch + '/BGS/SV-ASSIGN/masks/{}_stellar_mask.fits'.format(survey))
gaia       = Table(gaia[1].data)

print(gaia)

##
nside      = 64

(indesi, tindex)          = is_point_in_desi(tiles, gaia['RA'], gaia['DEC'], return_tile_index=True)

##  Prevent 'duplication' of e.g. RA cols after join. 
del tiles['RA']
del tiles['DEC']

##  Some objects in the .mtl are not in the tiles;  rough cut on creation I guess. 
gaia['TILEID']          = -99 * np.ones_like(gaia['RA'].quantity.value, dtype=np.int)
gaia['TILEID'][indesi]  = tiles['TILEID'].quantity[tindex[indesi]]
gaia.sort('TILEID')

print(gaia)

##  Only keep the targets in a given tile. 
gaia     = gaia[gaia['TILEID'] > 0]
gaia     = join(gaia, tiles, keys=['TILEID'], join_type='left')
Exemplo n.º 12
0
def lsscat_gen(root, prod, odir):
    '''
    # Get required parent pipeline files. 
    _mtl            = root + 'targets/mtl.fits'

    # Needed for fiberassign runs. 
    # _skies        = '/project/projectdirs/desi/target/catalogs/dr8/0.31.0/skies/skies-dr8-0.31.0.fits'

    # targets       = Table(fits.open(root + 'targets/targets.fits')[1].data)
    
    mtl             = Table(fits.open(root + 'targets/mtl.fits')[1].data)
    # exps          = Table(fits.open(root + 'survey/sv_exposures.fits')[1].data)

    # HACK:  Cumulative SNR2 achieved on a given tile by this exposure. Weird that this was missing?                                                                                                               
    # exps['SNR2']  = np.random.uniform(0.9, 1.1, len(exps))
    
    _tiles          = fits.open(root + 'survey/SV-tiles.fits')[1]
    tiles           = Table(fits.open(root + 'survey/SV-tiles.fits')[1].data)

    # zcat          = Table(fits.open(root + 'spectro/redux/{}/zcatalog-{}.fits'.format(prod, prod))[1].data)
    
    # DR8 randoms. 
    rows          = np.arange(len(mtl))
    randoms       = fitsio.read('/project/projectdirs/desi/target/catalogs/dr8/0.31.0/randoms/randoms-inside-dr8-0.31.0-4.fits', rows=rows)

    # A list of targets that could have been assigned to each fiber;
    # desidatamodel.readthedocs.io/en/latest/DESI_TARGET/fiberassign/tile-TILEID-FIELDNUM.html#hdu2

    rand_assignable, rand_assigned = get_randassign(root + 'fiberassign/tile-072015.fits', mtl=randoms, gen=True, randoms=randoms)
    '''
    rand_assignable, rand_assigned = get_randassign(root + 'fiberassign/tile-072015.fits', mtl=None, gen=True, randoms=None, lite=True)
    
    exit(0)
    
    # TARGETID | FIBERID | LOCATION
    # print(rand_assignable)
    
    # Date stamped tiles meeting quality requirements.  
    MJD0            = 58853.51129915193
    MJD1            = 58881.40164263005
    
    dst_tiles, hdr           = datestamped_tiles(exps, MJD0, MJD1, printit=False)
    
    # Write here (should add header).  Assign tile assignment completeness below. 
    dst_tiles.write(odir + 'dst_tiles.fits', format='fits', overwrite=True)
    
    '''
    # Sort Targets by TARGETID
    zcat.sort('TARGETID')
    targets.sort('TARGETID')

    # Unique TARGETID lists.
    assert  np.all(np.unique(zcat['TARGETID'])    == zcat['TARGETID'])
    assert  np.all(np.unique(targets['TARGETID']) == targets['TARGETID'])
    '''
    # Catalogue generation.
    ntarget                        = len(targets)
    
    final                          = lss_catalog(ntarget)

    final['TARGETID']              = targets['TARGETID']
    
    final['RA']                    = targets['RA']
    final['DEC']                   = targets['DEC']

    final['IN_IMAGING']            = 1.0

    ##  Neglects area lost to e.g. GFA.
    final['IN_DESI']               = is_point_in_desi(tiles, final['RA'], final['DEC']).astype(np.int)

    final['DESI_TARGET']           = targets['SV1_DESI_TARGET']
    final['BGS_TARGET']            = targets['SV1_BGS_TARGET']

    final['PRIORITY_INIT']         = targets['PRIORITY_INIT']
    final['SUBPRIORITY']           = targets['SUBPRIORITY']

    in_zcat                        = np.in1d( zcat['TARGETID'], final['TARGETID'])
    in_final                       = np.in1d(final['TARGETID'],  zcat['TARGETID'])
    
    final['Z'][in_final]           = zcat['Z'][in_zcat]
    final['ZERR'][in_final]        = zcat['ZERR'][in_zcat]
    final['ZWARN'][in_final]       = zcat['ZWARN'][in_zcat]
    final['SPECTYPE'][in_final]    = zcat['SPECTYPE'][in_zcat]

    ##  Tag all galaxies that appeared in the redshift cats., even if bad etc. 
    final['Z_VETO_FLAG']           = 0

    final                          = pop_assigned(final, printit=True)

    final                          = set_ang_veto(final)          
    
    final                          = set_z_veto(final)
    
    final                          = blind_distances(final)
        
    final                          = in_cosmo(final)
    
    final                          = final[final['Z'] > 0.]
    
    final.write(odir + 'lsscat_v1.fits', format='fits', overwrite=True)

    final.pprint(max_width=-1)
    
    # Add tile assignment completeness based on reachable fibers from final. 
    dst_tiles                      = set_assigncomplete(dst_tiles, final)
Exemplo n.º 13
0
import fitsio
import desimodel
import numpy as np
import astropy.io.fits as fits

from   astropy.table import Table, unique
from   desisurvey.utils import get_date
from   desimodel.footprint import is_point_in_desi

##
tfiles  = '/global/homes/m/mjwilson/desi/survey-validation/svdc-spring2020e-onepercent/run/survey/tiles/des.fits'
tiles   = desimodel.io.load_tiles(tilesfile=tfiles)

# ids   = complete['TILEID']
# isin  = np.isin(tiles['TILEID'], ids)

skies   = fitsio.read('/global/cfs/cdirs/desi/target/catalogs/dr8/0.32.0/skies/skies-dr8-0.32.0.fits')

# Cut to observed tiles.                                                                                                                                                                                                       
isin    = is_point_in_desi(tiles, skies['RA'], skies['DEC'])

skies   = skies[isin]

fitsio.write('/global/cscratch1/sd/mjwilson/svdc-spring2020e-onepercent/targets/skies.fits', skies)
Exemplo n.º 14
0
def make_mock_sweeps(config_file,
                     source_name,
                     input_dir,
                     epoch_dir,
                     map_id_file_path,
                     sweep_mock_root,
                     override_root=None,
                     dry_run=True,
                     match_on_objid=False):
    """
    """
    if match_on_objid:
        print(
            'NOTE: Matching on mock column "objid", only makes sense for 2016 data challenge outputs!'
        )

    # Load the tile definitions
    TILES = desimodel.io.load_tiles()

    epoch = int(os.path.split(epoch_dir.strip(os.path.sep))[-1])
    print('Epoch: {}'.format(epoch))

    # Read parameters for quicksurvey
    with open(config_file, 'r') as pfile:
        params = yaml.load(pfile)

    # Original mock path.
    root_mock_dir = params['sources'][source_name]['root_mock_dir']

    if override_root is not None:
        original_root_mock_dir = root_mock_dir
        root_mock_dir = override_root

    # Read zcat and truth, and construct the mapping between them
    zcat, truth_table, target_table, itruth_for_izcat = match_zcat_truth(
        input_dir, epoch_dir)

    # Read the table of mock paths for this source
    map_id_file = np.loadtxt(map_id_file_path,
                             dtype={
                                 'names': ('SOURCENAME', 'FILEID', 'FILENAME'),
                                 'formats': ('S10', 'i4', 'S256')
                             })

    # Filter truth by source
    truth_this_source = truth_table['SOURCETYPE'] == source_name
    # Decode rowid and fileid for all targets associated with this source
    rowid, fileid = decode_rownum_filenum(
        truth_table['MOCKID'][truth_this_source])

    # Store the row indices for this source, will save them later
    truth_rows_for_source = np.where(truth_this_source)[0]

    # Filter truth by source for sources in zcat
    obs_this_source = truth_table['SOURCETYPE'][
        itruth_for_izcat] == source_name
    # Decode rowid and fileid for observed targets associated with this source
    obs_rowid, obs_fileid = decode_rownum_filenum(
        truth_table['MOCKID'][itruth_for_izcat][obs_this_source])

    # Determine if target/truth rows for this source are in footprint
    print('Testing truth coordinates against DESIMODEL v %s' %
          (desimodel.__version__))
    footprint_flag_targets_truth = footprint.is_point_in_desi(
        TILES, truth_table['RA'][truth_rows_for_source],
        truth_table['DEC'][truth_rows_for_source])

    # Read original files
    fileid_to_read = np.array(list(set(fileid)))

    # Different mocks have different names for sky coordinate columns
    # Auto-detect this based on a list of possibilities.
    ra_column, dec_column = None, None
    # Names to try in order of priority
    ra_names = ['RA', 'ra', '_RA', '_ra']
    dec_names = ['DEC', 'dec', 'DE', '_DEC', '_dec', '_DE']

    for ifile in fileid_to_read:

        # Filemap entry for this file ID
        row_in_map = (map_id_file['SOURCENAME'] == source_name.encode()) & (
            map_id_file['FILEID'] == ifile)
        filename = map_id_file['FILENAME'][row_in_map]
        filename = filename[0].decode()

        if override_root:
            filename = filename.replace(original_root_mock_dir, root_mock_dir)

        print('Reading mock file: {}'.format(filename))
        sys.stdout.flush()

        if source_name in ['MWS_MAIN', 'MWS_WD', 'MWS_NEARBY']:
            n_this_mock_file = fits.getheader(filename, 1)['NAXIS2']
        elif source_name in ['BGS']:
            with h5py.File(filename) as f:
                n_this_mock_file = f['/Header/number_galaxies'][...][0]
        else:
            raise Exception('Unrecognized source: {}'.format(source))

        print('Mock file nrows: {}'.format(n_this_mock_file))

        # This will be of the length of the total number of targets in this
        # mock file. Not all of these will be selected, and not all of these
        # will be observed.
        selected_as_target = np.repeat(False, n_this_mock_file)
        observed_this_epoch = np.repeat(False, n_this_mock_file)

        # Read the mock data for this file ID
        if source_name in ['MWS_MAIN', 'MWS_WD', 'MWS_NEARBY']:
            data = fits.getdata(filename, 1)
        elif source_name in ['BGS']:
            data = Table()
            with h5py.File(filename) as f:
                for recname in f['/Data']:
                    d = f['/Data'][recname][...]
                    data.add_column(Column(name=recname, data=d))
        else:
            raise Exception('Unrecognized source: {}'.format(source))

        # Set column names if not already set
        if ra_column is None:
            for ra_name in ra_names:
                if ra_name in data.dtype.names:
                    ra_column = ra_name
                    break
            if ra_column is None:
                raise Exception('No RA column found in mock data columns')

        if dec_column is None:
            for dec_name in dec_names:
                if dec_name in data.dtype.names:
                    dec_column = dec_name
                    break
            if dec_column is None:
                raise Exception('No DEC column found in mock data columns')

        # Flag mock rows in footprint
        print('Testing mock coordinates against DESIMODEL v %s' %
              (desimodel.__version__))
        footprint_flag = footprint.is_point_in_desi(TILES, data[ra_column],
                                                    data[dec_column])

        # Select truth rows for this file ID
        fileid_mask = fileid == ifile
        obs_fileid_mask = obs_fileid == ifile

        if match_on_objid:
            # In the Dec 2016 data challenge, the 'row number' is actually the
            # Galaxia mock objid for MWS_MAIN.
            objid = data['objid']

            objid_this_file_all = rowid[fileid_mask]
            objid_this_file_obs = obs_rowid[obs_fileid_mask]

            m = match(objid_this_file_all, objid)
            rows_this_file_all = m[np.where(m >= 0)[0]]

            m = match(objid_this_file_obs, objid)
            rows_this_file_obs = m[np.where(m >= 0)[0]]
        else:
            rows_this_file_all = rowid[fileid_mask]
            rows_this_file_obs = obs_rowid[obs_fileid_mask]

        print('  -- N in truth =  {}'.format(len(rows_this_file_all)))
        print('  -- N observed =  {}'.format(len(rows_this_file_obs)))

        selected_as_target[rows_this_file_all] = True
        observed_this_epoch[rows_this_file_obs] = True

        assert (not np.any((observed_this_epoch) & (~selected_as_target)))

        # Targets that could have been observed this epoch, but were not.
        selected_not_observed = np.where((selected_as_target)
                                         & (~observed_this_epoch))[0]

        # Sort rows in this file by row number. Can use this to reorder subsets
        # of targets and truth, if we want to output them.
        rows_this_file_all_sort = np.argsort(rows_this_file_all)

        # Write various outputs
        filename_path, filename_file = os.path.split(filename)
        base, original_ext = filename_file.split(os.path.extsep)
        ext = 'fits'  # Output fits, even if input is hdf5

        new_filename_status = base + os.path.extsep + 'status' + os.path.extsep + ext
        new_filename_observed = base + os.path.extsep + 'observed' + os.path.extsep + ext
        new_filename_unobserved = base + os.path.extsep + 'unobserved' + os.path.extsep + ext
        new_filename_targets_subset = base + os.path.extsep + 'targets' + os.path.extsep + ext
        new_filename_truth_subset = base + os.path.extsep + 'truth' + os.path.extsep + ext

        if override_root:
            filename_path = filename_path.replace(original_root_mock_dir,
                                                  root_mock_dir)

        if filename_path.startswith(os.path.sep):
            filename_path = os.path.curdir + filename_path

        new_dir = os.path.normpath(
            os.path.join(sweep_mock_root, filename_path, str(epoch)))
        if not os.path.exists(new_dir): os.makedirs(new_dir)

        new_path_status = os.path.normpath(
            os.path.join(sweep_mock_root, filename_path, str(epoch),
                         new_filename_status))
        new_path_observed = os.path.normpath(
            os.path.join(sweep_mock_root, filename_path, str(epoch),
                         new_filename_observed))
        new_path_unobserved = os.path.normpath(
            os.path.join(sweep_mock_root, filename_path, str(epoch),
                         new_filename_unobserved))
        new_path_targets_subset = os.path.normpath(
            os.path.join(sweep_mock_root, filename_path, str(epoch),
                         new_filename_targets_subset))
        new_path_truth_subset = os.path.normpath(
            os.path.join(sweep_mock_root, filename_path, str(epoch),
                         new_filename_truth_subset))

        # Write the status table, which has the same number of rows as the
        # origianl mock file (hence generally more than the sweep).
        t = Table((selected_as_target, observed_this_epoch),
                  names=('SELECTED', 'OBSERVED'))
        print('Writing {}'.format(new_path_status))
        t.write(new_path_status, overwrite=True)

        # Provided that at least one target in this mock file has been
        # selected, write subsets of the mock file.

        # Previous had rows_this_file_obs here
        if len(rows_this_file_all) > 0:

            # For each row in mock that appears in targets/truth, get the
            # corresponding row number in the target/truth.
            input_target_row = np.zeros(n_this_mock_file, dtype=np.int32) - 1
            # ... These are the t/t rows that are in this mock file.
            w_fileid_mask = np.where(fileid_mask)[0]
            # Store row
            input_target_row[rows_this_file_all] = truth_rows_for_source[
                w_fileid_mask]
            assert (np.all(input_target_row[observed_this_epoch] >= 0))

            # 1. Make the mock sweep for observed targets
            t = Table(data[observed_this_epoch])
            t.add_column(
                Column(input_target_row[observed_this_epoch],
                       name='TARGETROW'))
            t.add_column(
                Column(target_table[input_target_row[observed_this_epoch]]
                       ['TARGETID'],
                       name='TARGETID'))
            # Include flag for targets in actual footprint
            t.add_column(
                Column(footprint_flag[observed_this_epoch],
                       name='IN_FOOTPRINT'))

            # Strict check
            assert (np.allclose(t[ra_column],
                                target_table[t['TARGETROW']]['RA'],
                                atol=1e-5))
            t.write(new_path_observed, overwrite=True)
            print('Wrote {}'.format(new_path_observed))

            # 2. Make the mock sweep for unobserved targets
            t = Table(data[selected_not_observed])
            t.add_column(
                Column(input_target_row[selected_not_observed],
                       name='TARGETROW'))
            t.add_column(
                Column(target_table[input_target_row[selected_not_observed]]
                       ['TARGETID'],
                       name='TARGETID'))
            # Include flag for targets in actual footprint
            t.add_column(
                Column(footprint_flag[selected_not_observed],
                       name='IN_FOOTPRINT'))

            # Strict check
            assert (np.allclose(t[ra_column],
                                target_table[t['TARGETROW']]['RA'],
                                atol=1e-5))
            t.write(new_path_unobserved, overwrite=True)
            print('Wrote {}'.format(new_path_unobserved))

            # 3. Make a file extracted from targets for this mock brick
            # Save in rowid order
            subset_this_file = truth_rows_for_source[fileid_mask]
            t = Table(target_table[subset_this_file])[rows_this_file_all_sort]
            t.add_column(
                Column((footprint_flag_targets_truth[fileid_mask]
                        )[rows_this_file_all_sort],
                       name='IN_FOOTPRINT'))
            t.write(new_path_targets_subset, overwrite=True)
            print('Wrote {}'.format(new_path_targets_subset))

            # 4. Make a file extracted from truth for this mock brick
            subset_this_file = truth_rows_for_source[fileid_mask]
            t = Table(truth_table[subset_this_file])[rows_this_file_all_sort]
            t.add_column(
                Column((footprint_flag_targets_truth[fileid_mask]
                        )[rows_this_file_all_sort],
                       name='IN_FOOTPRINT'))
            t.write(new_path_truth_subset, overwrite=True)
            print('Wrote {}'.format(new_path_truth_subset))
    return
Exemplo n.º 15
0
def select_gfas(infiles, maglim=18, numproc=4, tilesfile=None, cmx=False):
    """Create a set of GFA locations using Gaia.

    Parameters
    ----------
    infiles : :class:`list` or `str`
        A list of input filenames (sweep files) OR a single filename.
    maglim : :class:`float`, optional, defaults to 18
        Magnitude limit for GFAs in Gaia G-band.
    numproc : :class:`int`, optional, defaults to 4
        The number of parallel processes to use.
    tilesfile : :class:`str`, optional, defaults to ``None``
        Name of tiles file to load. For full details, see
        :func:`~desimodel.io.load_tiles`.
    cmx : :class:`bool`,  defaults to ``False``
        If ``True``, do not limit output to DESI tiling footprint.
        Used for selecting wider-ranging commissioning targets.

    Returns
    -------
    :class:`~numpy.ndarray`
        GFA objects from Gaia across all of the passed input files, formatted
        according to `desitarget.gfa.gfadatamodel`.

    Notes
    -----
        - If numproc==1, use the serial code instead of the parallel code.
        - The tiles loaded from `tilesfile` will only be those in DESI.
          So, for custom tilings, set IN_DESI==1 in your tiles file.
    """
    # ADM convert a single file, if passed to a list of files.
    if isinstance(infiles, str):
        infiles = [infiles, ]
    nfiles = len(infiles)

    # ADM check that files exist before proceeding.
    for filename in infiles:
        if not os.path.exists(filename):
            msg = "{} doesn't exist".format(filename)
            log.critical(msg)
            raise ValueError(msg)

    # ADM load the tiles file.
    tiles = desimodel.io.load_tiles(tilesfile=tilesfile)
    # ADM check some files loaded.
    if len(tiles) == 0:
        msg = "no tiles found in {}".format(tilesfile)
        log.critical(msg)
        raise ValueError(msg)

    # ADM the critical function to run on every file.
    def _get_gfas(fn):
        '''wrapper on gaia_gfas_from_sweep() given a file name'''
        return gaia_gfas_from_sweep(fn, maglim=maglim)

    # ADM this is just to count sweeps files in _update_status.
    nfile = np.zeros((), dtype='i8')
    t0 = time()

    def _update_status(result):
        """wrapper function for the critical reduction operation,
        that occurs on the main parallel process"""
        if nfile % 50 == 0 and nfile > 0:
            elapsed = (time()-t0)/60.
            rate = nfile/elapsed/60.
            log.info('{}/{} files; {:.1f} files/sec...t = {:.1f} mins'
                     .format(nfile, nfiles, rate, elapsed))
        nfile[...] += 1    # this is an in-place modification.
        return result

    # - Parallel process input files.
    if numproc > 1:
        pool = sharedmem.MapReduce(np=numproc)
        with pool:
            gfas = pool.map(_get_gfas, infiles, reduce=_update_status)
    else:
        gfas = list()
        for file in infiles:
            gfas.append(_update_status(_get_gfas(file)))

    gfas = np.concatenate(gfas)

    # ADM resolve any duplicates between imaging data releases.
    gfas = resolve(gfas)

    # ADM retrieve Gaia objects in the DESI footprint or passed tiles.
    log.info('Retrieving additional Gaia objects...t = {:.1f} mins'
             .format((time()-t0)/60))
    gaia = all_gaia_in_tiles(maglim=maglim, numproc=numproc, allsky=cmx,
                             tiles=tiles)
    # ADM and limit them to just any missing bricks...
    brickids = set(gfas['BRICKID'])
    ii = [gbrickid not in brickids for gbrickid in gaia["BRICKID"]]
    gaia = gaia[ii]

    gfas = np.concatenate([gfas, gaia])
    # ADM limit to DESI footprint or passed tiles, if not cmx'ing.
    if not cmx:
        ii = is_point_in_desi(tiles, gfas["RA"], gfas["DEC"])
        gfas = gfas[ii]

    return gfas