def all_gaia_in_tiles(maglim=18,
numproc=4,
allsky=False,
tiles=None,
mindec=-30,
mingalb=10):
"""An array of all Gaia objects in the DESI tiling footprint
Parameters
----------
maglim : :class:`float`, optional, defaults to 18
Magnitude limit for GFAs in Gaia G-band.
numproc : :class:`int`, optional, defaults to 4
The number of parallel processes to use.
allsky : :class:`bool`, defaults to ``False``
If ``True``, assume that the DESI tiling footprint is the
entire sky (i.e. return *all* Gaia objects across the sky).
tiles : :class:`~numpy.ndarray`, optional, defaults to ``None``
Array of DESI tiles. If None, then load the entire footprint.
mindec : :class:`float`, optional, defaults to -30
Minimum declination (o) to include for output Gaia objects.
mingalb : :class:`float`, optional, defaults to 10
Closest latitude to Galactic plane for output Gaia objects
(e.g. send 10 to limit to areas beyond -10o <= b < 10o).
Returns
-------
:class:`~numpy.ndarray`
Gaia objects within the passed geometric constraints brighter
than `maglim`, formatted like `desitarget.gfa.gfadatamodel`.
Notes
-----
- The environment variables $GAIA_DIR and $DESIMODEL must be set.
"""
# ADM grab paths to Gaia files in the sky or the DESI footprint.
if allsky:
infilesbox = find_gaia_files_box([0, 360, mindec, 90])
infilesgalb = find_gaia_files_beyond_gal_b(mingalb)
infiles = list(set(infilesbox).intersection(set(infilesgalb)))
else:
infiles = find_gaia_files_tiles(tiles=tiles, neighbors=False)
nfiles = len(infiles)
# ADM the critical function to run on every file.
def _get_gaia_gfas(fn):
'''wrapper on gaia_in_file() given a file name'''
return gaia_in_file(fn, maglim=maglim, mindec=mindec, mingalb=mingalb)
# ADM this is just to count sweeps files in _update_status.
nfile = np.zeros((), dtype='i8')
t0 = time()
def _update_status(result):
"""wrapper function for the critical reduction operation,
that occurs on the main parallel process"""
if nfile % 1000 == 0 and nfile > 0:
elapsed = (time() - t0) / 60.
rate = nfile / elapsed / 60.
log.info('{}/{} files; {:.1f} files/sec...t = {:.1f} mins'.format(
nfile, nfiles, rate, elapsed))
nfile[...] += 1 # this is an in-place modification.
return result
# - Parallel process Gaia files.
if numproc > 1:
pool = sharedmem.MapReduce(np=numproc)
with pool:
gfas = pool.map(_get_gaia_gfas, infiles, reduce=_update_status)
else:
gfas = list()
for file in infiles:
gfas.append(_update_status(_get_gaia_gfas(file)))
gfas = np.concatenate(gfas)
log.info('Retrieved {} Gaia objects...t = {:.1f} mins'.format(
len(gfas), (time() - t0) / 60.))
return gfas
def all_gaia_in_tiles(maglim=18,
numproc=4,
allsky=False,
tiles=None,
mindec=-30,
mingalb=10,
nside=None,
pixlist=None,
addobjid=False):
"""An array of all Gaia objects in the DESI tiling footprint
Parameters
----------
maglim : :class:`float`, optional, defaults to 18
Magnitude limit for GFAs in Gaia G-band.
numproc : :class:`int`, optional, defaults to 4
The number of parallel processes to use.
allsky : :class:`bool`, defaults to ``False``
If ``True``, assume that the DESI tiling footprint is the
entire sky regardless of the value of `tiles`.
tiles : :class:`~numpy.ndarray`, optional, defaults to ``None``
Array of DESI tiles. If None, then load the entire footprint.
mindec : :class:`float`, optional, defaults to -30
Minimum declination (o) to include for output Gaia objects.
mingalb : :class:`float`, optional, defaults to 10
Closest latitude to Galactic plane for output Gaia objects
(e.g. send 10 to limit to areas beyond -10o <= b < 10o).
nside : :class:`int`, optional, defaults to `None`
(NESTED) HEALPix `nside` to use with `pixlist`.
pixlist : :class:`list` or `int`, optional, defaults to `None`
Only return sources in a set of (NESTED) HEALpixels at the
supplied `nside`.
addobjid : :class:`bool`, optional, defaults to ``False``
If ``True``, include, in the output, a column "GAIA_OBJID"
that is the integer number of each row read from each Gaia file.
Returns
-------
:class:`~numpy.ndarray`
Gaia objects within the passed geometric constraints brighter
than `maglim`, formatted like `desitarget.gfa.gfadatamodel`.
Notes
-----
- The environment variables $GAIA_DIR and $DESIMODEL must be set.
"""
# ADM to guard against no files being found.
if pixlist is None:
dummyfile = find_gaia_files_hp(_get_gaia_nside(), [0],
neighbors=False)[0]
else:
# ADM this is critical for, e.g., unit tests for which the
# ADM Gaia "00000" pixel file might not exist.
dummyfile = find_gaia_files_hp(_get_gaia_nside(),
pixlist[0],
neighbors=False)[0]
dummygfas = np.array([], gaia_in_file(dummyfile).dtype)
# ADM grab paths to Gaia files in the sky or the DESI footprint.
if allsky:
infilesbox = find_gaia_files_box([0, 360, mindec, 90])
infilesgalb = find_gaia_files_beyond_gal_b(mingalb)
infiles = list(set(infilesbox).intersection(set(infilesgalb)))
if pixlist is not None:
infileshp = find_gaia_files_hp(nside, pixlist, neighbors=False)
infiles = list(set(infiles).intersection(set(infileshp)))
else:
infiles = find_gaia_files_tiles(tiles=tiles, neighbors=False)
nfiles = len(infiles)
# ADM the critical function to run on every file.
def _get_gaia_gfas(fn):
'''wrapper on gaia_in_file() given a file name'''
return gaia_in_file(fn,
maglim=maglim,
mindec=mindec,
mingalb=mingalb,
nside=nside,
pixlist=pixlist,
addobjid=addobjid)
# ADM this is just to count sweeps files in _update_status.
nfile = np.zeros((), dtype='i8')
t0 = time()
def _update_status(result):
"""wrapper function for the critical reduction operation,
that occurs on the main parallel process"""
if nfile % 100 == 0 and nfile > 0:
elapsed = (time() - t0) / 60.
rate = nfile / elapsed / 60.
log.info('{}/{} files; {:.1f} files/sec...t = {:.1f} mins'.format(
nfile, nfiles, rate, elapsed))
nfile[...] += 1 # this is an in-place modification.
return result
# - Parallel process Gaia files.
if numproc > 1:
pool = sharedmem.MapReduce(np=numproc)
with pool:
gfas = pool.map(_get_gaia_gfas, infiles, reduce=_update_status)
else:
gfas = list()
for file in infiles:
gfas.append(_update_status(_get_gaia_gfas(file)))
if len(gfas) > 0:
gfas = np.concatenate(gfas)
else:
# ADM if nothing was found, return an empty np array.
gfas = dummygfas
log.info('Retrieved {} Gaia objects...t = {:.1f} mins'.format(
len(gfas), (time() - t0) / 60.))
return gfas