def add_brick_data(T, north):
B = fits_table('survey-bricks.fits.gz')
print('Looking up brick bounds')
ibrick = dict([(n, i) for i, n in enumerate(B.brickname)])
bi = np.array([ibrick[n] for n in T.brickname])
T.brickid = B.brickid[bi]
T.ra1 = B.ra1[bi]
T.ra2 = B.ra2[bi]
T.dec1 = B.dec1[bi]
T.dec2 = B.dec2[bi]
assert (np.all(T.ra2 > T.ra1))
T.area = ((T.ra2 - T.ra1) * (T.dec2 - T.dec1) *
np.cos(np.deg2rad((T.dec1 + T.dec2) / 2.)))
print('Resolving north/south split')
from astrometry.util.starutil_numpy import radectolb
ll, bb = radectolb(T.ra, T.dec)
from desitarget.io import desitarget_resolve_dec
decsplit = desitarget_resolve_dec()
if north:
T.survey_primary = (bb > 0) * (T.dec >= decsplit)
else:
T.survey_primary = np.logical_not((bb > 0) * (T.dec >= decsplit))
print('Looking up in_desi')
from desimodel.io import load_tiles
from desimodel.footprint import is_point_in_desi
desitiles = load_tiles()
T.in_desi = is_point_in_desi(desitiles, T.ra, T.dec)
def _get_healpixels_in_footprint(nside=64):
"""Obtain a list of HEALPix pixels in the DESI footprint.
Parameters
----------
nside : int
HEALPix nside parameter (in form nside=2**k, k=[1,2,3,...]).
Returns
-------
healpixels : ndarray
List of HEALPix pixels within the DESI footprint.
"""
from desimodel import footprint
from desimodel.io import load_tiles
# Load DESI tiles.
tile_tab = load_tiles()
npix = hp.nside2npix(nside)
pix_ids = np.arange(npix)
ra, dec = hp.pix2ang(nside, pix_ids, lonlat=True)
# Get a list of pixel IDs inside the DESI footprint.
in_desi = footprint.is_point_in_desi(tile_tab, ra, dec)
healpixels = pix_ids[in_desi]
return healpixels
def apply_footprint(ra, dec, zz, zz_rsd, nzz):
""" apply desi footprint """
tiles = desimodel.io.load_tiles()
point = foot.is_point_in_desi(tiles, ra, dec)
idx = np.where(point)
print("Selected {} out of {} galaxies.".format(len(idx[0]), len(ra)))
return ra[idx], dec[idx], zz[idx], zz_rsd[idx], nzz[idx]
def generate_rnd(factor=3, out_path= None, method='random_choice'):
"""
Routine to generate a random catalog in 3D following
certain N(z) distribution
Args:
----
rad: z-values of the data catalog
factor: Size of the generated catalog (before masking)
out_path: Name of output file where randoms will be saved (default: None)
method: Method to generate the random catalog (default: 'random_choice')
"""
#Creating random that follows N(z)
tiles = load_tiles()
nz_file = astropy.table.Table.read('example_data/Nz_qso_2_highZ.txt',format='ascii')
zvec = np.linspace(np.min(nz_file['col1']),np.max(nz_file['col1']),5000)
spl_z = interp1d(nz_file['col1'],nz_file['col2'])
dndz = spl_z(zvec)
ntot = 4*180**2/np.pi*np.sum(nz_file['col2'])*(nz_file['col1'][1]-nz_file['col1'][0])
ntot = int(factor*ntot)
if method=='rnd_choice':
z_rnd = np.random.choice(zvec[zvec>1.8],p=dndz[zvec>1.8]/np.sum(dndz[zvec>1.8]),size=ntot)+2*(z[1]-z[0])*np.random.normal(size=ntot)
if method=='cdf':
cdf = np.cumsum(dndz[zvec>=1.8])/np.sum(dndz[zvec>=1.8])
icdf = interp1d(cdf,zvec[zvec>=1.8],fill_value='extrapolate',bounds_error=False)
z_rnd = icdf(np.random.random(size=ntot))
ra_rnd = 360.*np.random.random(size=len(z_rnd))
cth_rnd = -1+2.*np.random.random(size=len(z_rnd))
dec_rnd = np.arcsin(cth_rnd)*180/np.pi
good = is_point_in_desi(tiles,ra_rnd,dec_rnd)
ra_rnd = ra_rnd[good]
dec_rnd = dec_rnd[good]
z_rnd = z_rnd[good]
if out_path is not None:
tab_out = astropy.table.Table([ra_rnd,dec_rnd,z_rnd],names=('RA','DEC','Z'))
tab_out.write(out_path,overwrite=True)
return None
def select_gfas(infiles,
maglim=18,
numproc=4,
tilesfile=None,
cmx=False,
mindec=-30,
mingalb=10,
addurat=True):
"""Create a set of GFA locations using Gaia and matching to sweeps.
Parameters
----------
infiles : :class:`list` or `str`
A list of input filenames (sweep files) OR a single filename.
maglim : :class:`float`, optional, defaults to 18
Magnitude limit for GFAs in Gaia G-band.
numproc : :class:`int`, optional, defaults to 4
The number of parallel processes to use.
tilesfile : :class:`str`, optional, defaults to ``None``
Name of tiles file to load. For full details, see
:func:`~desimodel.io.load_tiles`.
cmx : :class:`bool`, defaults to ``False``
If ``True``, do not limit output to DESI tiling footprint.
Used for selecting wider-ranging commissioning targets.
mindec : :class:`float`, optional, defaults to -30
Minimum declination (o) for output sources that do NOT match
an object in the passed `infiles`.
mingalb : :class:`float`, optional, defaults to 10
Closest latitude to Galactic plane for output sources that
do NOT match an object in the passed `infiles` (e.g. send
10 to limit to regions beyond -10o <= b < 10o)".
addurat : :class:`bool`, optional, defaults to ``True``
If ``True`` then substitute proper motions from the URAT
catalog where Gaia is missing proper motions. Requires that
the :envvar:`URAT_DIR` is set and points to data downloaded and
formatted by, e.g., :func:`~desitarget.uratmatch.make_urat_files`.
Returns
-------
:class:`~numpy.ndarray`
GFA objects from Gaia with the passed geometric constraints
limited to the passed maglim and matched to the passed input
files, formatted according to `desitarget.gfa.gfadatamodel`.
Notes
-----
- If numproc==1, use the serial code instead of parallel code.
- If numproc > 4, then numproc=4 is enforced for (just those)
parts of the code that are I/O limited.
- The tiles loaded from `tilesfile` will only be those in DESI.
So, for custom tilings, set IN_DESI==1 in your tiles file.
"""
# ADM force to no more than numproc=4 for I/O limited processes.
numproc4 = numproc
if numproc4 > 4:
log.info('Forcing numproc to 4 for I/O limited parts of code')
numproc4 = 4
# ADM convert a single file, if passed to a list of files.
if isinstance(infiles, str):
infiles = [
infiles,
]
nfiles = len(infiles)
# ADM check that files exist before proceeding.
for filename in infiles:
if not os.path.exists(filename):
msg = "{} doesn't exist".format(filename)
log.critical(msg)
raise ValueError(msg)
# ADM load the tiles file.
tiles = desimodel.io.load_tiles(tilesfile=tilesfile)
# ADM check some files loaded.
if len(tiles) == 0:
msg = "no tiles found in {}".format(tilesfile)
log.critical(msg)
raise ValueError(msg)
# ADM the critical function to run on every file.
def _get_gfas(fn):
'''wrapper on gaia_gfas_from_sweep() given a file name'''
return gaia_gfas_from_sweep(fn, maglim=maglim)
# ADM this is just to count sweeps files in _update_status.
t0 = time()
nfile = np.zeros((), dtype='i8')
def _update_status(result):
"""wrapper function for the critical reduction operation,
that occurs on the main parallel process"""
if nfile % 50 == 0 and nfile > 0:
elapsed = (time() - t0) / 60.
rate = nfile / elapsed / 60.
log.info('{}/{} files; {:.1f} files/sec...t = {:.1f} mins'.format(
nfile, nfiles, rate, elapsed))
nfile[...] += 1 # this is an in-place modification.
return result
# - Parallel process input files.
if numproc4 > 1:
pool = sharedmem.MapReduce(np=numproc4)
with pool:
gfas = pool.map(_get_gfas, infiles, reduce=_update_status)
else:
gfas = list()
for file in infiles:
gfas.append(_update_status(_get_gfas(file)))
gfas = np.concatenate(gfas)
# ADM resolve any duplicates between imaging data releases.
gfas = resolve(gfas)
# ADM retrieve Gaia objects in the DESI footprint or passed tiles.
log.info('Retrieving additional Gaia objects...t = {:.1f} mins'.format(
(time() - t0) / 60))
gaia = all_gaia_in_tiles(maglim=maglim,
numproc=numproc4,
allsky=cmx,
tiles=tiles,
mindec=mindec,
mingalb=mingalb)
# ADM remove any duplicates. Order is important here, as np.unique
# ADM keeps the first occurence, and we want to retain sweeps
# ADM information as much as possible.
gfas = np.concatenate([gfas, gaia])
_, ind = np.unique(gfas["REF_ID"], return_index=True)
gfas = gfas[ind]
# ADM for zero/NaN proper motion objects, add in URAT proper motions.
if addurat:
ii = ((np.isnan(gfas["PMRA"]) | (gfas["PMRA"] == 0)) &
(np.isnan(gfas["PMDEC"]) | (gfas["PMDEC"] == 0)))
log.info(
'Adding URAT for {} objects with no PMs...t = {:.1f} mins'.format(
np.sum(ii), (time() - t0) / 60))
urat = add_urat_pms(gfas[ii], numproc=numproc)
log.info(
'Found an additional {} URAT objects...t = {:.1f} mins'.format(
np.sum(urat["URAT_ID"] != -1), (time() - t0) / 60))
for col in "PMRA", "PMDEC", "URAT_ID", "URAT_SEP":
gfas[col][ii] = urat[col]
# ADM a final clean-up to remove columns that are NaN (from
# ADM Gaia-matching) or that are exactly 0 (in the sweeps).
ii = ((np.isnan(gfas["PMRA"]) | (gfas["PMRA"] == 0)) &
(np.isnan(gfas["PMDEC"]) | (gfas["PMDEC"] == 0)))
gfas = gfas[~ii]
# ADM limit to DESI footprint or passed tiles, if not cmx'ing.
if not cmx:
ii = is_point_in_desi(tiles, gfas["RA"], gfas["DEC"])
gfas = gfas[ii]
return gfas
files = glob.glob(
'/global/cfs/cdirs/desi/spectro/redux/andes/tiles/*/*/cframe-b0-*.fits')
plates = []
for p in files:
p = fits.open(p)[0]
plates.append(
[p.header['TILEID'], p.header['TILERA'], p.header['TILEDEC']])
plates = np.array(plates)
plates = Table(plates, names=['TILEID', 'RA', 'DEC'])
# tiles = load_tiles(extra=True, onlydesi=False)
isin, indx = is_point_in_desi(plates,
tsz['RA'],
tsz['DEC'],
return_tile_index=True)
indx = np.unique(indx).astype(np.int)
plates = plates[indx]
desitsz = tsz[isin]
print(plates)
# andes_tsz = dat[isin]
# andes_tsz.to_csv('andes_tsz.txt')
dec = dat.iloc[:, 6]
tsz = Table(np.c_[ra, dec], names=['RA', 'DEC'])
##
files = glob.glob(
'/global/cfs/cdirs/desi/spectro/redux/andes/tiles/*/*/cframe-b0-*.fits')
plates = []
for p in files:
p = fits.open(p)[0]
plates.append(
[p.header['TILEID'], p.header['TILERA'], p.header['TILEDEC']])
plates = np.array(plates)
plates = Table(plates, names=['TILEID', 'RA', 'DEC'])
# tiles = load_tiles(extra=True, onlydesi=False)
isin = is_point_in_desi(plates, tsz['RA'], tsz['DEC'], return_tile_index=False)
# indx = np.unique(indx).astype(np.int)
# plates = plates[indx]
desitsz = tsz[isin]
print(plates)
desitsz.write('andes_tsz.txt', format='ascii', overwrite=True)
import numpy as np
import astropy.io.fits as fits
import astropy.units as u
from astropy.table import Table, vstack
from desimodel.footprint import is_point_in_desi, tiles2pix
from desimodel.io import load_tiles
from desitarget.targetmask import desi_mask
gcs = Table(fits.open('/global/cscratch1/sd/mjwilson/BGS/SV-ASSIGN/masks/NGC-star-clusters.fits')[1].data)
tiles = np.array([list(x) for x in load_tiles(onlydesi=True)])
tiles = Table([tiles[:, 0].astype(np.int), tiles[:, 1].astype(np.float32), tiles[:,2].astype(np.float32)], names=['TILEID', 'RA', 'DEC'])
isin, index = is_point_in_desi(tiles, gcs['ra'], gcs['dec'], return_tile_index=True)
gcs['TILEID'] = -99
gcs['TILEID'][isin] = tiles['TILEID'][index[isin]]
gcs = gcs[gcs['name'] == 'NGC5904']
gcs['RA'] = gcs['ra']
gcs['DEC'] = gcs['dec']
gcs = gcs['RA', 'DEC', 'name', 'TILEID']
tiles = tiles[tiles['TILEID'] == gcs['TILEID']]
gcs.pprint()
tiles.pprint()
pix = tiles2pix(nside=2, tiles=tiles)
release=sv_mtl['RELEASE'],
sky=0,
mock=0)
sv_mtl.pprint()
sv_mtl.write(
scratch +
'/BGS/SV-ASSIGN/mtls/MTL_ALLBGS_STDFAINT_STDBRIGHT_svresolve.0.31.0_49677629_samePRIORITY.fits',
format='fits',
overwrite=True)
##
nside = 64
(indesi, tindex) = is_point_in_desi(tiles,
sv_mtl['RA'].quantity.value,
sv_mtl['DEC'].quantity.value,
return_tile_index=True)
## Prevent 'duplication' of e.g. RA cols after join.
del tiles['RA']
del tiles['DEC']
## Some objects in the .mtl are not in the tiles; rough cut on creation I guess.
sv_mtl['TILEID'] = -99 * np.ones_like(sv_mtl['RA'].quantity.value,
dtype=np.int)
sv_mtl['TILEID'][indesi] = tiles['TILEID'].quantity[tindex[indesi]]
sv_mtl.sort('TILEID')
## Not associated to a tile?
## notin = sv_mtl[sv_mtl['TILEID'] < 0]
## notin.sort('DEC')
def QSO_filter(RA, DEC):
return is_point_in_desi(tiles, RA, DEC)
## ['tycho', 'gaia']
survey = 'gaia'
##
scratch = os.environ['CSCRATCH']
gaia = fits.open(scratch + '/BGS/SV-ASSIGN/masks/{}_stellar_mask.fits'.format(survey))
gaia = Table(gaia[1].data)
print(gaia)
##
nside = 64
(indesi, tindex) = is_point_in_desi(tiles, gaia['RA'], gaia['DEC'], return_tile_index=True)
## Prevent 'duplication' of e.g. RA cols after join.
del tiles['RA']
del tiles['DEC']
## Some objects in the .mtl are not in the tiles; rough cut on creation I guess.
gaia['TILEID'] = -99 * np.ones_like(gaia['RA'].quantity.value, dtype=np.int)
gaia['TILEID'][indesi] = tiles['TILEID'].quantity[tindex[indesi]]
gaia.sort('TILEID')
print(gaia)
## Only keep the targets in a given tile.
gaia = gaia[gaia['TILEID'] > 0]
gaia = join(gaia, tiles, keys=['TILEID'], join_type='left')
def lsscat_gen(root, prod, odir):
'''
# Get required parent pipeline files.
_mtl = root + 'targets/mtl.fits'
# Needed for fiberassign runs.
# _skies = '/project/projectdirs/desi/target/catalogs/dr8/0.31.0/skies/skies-dr8-0.31.0.fits'
# targets = Table(fits.open(root + 'targets/targets.fits')[1].data)
mtl = Table(fits.open(root + 'targets/mtl.fits')[1].data)
# exps = Table(fits.open(root + 'survey/sv_exposures.fits')[1].data)
# HACK: Cumulative SNR2 achieved on a given tile by this exposure. Weird that this was missing?
# exps['SNR2'] = np.random.uniform(0.9, 1.1, len(exps))
_tiles = fits.open(root + 'survey/SV-tiles.fits')[1]
tiles = Table(fits.open(root + 'survey/SV-tiles.fits')[1].data)
# zcat = Table(fits.open(root + 'spectro/redux/{}/zcatalog-{}.fits'.format(prod, prod))[1].data)
# DR8 randoms.
rows = np.arange(len(mtl))
randoms = fitsio.read('/project/projectdirs/desi/target/catalogs/dr8/0.31.0/randoms/randoms-inside-dr8-0.31.0-4.fits', rows=rows)
# A list of targets that could have been assigned to each fiber;
# desidatamodel.readthedocs.io/en/latest/DESI_TARGET/fiberassign/tile-TILEID-FIELDNUM.html#hdu2
rand_assignable, rand_assigned = get_randassign(root + 'fiberassign/tile-072015.fits', mtl=randoms, gen=True, randoms=randoms)
'''
rand_assignable, rand_assigned = get_randassign(root + 'fiberassign/tile-072015.fits', mtl=None, gen=True, randoms=None, lite=True)
exit(0)
# TARGETID | FIBERID | LOCATION
# print(rand_assignable)
# Date stamped tiles meeting quality requirements.
MJD0 = 58853.51129915193
MJD1 = 58881.40164263005
dst_tiles, hdr = datestamped_tiles(exps, MJD0, MJD1, printit=False)
# Write here (should add header). Assign tile assignment completeness below.
dst_tiles.write(odir + 'dst_tiles.fits', format='fits', overwrite=True)
'''
# Sort Targets by TARGETID
zcat.sort('TARGETID')
targets.sort('TARGETID')
# Unique TARGETID lists.
assert np.all(np.unique(zcat['TARGETID']) == zcat['TARGETID'])
assert np.all(np.unique(targets['TARGETID']) == targets['TARGETID'])
'''
# Catalogue generation.
ntarget = len(targets)
final = lss_catalog(ntarget)
final['TARGETID'] = targets['TARGETID']
final['RA'] = targets['RA']
final['DEC'] = targets['DEC']
final['IN_IMAGING'] = 1.0
## Neglects area lost to e.g. GFA.
final['IN_DESI'] = is_point_in_desi(tiles, final['RA'], final['DEC']).astype(np.int)
final['DESI_TARGET'] = targets['SV1_DESI_TARGET']
final['BGS_TARGET'] = targets['SV1_BGS_TARGET']
final['PRIORITY_INIT'] = targets['PRIORITY_INIT']
final['SUBPRIORITY'] = targets['SUBPRIORITY']
in_zcat = np.in1d( zcat['TARGETID'], final['TARGETID'])
in_final = np.in1d(final['TARGETID'], zcat['TARGETID'])
final['Z'][in_final] = zcat['Z'][in_zcat]
final['ZERR'][in_final] = zcat['ZERR'][in_zcat]
final['ZWARN'][in_final] = zcat['ZWARN'][in_zcat]
final['SPECTYPE'][in_final] = zcat['SPECTYPE'][in_zcat]
## Tag all galaxies that appeared in the redshift cats., even if bad etc.
final['Z_VETO_FLAG'] = 0
final = pop_assigned(final, printit=True)
final = set_ang_veto(final)
final = set_z_veto(final)
final = blind_distances(final)
final = in_cosmo(final)
final = final[final['Z'] > 0.]
final.write(odir + 'lsscat_v1.fits', format='fits', overwrite=True)
final.pprint(max_width=-1)
# Add tile assignment completeness based on reachable fibers from final.
dst_tiles = set_assigncomplete(dst_tiles, final)
import fitsio
import desimodel
import numpy as np
import astropy.io.fits as fits
from astropy.table import Table, unique
from desisurvey.utils import get_date
from desimodel.footprint import is_point_in_desi
##
tfiles = '/global/homes/m/mjwilson/desi/survey-validation/svdc-spring2020e-onepercent/run/survey/tiles/des.fits'
tiles = desimodel.io.load_tiles(tilesfile=tfiles)
# ids = complete['TILEID']
# isin = np.isin(tiles['TILEID'], ids)
skies = fitsio.read('/global/cfs/cdirs/desi/target/catalogs/dr8/0.32.0/skies/skies-dr8-0.32.0.fits')
# Cut to observed tiles.
isin = is_point_in_desi(tiles, skies['RA'], skies['DEC'])
skies = skies[isin]
fitsio.write('/global/cscratch1/sd/mjwilson/svdc-spring2020e-onepercent/targets/skies.fits', skies)
def make_mock_sweeps(config_file,
source_name,
input_dir,
epoch_dir,
map_id_file_path,
sweep_mock_root,
override_root=None,
dry_run=True,
match_on_objid=False):
"""
"""
if match_on_objid:
print(
'NOTE: Matching on mock column "objid", only makes sense for 2016 data challenge outputs!'
)
# Load the tile definitions
TILES = desimodel.io.load_tiles()
epoch = int(os.path.split(epoch_dir.strip(os.path.sep))[-1])
print('Epoch: {}'.format(epoch))
# Read parameters for quicksurvey
with open(config_file, 'r') as pfile:
params = yaml.load(pfile)
# Original mock path.
root_mock_dir = params['sources'][source_name]['root_mock_dir']
if override_root is not None:
original_root_mock_dir = root_mock_dir
root_mock_dir = override_root
# Read zcat and truth, and construct the mapping between them
zcat, truth_table, target_table, itruth_for_izcat = match_zcat_truth(
input_dir, epoch_dir)
# Read the table of mock paths for this source
map_id_file = np.loadtxt(map_id_file_path,
dtype={
'names': ('SOURCENAME', 'FILEID', 'FILENAME'),
'formats': ('S10', 'i4', 'S256')
})
# Filter truth by source
truth_this_source = truth_table['SOURCETYPE'] == source_name
# Decode rowid and fileid for all targets associated with this source
rowid, fileid = decode_rownum_filenum(
truth_table['MOCKID'][truth_this_source])
# Store the row indices for this source, will save them later
truth_rows_for_source = np.where(truth_this_source)[0]
# Filter truth by source for sources in zcat
obs_this_source = truth_table['SOURCETYPE'][
itruth_for_izcat] == source_name
# Decode rowid and fileid for observed targets associated with this source
obs_rowid, obs_fileid = decode_rownum_filenum(
truth_table['MOCKID'][itruth_for_izcat][obs_this_source])
# Determine if target/truth rows for this source are in footprint
print('Testing truth coordinates against DESIMODEL v %s' %
(desimodel.__version__))
footprint_flag_targets_truth = footprint.is_point_in_desi(
TILES, truth_table['RA'][truth_rows_for_source],
truth_table['DEC'][truth_rows_for_source])
# Read original files
fileid_to_read = np.array(list(set(fileid)))
# Different mocks have different names for sky coordinate columns
# Auto-detect this based on a list of possibilities.
ra_column, dec_column = None, None
# Names to try in order of priority
ra_names = ['RA', 'ra', '_RA', '_ra']
dec_names = ['DEC', 'dec', 'DE', '_DEC', '_dec', '_DE']
for ifile in fileid_to_read:
# Filemap entry for this file ID
row_in_map = (map_id_file['SOURCENAME'] == source_name.encode()) & (
map_id_file['FILEID'] == ifile)
filename = map_id_file['FILENAME'][row_in_map]
filename = filename[0].decode()
if override_root:
filename = filename.replace(original_root_mock_dir, root_mock_dir)
print('Reading mock file: {}'.format(filename))
sys.stdout.flush()
if source_name in ['MWS_MAIN', 'MWS_WD', 'MWS_NEARBY']:
n_this_mock_file = fits.getheader(filename, 1)['NAXIS2']
elif source_name in ['BGS']:
with h5py.File(filename) as f:
n_this_mock_file = f['/Header/number_galaxies'][...][0]
else:
raise Exception('Unrecognized source: {}'.format(source))
print('Mock file nrows: {}'.format(n_this_mock_file))
# This will be of the length of the total number of targets in this
# mock file. Not all of these will be selected, and not all of these
# will be observed.
selected_as_target = np.repeat(False, n_this_mock_file)
observed_this_epoch = np.repeat(False, n_this_mock_file)
# Read the mock data for this file ID
if source_name in ['MWS_MAIN', 'MWS_WD', 'MWS_NEARBY']:
data = fits.getdata(filename, 1)
elif source_name in ['BGS']:
data = Table()
with h5py.File(filename) as f:
for recname in f['/Data']:
d = f['/Data'][recname][...]
data.add_column(Column(name=recname, data=d))
else:
raise Exception('Unrecognized source: {}'.format(source))
# Set column names if not already set
if ra_column is None:
for ra_name in ra_names:
if ra_name in data.dtype.names:
ra_column = ra_name
break
if ra_column is None:
raise Exception('No RA column found in mock data columns')
if dec_column is None:
for dec_name in dec_names:
if dec_name in data.dtype.names:
dec_column = dec_name
break
if dec_column is None:
raise Exception('No DEC column found in mock data columns')
# Flag mock rows in footprint
print('Testing mock coordinates against DESIMODEL v %s' %
(desimodel.__version__))
footprint_flag = footprint.is_point_in_desi(TILES, data[ra_column],
data[dec_column])
# Select truth rows for this file ID
fileid_mask = fileid == ifile
obs_fileid_mask = obs_fileid == ifile
if match_on_objid:
# In the Dec 2016 data challenge, the 'row number' is actually the
# Galaxia mock objid for MWS_MAIN.
objid = data['objid']
objid_this_file_all = rowid[fileid_mask]
objid_this_file_obs = obs_rowid[obs_fileid_mask]
m = match(objid_this_file_all, objid)
rows_this_file_all = m[np.where(m >= 0)[0]]
m = match(objid_this_file_obs, objid)
rows_this_file_obs = m[np.where(m >= 0)[0]]
else:
rows_this_file_all = rowid[fileid_mask]
rows_this_file_obs = obs_rowid[obs_fileid_mask]
print(' -- N in truth = {}'.format(len(rows_this_file_all)))
print(' -- N observed = {}'.format(len(rows_this_file_obs)))
selected_as_target[rows_this_file_all] = True
observed_this_epoch[rows_this_file_obs] = True
assert (not np.any((observed_this_epoch) & (~selected_as_target)))
# Targets that could have been observed this epoch, but were not.
selected_not_observed = np.where((selected_as_target)
& (~observed_this_epoch))[0]
# Sort rows in this file by row number. Can use this to reorder subsets
# of targets and truth, if we want to output them.
rows_this_file_all_sort = np.argsort(rows_this_file_all)
# Write various outputs
filename_path, filename_file = os.path.split(filename)
base, original_ext = filename_file.split(os.path.extsep)
ext = 'fits' # Output fits, even if input is hdf5
new_filename_status = base + os.path.extsep + 'status' + os.path.extsep + ext
new_filename_observed = base + os.path.extsep + 'observed' + os.path.extsep + ext
new_filename_unobserved = base + os.path.extsep + 'unobserved' + os.path.extsep + ext
new_filename_targets_subset = base + os.path.extsep + 'targets' + os.path.extsep + ext
new_filename_truth_subset = base + os.path.extsep + 'truth' + os.path.extsep + ext
if override_root:
filename_path = filename_path.replace(original_root_mock_dir,
root_mock_dir)
if filename_path.startswith(os.path.sep):
filename_path = os.path.curdir + filename_path
new_dir = os.path.normpath(
os.path.join(sweep_mock_root, filename_path, str(epoch)))
if not os.path.exists(new_dir): os.makedirs(new_dir)
new_path_status = os.path.normpath(
os.path.join(sweep_mock_root, filename_path, str(epoch),
new_filename_status))
new_path_observed = os.path.normpath(
os.path.join(sweep_mock_root, filename_path, str(epoch),
new_filename_observed))
new_path_unobserved = os.path.normpath(
os.path.join(sweep_mock_root, filename_path, str(epoch),
new_filename_unobserved))
new_path_targets_subset = os.path.normpath(
os.path.join(sweep_mock_root, filename_path, str(epoch),
new_filename_targets_subset))
new_path_truth_subset = os.path.normpath(
os.path.join(sweep_mock_root, filename_path, str(epoch),
new_filename_truth_subset))
# Write the status table, which has the same number of rows as the
# origianl mock file (hence generally more than the sweep).
t = Table((selected_as_target, observed_this_epoch),
names=('SELECTED', 'OBSERVED'))
print('Writing {}'.format(new_path_status))
t.write(new_path_status, overwrite=True)
# Provided that at least one target in this mock file has been
# selected, write subsets of the mock file.
# Previous had rows_this_file_obs here
if len(rows_this_file_all) > 0:
# For each row in mock that appears in targets/truth, get the
# corresponding row number in the target/truth.
input_target_row = np.zeros(n_this_mock_file, dtype=np.int32) - 1
# ... These are the t/t rows that are in this mock file.
w_fileid_mask = np.where(fileid_mask)[0]
# Store row
input_target_row[rows_this_file_all] = truth_rows_for_source[
w_fileid_mask]
assert (np.all(input_target_row[observed_this_epoch] >= 0))
# 1. Make the mock sweep for observed targets
t = Table(data[observed_this_epoch])
t.add_column(
Column(input_target_row[observed_this_epoch],
name='TARGETROW'))
t.add_column(
Column(target_table[input_target_row[observed_this_epoch]]
['TARGETID'],
name='TARGETID'))
# Include flag for targets in actual footprint
t.add_column(
Column(footprint_flag[observed_this_epoch],
name='IN_FOOTPRINT'))
# Strict check
assert (np.allclose(t[ra_column],
target_table[t['TARGETROW']]['RA'],
atol=1e-5))
t.write(new_path_observed, overwrite=True)
print('Wrote {}'.format(new_path_observed))
# 2. Make the mock sweep for unobserved targets
t = Table(data[selected_not_observed])
t.add_column(
Column(input_target_row[selected_not_observed],
name='TARGETROW'))
t.add_column(
Column(target_table[input_target_row[selected_not_observed]]
['TARGETID'],
name='TARGETID'))
# Include flag for targets in actual footprint
t.add_column(
Column(footprint_flag[selected_not_observed],
name='IN_FOOTPRINT'))
# Strict check
assert (np.allclose(t[ra_column],
target_table[t['TARGETROW']]['RA'],
atol=1e-5))
t.write(new_path_unobserved, overwrite=True)
print('Wrote {}'.format(new_path_unobserved))
# 3. Make a file extracted from targets for this mock brick
# Save in rowid order
subset_this_file = truth_rows_for_source[fileid_mask]
t = Table(target_table[subset_this_file])[rows_this_file_all_sort]
t.add_column(
Column((footprint_flag_targets_truth[fileid_mask]
)[rows_this_file_all_sort],
name='IN_FOOTPRINT'))
t.write(new_path_targets_subset, overwrite=True)
print('Wrote {}'.format(new_path_targets_subset))
# 4. Make a file extracted from truth for this mock brick
subset_this_file = truth_rows_for_source[fileid_mask]
t = Table(truth_table[subset_this_file])[rows_this_file_all_sort]
t.add_column(
Column((footprint_flag_targets_truth[fileid_mask]
)[rows_this_file_all_sort],
name='IN_FOOTPRINT'))
t.write(new_path_truth_subset, overwrite=True)
print('Wrote {}'.format(new_path_truth_subset))
return
def select_gfas(infiles, maglim=18, numproc=4, tilesfile=None, cmx=False):
"""Create a set of GFA locations using Gaia.
Parameters
----------
infiles : :class:`list` or `str`
A list of input filenames (sweep files) OR a single filename.
maglim : :class:`float`, optional, defaults to 18
Magnitude limit for GFAs in Gaia G-band.
numproc : :class:`int`, optional, defaults to 4
The number of parallel processes to use.
tilesfile : :class:`str`, optional, defaults to ``None``
Name of tiles file to load. For full details, see
:func:`~desimodel.io.load_tiles`.
cmx : :class:`bool`, defaults to ``False``
If ``True``, do not limit output to DESI tiling footprint.
Used for selecting wider-ranging commissioning targets.
Returns
-------
:class:`~numpy.ndarray`
GFA objects from Gaia across all of the passed input files, formatted
according to `desitarget.gfa.gfadatamodel`.
Notes
-----
- If numproc==1, use the serial code instead of the parallel code.
- The tiles loaded from `tilesfile` will only be those in DESI.
So, for custom tilings, set IN_DESI==1 in your tiles file.
"""
# ADM convert a single file, if passed to a list of files.
if isinstance(infiles, str):
infiles = [infiles, ]
nfiles = len(infiles)
# ADM check that files exist before proceeding.
for filename in infiles:
if not os.path.exists(filename):
msg = "{} doesn't exist".format(filename)
log.critical(msg)
raise ValueError(msg)
# ADM load the tiles file.
tiles = desimodel.io.load_tiles(tilesfile=tilesfile)
# ADM check some files loaded.
if len(tiles) == 0:
msg = "no tiles found in {}".format(tilesfile)
log.critical(msg)
raise ValueError(msg)
# ADM the critical function to run on every file.
def _get_gfas(fn):
'''wrapper on gaia_gfas_from_sweep() given a file name'''
return gaia_gfas_from_sweep(fn, maglim=maglim)
# ADM this is just to count sweeps files in _update_status.
nfile = np.zeros((), dtype='i8')
t0 = time()
def _update_status(result):
"""wrapper function for the critical reduction operation,
that occurs on the main parallel process"""
if nfile % 50 == 0 and nfile > 0:
elapsed = (time()-t0)/60.
rate = nfile/elapsed/60.
log.info('{}/{} files; {:.1f} files/sec...t = {:.1f} mins'
.format(nfile, nfiles, rate, elapsed))
nfile[...] += 1 # this is an in-place modification.
return result
# - Parallel process input files.
if numproc > 1:
pool = sharedmem.MapReduce(np=numproc)
with pool:
gfas = pool.map(_get_gfas, infiles, reduce=_update_status)
else:
gfas = list()
for file in infiles:
gfas.append(_update_status(_get_gfas(file)))
gfas = np.concatenate(gfas)
# ADM resolve any duplicates between imaging data releases.
gfas = resolve(gfas)
# ADM retrieve Gaia objects in the DESI footprint or passed tiles.
log.info('Retrieving additional Gaia objects...t = {:.1f} mins'
.format((time()-t0)/60))
gaia = all_gaia_in_tiles(maglim=maglim, numproc=numproc, allsky=cmx,
tiles=tiles)
# ADM and limit them to just any missing bricks...
brickids = set(gfas['BRICKID'])
ii = [gbrickid not in brickids for gbrickid in gaia["BRICKID"]]
gaia = gaia[ii]
gfas = np.concatenate([gfas, gaia])
# ADM limit to DESI footprint or passed tiles, if not cmx'ing.
if not cmx:
ii = is_point_in_desi(tiles, gfas["RA"], gfas["DEC"])
gfas = gfas[ii]
return gfas