def test_tractor_columns(self):
# ADM Gaia columns that get added on input.
from desitarget.gaiamatch import gaiadatamodel
from desitarget.gaiamatch import pop_gaia_coords, pop_gaia_columns
# ADM have to remove the GAIA_RA, GAIA_DEC columns used for matching.
gaiadatamodel = pop_gaia_coords(gaiadatamodel)
# ADM prior to DR8, we're also missing some other Gaia columns.
# gaiadatamodel = pop_gaia_columns(
# gaiadatamodel,
# ['REF_CAT', 'GAIA_PHOT_BP_RP_EXCESS_FACTOR',
# 'GAIA_ASTROMETRIC_SIGMA5D_MAX', 'GAIA_ASTROMETRIC_PARAMS_SOLVED']
# )
# ADM PHOTSYS gets added on input.
tscolumns = list(io.tsdatamodel.dtype.names) \
+ ['PHOTSYS'] \
+ list(gaiadatamodel.dtype.names)
tractorfile = io.list_tractorfiles(self.datadir)[0]
data = io.read_tractor(tractorfile)
self.assertEqual(set(data.dtype.names), set(tscolumns))
# columns = ['BX', 'BY']
columns = ['RA', 'DEC']
data = io.read_tractor(tractorfile, columns=columns)
self.assertEqual(set(data.dtype.names), set(columns))
data = io.read_tractor(tractorfile, columns=tuple(columns))
self.assertEqual(set(data.dtype.names), set(columns))
def test_tractor_columns(self):
# ADM Gaia columns that get added on input.
from desitarget.gaiamatch import gaiadatamodel
from desitarget.gaiamatch import pop_gaia_coords, pop_gaia_columns
# ADM remove the GAIA_RA, GAIA_DEC columns used for matching.
gaiadatamodel = pop_gaia_coords(gaiadatamodel)
tractorfile = io.list_tractorfiles(self.datadir)[0]
data = io.read_tractor(tractorfile)
# ADM form the final data model in a manner that maintains
# ADM backwards-compatability with DR8.
if "FRACDEV" in data.dtype.names:
tsdatamodel = np.array(
[], dtype=io.basetsdatamodel.dtype.descr +
io.dr8addedcols.dtype.descr)
else:
tsdatamodel = np.array(
[], dtype=io.basetsdatamodel.dtype.descr +
io.dr9addedcols.dtype.descr)
# ADM PHOTSYS gets added on input.
tscolumns = list(tsdatamodel.dtype.names) \
+ ['PHOTSYS'] \
+ list(gaiadatamodel.dtype.names)
self.assertEqual(set(data.dtype.names), set(tscolumns))
# columns = ['BX', 'BY']
columns = ['RA', 'DEC']
data = io.read_tractor(tractorfile, columns=columns)
self.assertEqual(set(data.dtype.names), set(columns))
data = io.read_tractor(tractorfile, columns=tuple(columns))
self.assertEqual(set(data.dtype.names), set(columns))
def add_gaia_columns(indata):
"""Add columns needed for MWS targeting to a sweeps-style array.
Parameters
----------
indata : :class:`~numpy.ndarray`
Numpy structured array to which to add Gaia-relevant columns.
Returns
-------
:class:`~numpy.ndarray`
Input array with the Gaia columns added.
Notes
-----
- Gaia columns resemble the data model in :mod:`desitarget.gaiamatch`
but with "GAIA_RA" and "GAIA_DEC" removed.
"""
# ADM remove the Gaia coordinates from the Gaia data model as they aren't
# ADM in the imaging surveys data model.
from desitarget.gaiamatch import gaiadatamodel, pop_gaia_coords
gaiadatamodel = pop_gaia_coords(gaiadatamodel)
# ADM create the combined data model.
dt = indata.dtype.descr + gaiadatamodel.dtype.descr
# ADM create a new numpy array with the fields from the new data model...
nrows = len(indata)
outdata = np.zeros(nrows, dtype=dt)
# ADM ...and populate them with the passed columns of data.
for col in indata.dtype.names:
outdata[col] = indata[col]
# ADM set REF_ID to -1 to indicate nothing has a Gaia match (yet).
outdata['REF_ID'] = -1
return outdata
def read_tractor(filename, header=False, columns=None):
"""Read a tractor catalogue file.
Parameters
----------
filename : :class:`str`
File name of one Tractor or sweeps file.
header : :class:`bool`, optional
If ``True``, return (data, header) instead of just data.
columns: :class:`list`, optional
Specify the desired Tractor catalog columns to read; defaults to
desitarget.io.tsdatamodel.dtype.names.
Returns
-------
:class:`~numpy.ndarray`
Array with the tractor schema, uppercase field names.
"""
check_fitsio_version()
fx = fitsio.FITS(filename, upper=True)
fxcolnames = fx[1].get_colnames()
hdr = fx[1].read_header()
if columns is None:
readcolumns = list(tsdatamodel.dtype.names)
# ADM if RELEASE doesn't exist, then we're pre-DR3 and need the old data model.
if (('RELEASE' not in fxcolnames) and ('release' not in fxcolnames)):
readcolumns = list(oldtscolumns)
else:
readcolumns = list(columns)
# - tractor files have BRICK_PRIMARY; sweep files don't
if (columns is None) and \
(('BRICK_PRIMARY' in fxcolnames) or ('brick_primary' in fxcolnames)):
readcolumns.append('BRICK_PRIMARY')
# ADM if BRIGHTSTARINBLOB exists (it does for DR7, not for DR6) add it and
# ADM the other DR6->DR7 data model updates.
if (columns is None) and \
(('BRIGHTSTARINBLOB' in fxcolnames) or ('brightstarinblob' in fxcolnames)):
for col in dr7datamodel.dtype.names:
readcolumns.append(col)
# ADM if BRIGHTBLOB exists (it does for DR8, not for DR7) add it and
# ADM the other DR6->DR8 data model updates.
else:
if (columns is None) and \
(('BRIGHTBLOB' in fxcolnames) or ('brightblob' in fxcolnames)):
for col in dr8datamodel.dtype.names:
readcolumns.append(col)
# ADM if Gaia information was passed, add it to the columns to read.
if (columns is None):
if (('REF_ID' in fxcolnames) or ('ref_id' in fxcolnames)):
# ADM remove the Gaia coordinates as they aren't in the imaging data model.
from desitarget.gaiamatch import gaiadatamodel, pop_gaia_coords, pop_gaia_columns
gaiadatamodel = pop_gaia_coords(gaiadatamodel)
# ADM the DR7 sweeps don't contain these columns, but DR8 should.
if 'REF_CAT' not in fxcolnames:
gaiadatamodel = pop_gaia_columns(
gaiadatamodel,
['REF_CAT', 'GAIA_PHOT_BP_RP_EXCESS_FACTOR',
'GAIA_ASTROMETRIC_SIGMA5D_MAX', 'GAIA_ASTROMETRIC_PARAMS_SOLVED']
)
gaiacols = gaiadatamodel.dtype.names
readcolumns += gaiacols
if (columns is None) and \
(('RELEASE' not in fxcolnames) and ('release' not in fxcolnames)):
# ADM Rewrite the data completely to correspond to the DR4+ data model.
# ADM we default to writing RELEASE = 3000 ("DR3, or before, data")
data = convert_from_old_data_model(fx, columns=readcolumns)
else:
data = fx[1].read(columns=readcolumns)
# ADM add Gaia columns if not passed.
if (columns is None) and \
(('REF_ID' not in fxcolnames) and ('ref_id' not in fxcolnames)):
data = add_gaia_columns(data)
# ADM add DR8 data model updates (with zero/False) columns if not passed.
if (columns is None) and \
(('BRIGHTBLOB' not in fxcolnames) and ('brightblob' not in fxcolnames)):
data = add_dr8_columns(data)
# ADM Empty (length 0) files have dtype='>f8' instead of 'S8' for brickname.
if len(data) == 0:
log.warning('WARNING: Empty file>', filename)
dt = data.dtype.descr
dt[1] = ('BRICKNAME', 'S8')
data = data.astype(np.dtype(dt))
# ADM To circumvent whitespace bugs on I/O from fitsio.
# ADM need to strip any white space from string columns.
for colname in data.dtype.names:
kind = data[colname].dtype.kind
if kind == 'U' or kind == 'S':
data[colname] = np.char.rstrip(data[colname])
# ADM add the PHOTSYS column to unambiguously check whether we're using imaging
# ADM from the "North" or "South".
data = add_photsys(data)
if header:
fx.close()
return data, hdr
else:
fx.close()
return data