def allStar(dr=None):
"""
NAME:
allStar
PURPOSE:
download the allStar file
INPUT:
dr= return the path corresponding to this data release (general default)
OUTPUT:
(none; just downloads)
HISTORY:
2014-11-26 - Written - Bovy (IAS)
2015-08-17 - Adjusted for new path (mv old to new) - Bovy (UofT)
"""
if dr is None: dr = path._default_dr()
# First make sure the file doesn't exist
filePath = path.allStarPath(dr=dr)
if os.path.exists(filePath): return None
# Check whether we can find it in its old place
oldFilePath = path.allStarPath(dr=dr, _old=True)
if os.path.exists(oldFilePath):
# mv to new place
try:
os.makedirs(os.path.dirname(filePath))
except OSError:
pass
shutil.move(oldFilePath, filePath)
return None
# Create the file path
downloadPath = filePath.replace(
os.path.join(path._APOGEE_DATA, _dr_string(dr)), _base_url(dr=dr))
_download_file(downloadPath, filePath, dr, verbose=True)
return None
def allStar(dr=None):
"""
NAME:
allStar
PURPOSE:
download the allStar file
INPUT:
dr= return the path corresponding to this data release (general default)
OUTPUT:
(none; just downloads)
HISTORY:
2014-11-26 - Written - Bovy (IAS)
2015-08-17 - Adjusted for new path (mv old to new) - Bovy (UofT)
"""
if dr is None: dr= path._default_dr()
# First make sure the file doesn't exist
filePath= path.allStarPath(dr=dr)
if os.path.exists(filePath): return None
# Check whether we can find it in its old place
oldFilePath= path.allStarPath(dr=dr,_old=True)
if os.path.exists(oldFilePath):
# mv to new place
try:
os.makedirs(os.path.dirname(filePath))
except OSError: pass
shutil.move(oldFilePath,filePath)
return None
# Create the file path
downloadPath= filePath.replace(os.path.join(path._APOGEE_DATA,
_dr_string(dr)),
_base_url(dr=dr))
_download_file(downloadPath,filePath,dr,verbose=True)
return None
def allStar(dr=None):
"""
NAME:
allStar
PURPOSE:
download the allStar file
INPUT:
dr= return the path corresponding to this data release (general default)
OUTPUT:
(none; just downloads)
HISTORY:
2014-11-26 - Written - Bovy (IAS)
"""
if dr is None: dr= path._default_dr()
# First make sure the file doesn't exist
filePath= path.allStarPath(dr=dr)
if os.path.exists(filePath): return None
# Create the file path, hacked from aspcapStar path
aspPath= path.aspcapStarPath(4140,'dum',dr=dr)
downloadPath= aspPath.replace(os.path.join(path._APOGEE_DATA,
'dr%s' % dr),
_base_url(dr=dr))
head, tail= os.path.split(downloadPath) #strips off filename
downloadPath, tail= os.path.split(head) #strips off location_id
downloadPath= os.path.join(downloadPath,os.path.basename(filePath))
_download_file(downloadPath,filePath,dr,verbose=True)
return None
def allStar(rmcommissioning=True,
main=False,
exclude_star_bad=False,
exclude_star_warn=False,
ak=True,
akvers='targ',
rmnovisits=False,
adddist=False,
distredux=None,
rmdups=False,
raw=False):
"""
NAME:
allStar
PURPOSE:
read the allStar file
INPUT:
rmcommissioning= (default: True) if True, only use data obtained after commissioning
main= (default: False) if True, only select stars in the main survey
exclude_star_bad= (False) if True, remove stars with the STAR_BAD flag set in ASPCAPFLAG
exclude_star_warn= (False) if True, remove stars with the STAR_WARN flag set in ASPCAPFLAG
ak= (default: True) only use objects for which dereddened mags exist
akvers= 'targ' (default) or 'wise': use target AK (AK_TARG) or AK derived from all-sky WISE (AK_WISE)
rmnovisits= (False) if True, remove stars with no good visits (to go into the combined spectrum); shouldn't be necessary
adddist= (default: False) add distances (DR10/11 Hayden distances, DR12 combined distances)
distredux= (default: DR default) reduction on which the distances are based
rmdups= (False) if True, remove duplicates (very slow)
raw= (False) if True, just return the raw file, read w/ fitsio
OUTPUT:
allStar data
HISTORY:
2013-09-06 - Written - Bovy (IAS)
"""
filePath = path.allStarPath()
if not os.path.exists(filePath):
download.allStar()
#read allStar file
data = fitsio.read(path.allStarPath())
if raw: return data
#Remove duplicates, cache
if rmdups:
dupsFilename = path.allStarPath().replace('.fits', '-nodups.fits')
if os.path.exists(dupsFilename):
data = fitsio.read(dupsFilename)
else:
sys.stdout.write(
'\r' +
"Removing duplicates (might take a while) and caching the duplicate-free file ...\r"
)
sys.stdout.flush()
data = remove_duplicates(data)
#Cache this file for subsequent use of rmdups
fitsio.write(dupsFilename, data, clobber=True)
sys.stdout.write('\r' + _ERASESTR + '\r')
sys.stdout.flush()
#Some cuts
if rmcommissioning:
indx = numpy.array(
['apogee.n.c'.encode('utf-8') in s for s in data['APSTAR_ID']])
indx += numpy.array(
['apogee.s.c'.encode('utf-8') in s for s in data['APSTAR_ID']])
data = data[True - indx]
if rmnovisits:
indx = numpy.array([s.strip() != '' for s in data['VISITS']])
data = data[indx]
if main:
indx = mainIndx(data)
data = data[indx]
if akvers.lower() == 'targ':
aktag = 'AK_TARG'
elif akvers.lower() == 'wise':
aktag = 'AK_WISE'
if ak:
data = data[True - numpy.isnan(data[aktag])]
data = data[(data[aktag] > -50.)]
if exclude_star_bad:
data = data[(data['ASPCAPFLAG'] & 2**23) == 0]
if exclude_star_warn:
data = data[(data['ASPCAPFLAG'] & 2**7) == 0]
#Add dereddened J, H, and Ks
aj = data[aktag] * 2.5
ah = data[aktag] * 1.55
if _ESUTIL_LOADED:
data = esutil.numpy_util.add_fields(data,
[('J0', float), ('H0', float),
('K0', float)])
data['J0'] = data['J'] - aj
data['H0'] = data['H'] - ah
data['K0'] = data['K'] - data[aktag]
data['J0'][(data[aktag] <= -50.)] = -9999.9999
data['H0'][(data[aktag] <= -50.)] = -9999.9999
data['K0'][(data[aktag] <= -50.)] = -9999.9999
else:
warnings.warn(
"Extinction-corrected J,H,K not added because esutil is not installed",
RuntimeWarning)
#Add distances
if adddist and _ESUTIL_LOADED:
dist = fitsio.read(path.distPath(), 1)
h = esutil.htm.HTM()
m1, m2, d12 = h.match(dist['RA'],
dist['DEC'],
data['RA'],
data['DEC'],
2. / 3600.,
maxmatch=1)
data = data[m2]
dist = dist[m1]
distredux = path._redux_dr()
if distredux.lower() == 'v302' or distredux.lower() == path._DR10REDUX:
data = esutil.numpy_util.add_fields(data, [('DM05', float),
('DM16', float),
('DM50', float),
('DM84', float),
('DM95', float),
('DMPEAK', float),
('DMAVG', float),
('SIG_DM', float),
('DIST_SOL', float),
('SIG_DISTSOL', float)])
data['DM05'] = dist['DM05']
data['DM16'] = dist['DM16']
data['DM50'] = dist['DM50']
data['DM84'] = dist['DM84']
data['DM95'] = dist['DM95']
data['DMPEAK'] = dist['DMPEAK']
data['DMAVG'] = dist['DMAVG']
data['SIG_DM'] = dist['SIG_DM']
data['DIST_SOL'] = dist['DIST_SOL'] / 1000.
data['SIG_DISTSOL'] = dist['SIG_DISTSOL'] / 1000.
elif distredux.lower() == path._DR11REDUX:
data = esutil.numpy_util.add_fields(data, [('DISO', float),
('DMASS', float),
('DISO_GAL', float),
('DMASS_GAL', float)])
data['DISO'] = dist['DISO'][:, 1]
data['DMASS'] = dist['DMASS'][:, 1]
data['DISO_GAL'] = dist['DISO_GAL'][:, 1]
data['DMASS_GAL'] = dist['DMASS_GAL'][:, 1]
elif distredux.lower() == path._DR12REDUX:
data = esutil.numpy_util.add_fields(
data, [('HIP_PLX', float), ('HIP_E_PLX', float),
('RC_DIST', float), ('APOKASC_DIST_DIRECT', float),
('BPG_DIST1_MEAN', float), ('HAYDEN_DIST_PEAK', float),
('SCHULTHEIS_DIST', float)])
data['HIP_PLX'] = dist['HIP_PLX']
data['HIP_E_PLX'] = dist['HIP_E_PLX']
data['RC_DIST'] = dist['RC_dist_pc']
data[
'APOKASC_DIST_DIRECT'] = dist['APOKASC_dist_direct_pc'] / 1000.
data['BPG_DIST1_MEAN'] = dist['BPG_dist1_mean']
data['HAYDEN_DIST_PEAK'] = 10.**(dist['HAYDEN_distmod_PEAK'] / 5. -
2.)
data['SCHULTHEIS_DIST'] = dist['SCHULTHEIS_dist']
elif adddist:
warnings.warn(
"Distances not added because matching requires the uninstalled esutil module",
RuntimeWarning)
if _ESUTIL_LOADED and (path._APOGEE_REDUX.lower() == 'current' \
or 'l30' in path._APOGEE_REDUX.lower() \
or int(path._APOGEE_REDUX[1:]) > 600):
data = esutil.numpy_util.add_fields(data, [('METALS', float),
('ALPHAFE', float)])
data['METALS'] = data['PARAM'][:, paramIndx('metals')]
data['ALPHAFE'] = data['PARAM'][:, paramIndx('alpha')]
return data
def allStar(rmcommissioning=True,
main=False,
exclude_star_bad=False,
exclude_star_warn=False,
ak=True,
akvers='targ',
rmnovisits=False,
use_astroNN=False,
use_astroNN_abundances=False,
use_astroNN_distances=False,
use_astroNN_ages=False,
adddist=False,
distredux=None,
rmdups=False,
raw=False,
mjd=58104,
xmatch=None,
**kwargs):
"""
NAME:
allStar
PURPOSE:
read the allStar file
INPUT:
rmcommissioning= (default: True) if True, only use data obtained after commissioning
main= (default: False) if True, only select stars in the main survey
exclude_star_bad= (False) if True, remove stars with the STAR_BAD flag set in ASPCAPFLAG
exclude_star_warn= (False) if True, remove stars with the STAR_WARN flag set in ASPCAPFLAG
ak= (default: True) only use objects for which dereddened mags exist
akvers= 'targ' (default) or 'wise': use target AK (AK_TARG) or AK derived from all-sky WISE (AK_WISE)
rmnovisits= (False) if True, remove stars with no good visits (to go into the combined spectrum); shouldn't be necessary
use_astroNN= (False) if True, swap in astroNN (Leung & Bovy 2019a) parameters (get placed in, e.g., TEFF and TEFF_ERR), astroNN distances (Leung & Bovy 2019b), and astroNN ages (Mackereth, Bovy, Leung, et al. (2019)
use_astroNN_abundances= (False) only swap in astroNN parameters and abundances, not distances and ages
use_astroNN_distances= (False) only swap in astroNN distances, not parameters and abundances and ages
use_astroNN_ages= (False) only swap in astroNN ages, not parameters and abundances and distances
adddist= (default: False) add distances (DR10/11 Hayden distances, DR12 combined distances)
distredux= (default: DR default) reduction on which the distances are based
rmdups= (False) if True, remove duplicates (very slow)
raw= (False) if True, just return the raw file, read w/ fitsio
mjd= (58104) MJD of version for monthly internal pipeline runs
xmatch= (None) uses gaia_tools.xmatch.cds to x-match to an external catalog (eg., Gaia DR2 for xmatch='vizier:I/345/gaia2') and caches the result for re-use; requires jobovy/gaia_tools
+gaia_tools.xmatch.cds keywords
OUTPUT:
allStar data[,xmatched table]
HISTORY:
2013-09-06 - Written - Bovy (IAS)
2018-01-22 - Edited for new monthly pipeline runs - Bovy (UofT)
2018-05-09 - Add xmatch - Bovy (UofT)
2018-10-20 - Add use_astroNN option - Bovy (UofT)
2018-02-15 - Add astroNN distances and corresponding options - Bovy (UofT)
2018-02-16 - Add astroNN ages and corresponding options - Bovy (UofT)
"""
filePath = path.allStarPath(mjd=mjd)
if not os.path.exists(filePath):
download.allStar(mjd=mjd)
#read allStar file
data = fitsread(path.allStarPath(mjd=mjd))
#Add astroNN? astroNN file matched line-by-line to allStar, so match here
# [ages file not matched line-by-line]
if use_astroNN or kwargs.get('astroNN', False) or use_astroNN_abundances:
_warn_astroNN_abundances()
astroNNdata = astroNN()
data = _swap_in_astroNN(data, astroNNdata)
if use_astroNN or kwargs.get('astroNN', False) or use_astroNN_distances:
_warn_astroNN_distances()
astroNNdata = astroNNDistances()
data = _add_astroNN_distances(data, astroNNdata)
if use_astroNN or kwargs.get('astroNN', False) or use_astroNN_ages:
_warn_astroNN_ages()
astroNNdata = astroNNAges()
data = _add_astroNN_ages(data, astroNNdata)
if raw: return data
#Remove duplicates, cache
if rmdups:
dupsFilename = path.allStarPath(mjd=mjd).replace(
'.fits', '-nodups.fits')
if os.path.exists(dupsFilename):
data = fitsread(dupsFilename)
else:
sys.stdout.write(
'\r' +
"Removing duplicates (might take a while) and caching the duplicate-free file ...\r"
)
sys.stdout.flush()
data = remove_duplicates(data)
#Cache this file for subsequent use of rmdups
fitswrite(dupsFilename, data, clobber=True)
sys.stdout.write('\r' + _ERASESTR + '\r')
sys.stdout.flush()
if not xmatch is None:
from gaia_tools.load import _xmatch_cds
if rmdups:
matchFilePath = dupsFilename
else:
matchFilePath = filePath
if use_astroNN_ages:
matchFilePath = matchFilePath.replace('rc-', 'rc-astroNN-ages-')
ma, mai = _xmatch_cds(data, xmatch, filePath, **kwargs)
data = data[mai]
#Some cuts
if rmcommissioning:
try:
indx = numpy.array(
['apogee.n.c'.encode('utf-8') in s for s in data['APSTAR_ID']])
indx += numpy.array(
['apogee.s.c'.encode('utf-8') in s for s in data['APSTAR_ID']])
except TypeError:
indx = numpy.array(['apogee.n.c' in s for s in data['APSTAR_ID']])
indx += numpy.array(['apogee.s.c' in s for s in data['APSTAR_ID']])
data = data[True ^ indx]
if not xmatch is None: ma = ma[True ^ indx]
if rmnovisits:
indx = numpy.array([s.strip() != '' for s in data['VISITS']])
data = data[indx]
if not xmatch is None: ma = ma[indx]
if main:
indx = mainIndx(data)
data = data[indx]
if not xmatch is None: ma = ma[indx]
if akvers.lower() == 'targ':
aktag = 'AK_TARG'
elif akvers.lower() == 'wise':
aktag = 'AK_WISE'
if ak:
if not xmatch is None: ma = ma[True ^ numpy.isnan(data[aktag])]
data = data[True ^ numpy.isnan(data[aktag])]
if not xmatch is None: ma = ma[(data[aktag] > -50.)]
data = data[(data[aktag] > -50.)]
if exclude_star_bad:
if not xmatch is None: ma = ma[(data['ASPCAPFLAG'] & 2**23) == 0]
data = data[(data['ASPCAPFLAG'] & 2**23) == 0]
if exclude_star_warn:
if not xmatch is None: ma = ma[(data['ASPCAPFLAG'] & 2**7) == 0]
data = data[(data['ASPCAPFLAG'] & 2**7) == 0]
#Add dereddened J, H, and Ks
aj = data[aktag] * 2.5
ah = data[aktag] * 1.55
if _ESUTIL_LOADED:
data = esutil.numpy_util.add_fields(data,
[('J0', float), ('H0', float),
('K0', float)])
data['J0'] = data['J'] - aj
data['H0'] = data['H'] - ah
data['K0'] = data['K'] - data[aktag]
data['J0'][(data[aktag] <= -50.)] = -9999.9999
data['H0'][(data[aktag] <= -50.)] = -9999.9999
data['K0'][(data[aktag] <= -50.)] = -9999.9999
else:
warnings.warn(
"Extinction-corrected J,H,K not added because esutil is not installed",
RuntimeWarning)
#Add distances
if adddist and _ESUTIL_LOADED:
dist = fitsread(path.distPath(), 1)
h = esutil.htm.HTM()
m1, m2, d12 = h.match(dist['RA'],
dist['DEC'],
data['RA'],
data['DEC'],
2. / 3600.,
maxmatch=1)
data = data[m2]
if not xmatch is None: ma = ma[m2]
dist = dist[m1]
distredux = path._redux_dr()
if distredux.lower() == 'v302' or distredux.lower() == path._DR10REDUX:
data = esutil.numpy_util.add_fields(data, [('DM05', float),
('DM16', float),
('DM50', float),
('DM84', float),
('DM95', float),
('DMPEAK', float),
('DMAVG', float),
('SIG_DM', float),
('DIST_SOL', float),
('SIG_DISTSOL', float)])
data['DM05'] = dist['DM05']
data['DM16'] = dist['DM16']
data['DM50'] = dist['DM50']
data['DM84'] = dist['DM84']
data['DM95'] = dist['DM95']
data['DMPEAK'] = dist['DMPEAK']
data['DMAVG'] = dist['DMAVG']
data['SIG_DM'] = dist['SIG_DM']
data['DIST_SOL'] = dist['DIST_SOL'] / 1000.
data['SIG_DISTSOL'] = dist['SIG_DISTSOL'] / 1000.
elif distredux.lower() == path._DR11REDUX:
data = esutil.numpy_util.add_fields(data, [('DISO', float),
('DMASS', float),
('DISO_GAL', float),
('DMASS_GAL', float)])
data['DISO'] = dist['DISO'][:, 1]
data['DMASS'] = dist['DMASS'][:, 1]
data['DISO_GAL'] = dist['DISO_GAL'][:, 1]
data['DMASS_GAL'] = dist['DMASS_GAL'][:, 1]
elif distredux.lower() == path._DR12REDUX:
data = esutil.numpy_util.add_fields(
data, [('HIP_PLX', float), ('HIP_E_PLX', float),
('RC_DIST', float), ('APOKASC_DIST_DIRECT', float),
('BPG_DIST1_MEAN', float), ('HAYDEN_DIST_PEAK', float),
('SCHULTHEIS_DIST', float)])
data['HIP_PLX'] = dist['HIP_PLX']
data['HIP_E_PLX'] = dist['HIP_E_PLX']
data['RC_DIST'] = dist['RC_dist_pc']
data[
'APOKASC_DIST_DIRECT'] = dist['APOKASC_dist_direct_pc'] / 1000.
data['BPG_DIST1_MEAN'] = dist['BPG_dist1_mean']
data['HAYDEN_DIST_PEAK'] = 10.**(dist['HAYDEN_distmod_PEAK'] / 5. -
2.)
data['SCHULTHEIS_DIST'] = dist['SCHULTHEIS_dist']
elif adddist:
warnings.warn(
"Distances not added because matching requires the uninstalled esutil module",
RuntimeWarning)
if _ESUTIL_LOADED and (path._APOGEE_REDUX.lower() == 'current' \
or 'l3' in path._APOGEE_REDUX.lower() \
or int(path._APOGEE_REDUX[1:]) > 600):
data = esutil.numpy_util.add_fields(data, [('METALS', float),
('ALPHAFE', float)])
data['METALS'] = data['PARAM'][:, paramIndx('metals')]
data['ALPHAFE'] = data['PARAM'][:, paramIndx('alpha')]
if not xmatch is None:
return (data, ma)
else:
return data
def allStar(rmcommissioning=True,
main=False,
exclude_star_bad=False,
exclude_star_warn=False,
ak=True,
akvers='targ',
rmnovisits=False,
adddist=False,
distredux=None,
rmdups=False,
raw=False):
"""
NAME:
allStar
PURPOSE:
read the allStar file
INPUT:
rmcommissioning= (default: True) if True, only use data obtained after commissioning
main= (default: False) if True, only select stars in the main survey
exclude_star_bad= (False) if True, remove stars with the STAR_BAD flag set in ASPCAPFLAG
exclude_star_warn= (False) if True, remove stars with the STAR_WARN flag set in ASPCAPFLAG
ak= (default: True) only use objects for which dereddened mags exist
akvers= 'targ' (default) or 'wise': use target AK (AK_TARG) or AK derived from all-sky WISE (AK_WISE)
rmnovisits= (False) if True, remove stars with no good visits (to go into the combined spectrum); shouldn't be necessary
adddist= (default: False) add distances (DR10/11 Hayden distances, DR12 combined distances)
distredux= (default: DR default) reduction on which the distances are based
rmdups= (False) if True, remove duplicates (very slow)
raw= (False) if True, just return the raw file, read w/ fitsio
OUTPUT:
allStar data
HISTORY:
2013-09-06 - Written - Bovy (IAS)
"""
filePath= path.allStarPath()
if not os.path.exists(filePath):
download.allStar()
#read allStar file
data= fitsio.read(path.allStarPath())
if raw: return data
#Remove duplicates, cache
if rmdups:
dupsFilename= path.allStarPath().replace('.fits','-nodups.fits')
if os.path.exists(dupsFilename):
data= fitsio.read(dupsFilename)
else:
sys.stdout.write('\r'+"Removing duplicates (might take a while) and caching the duplicate-free file ...\r")
sys.stdout.flush()
data= remove_duplicates(data)
#Cache this file for subsequent use of rmdups
fitsio.write(dupsFilename,data,clobber=True)
sys.stdout.write('\r'+_ERASESTR+'\r')
sys.stdout.flush()
#Some cuts
if rmcommissioning:
indx= numpy.array(['apogee.n.c' in s for s in data['APSTAR_ID']])
indx+= numpy.array(['apogee.s.c' in s for s in data['APSTAR_ID']])
data= data[True-indx]
if rmnovisits:
indx= numpy.array([s.strip() != '' for s in data['VISITS']])
data= data[indx]
if main:
indx= mainIndx(data)
data= data[indx]
if akvers.lower() == 'targ':
aktag= 'AK_TARG'
elif akvers.lower() == 'wise':
aktag= 'AK_WISE'
if ak:
data= data[True-numpy.isnan(data[aktag])]
data= data[(data[aktag] > -50.)]
if exclude_star_bad:
data= data[(data['ASPCAPFLAG'] & 2**23) == 0]
if exclude_star_warn:
data= data[(data['ASPCAPFLAG'] & 2**7) == 0]
#Add dereddened J, H, and Ks
aj= data[aktag]*2.5
ah= data[aktag]*1.55
data= esutil.numpy_util.add_fields(data,[('J0', float),
('H0', float),
('K0', float)])
data['J0']= data['J']-aj
data['H0']= data['H']-ah
data['K0']= data['K']-data[aktag]
data['J0'][(data[aktag] <= -50.)]= -9999.9999
data['H0'][(data[aktag] <= -50.)]= -9999.9999
data['K0'][(data[aktag] <= -50.)]= -9999.9999
#Add distances
if adddist:
dist= fitsio.read(path.distPath(),1)
h=esutil.htm.HTM()
m1,m2,d12 = h.match(dist['RA'],dist['DEC'],
data['RA'],data['DEC'],
2./3600.,maxmatch=1)
data= data[m2]
dist= dist[m1]
distredux= path._redux_dr()
if distredux.lower() == 'v302' or distredux.lower() == path._DR10REDUX:
data= esutil.numpy_util.add_fields(data,[('DM05', float),
('DM16', float),
('DM50', float),
('DM84', float),
('DM95', float),
('DMPEAK', float),
('DMAVG', float),
('SIG_DM', float),
('DIST_SOL', float),
('SIG_DISTSOL', float)])
data['DM05']= dist['DM05']
data['DM16']= dist['DM16']
data['DM50']= dist['DM50']
data['DM84']= dist['DM84']
data['DM95']= dist['DM95']
data['DMPEAK']= dist['DMPEAK']
data['DMAVG']= dist['DMAVG']
data['SIG_DM']= dist['SIG_DM']
data['DIST_SOL']= dist['DIST_SOL']/1000.
data['SIG_DISTSOL']= dist['SIG_DISTSOL']/1000.
elif distredux.lower() == path._DR11REDUX:
data= esutil.numpy_util.add_fields(data,[('DISO', float),
('DMASS', float),
('DISO_GAL', float),
('DMASS_GAL', float)])
data['DISO']= dist['DISO'][:,1]
data['DMASS']= dist['DMASS'][:,1]
data['DISO_GAL']= dist['DISO_GAL'][:,1]
data['DMASS_GAL']= dist['DMASS_GAL'][:,1]
elif distredux.lower() == path._DR12REDUX:
data= esutil.numpy_util.add_fields(data,[('HIP_PLX', float),
('HIP_E_PLX', float),
('RC_DIST', float),
('APOKASC_DIST_DIRECT', float),
('BPG_DIST1_MEAN', float),
('HAYDEN_DIST_PEAK', float),
('SCHULTHEIS_DIST', float)])
data['HIP_PLX']= dist['HIP_PLX']
data['HIP_E_PLX']= dist['HIP_E_PLX']
data['RC_DIST']= dist['RC_dist_pc']
data['APOKASC_DIST_DIRECT']= dist['APOKASC_dist_direct_pc']/1000.
data['BPG_DIST1_MEAN']= dist['BPG_dist1_mean']
data['HAYDEN_DIST_PEAK']= 10.**(dist['HAYDEN_distmod_PEAK']/5.-2.)
data['SCHULTHEIS_DIST']= dist['SCHULTHEIS_dist']
if path._APOGEE_REDUX.lower() == 'current' \
or int(path._APOGEE_REDUX[1:]) > 600:
data= esutil.numpy_util.add_fields(data,[('METALS', float),
('ALPHAFE', float)])
data['METALS']= data['PARAM'][:,paramIndx('metals')]
data['ALPHAFE']= data['PARAM'][:,paramIndx('alpha')]
return data
raw= (False) if True, just return the raw file, read w/ fitsio
<<<<<<< HEAD
dr = (None) data release
=======
mjd= (58104) MJD of version for monthly internal pipeline runs
xmatch= (None) uses gaia_tools.xmatch.cds to x-match to an external catalog (eg., Gaia DR2 for xmatch='vizier:I/345/gaia2') and caches the result for re-use; requires jobovy/gaia_tools
+gaia_tools.xmatch.cds keywords
>>>>>>> upstream/master
OUTPUT:
allStar data[,xmatched table]
HISTORY:
2013-09-06 - Written - Bovy (IAS)
<<<<<<< HEAD
2016-11-23 - Modified - Price-Jones (UofT)
"""
filePath= path.allStarPath(dr=dr)
=======
2018-01-22 - Edited for new monthly pipeline runs - Bovy (UofT)
2018-05-09 - Add xmatch - Bovy (UofT)
2018-10-20 - Add use_astroNN option - Bovy (UofT)
2018-02-15 - Add astroNN distances and corresponding options - Bovy (UofT)
2018-02-16 - Add astroNN ages and corresponding options - Bovy (UofT)
"""
filePath= path.allStarPath(mjd=mjd)
>>>>>>> upstream/master
if not os.path.exists(filePath):
download.allStar(mjd=mjd)
#read allStar file
data= fitsread(path.allStarPath(mjd=mjd))
#Add astroNN? astroNN file matched line-by-line to allStar, so match here
# [ages file not matched line-by-line]
def allStar(rmcommissioning=True,
main=False,
exclude_star_bad=False,
exclude_star_warn=False,
ak=True,
akvers='targ',
rmnovisits=False,
use_astroNN=False,
use_astroNN_abundances=False,
use_astroNN_distances=False,
use_astroNN_ages=False,
adddist=False,
distredux=None,
rmdups=False,
raw=False,
mjd=58104,
xmatch=None,**kwargs):
"""
NAME:
allStar
PURPOSE:
read the allStar file
INPUT:
rmcommissioning= (default: True) if True, only use data obtained after commissioning
main= (default: False) if True, only select stars in the main survey
exclude_star_bad= (False) if True, remove stars with the STAR_BAD flag set in ASPCAPFLAG
exclude_star_warn= (False) if True, remove stars with the STAR_WARN flag set in ASPCAPFLAG
ak= (default: True) only use objects for which dereddened mags exist
akvers= 'targ' (default) or 'wise': use target AK (AK_TARG) or AK derived from all-sky WISE (AK_WISE)
rmnovisits= (False) if True, remove stars with no good visits (to go into the combined spectrum); shouldn't be necessary
use_astroNN= (False) if True, swap in astroNN (Leung & Bovy 2019a) parameters (get placed in, e.g., TEFF and TEFF_ERR), astroNN distances (Leung & Bovy 2019b), and astroNN ages (Mackereth, Bovy, Leung, et al. (2019)
use_astroNN_abundances= (False) only swap in astroNN parameters and abundances, not distances and ages
use_astroNN_distances= (False) only swap in astroNN distances, not parameters and abundances and ages
use_astroNN_ages= (False) only swap in astroNN ages, not parameters and abundances and distances
adddist= (default: False) add distances (DR10/11 Hayden distances, DR12 combined distances)
distredux= (default: DR default) reduction on which the distances are based
rmdups= (False) if True, remove duplicates (very slow)
raw= (False) if True, just return the raw file, read w/ fitsio
mjd= (58104) MJD of version for monthly internal pipeline runs
xmatch= (None) uses gaia_tools.xmatch.cds to x-match to an external catalog (eg., Gaia DR2 for xmatch='vizier:I/345/gaia2') and caches the result for re-use; requires jobovy/gaia_tools
+gaia_tools.xmatch.cds keywords
OUTPUT:
allStar data[,xmatched table]
HISTORY:
2013-09-06 - Written - Bovy (IAS)
2018-01-22 - Edited for new monthly pipeline runs - Bovy (UofT)
2018-05-09 - Add xmatch - Bovy (UofT)
2018-10-20 - Add use_astroNN option - Bovy (UofT)
2018-02-15 - Add astroNN distances and corresponding options - Bovy (UofT)
2018-02-16 - Add astroNN ages and corresponding options - Bovy (UofT)
"""
filePath= path.allStarPath(mjd=mjd)
if not os.path.exists(filePath):
download.allStar(mjd=mjd)
#read allStar file
data= fitsread(path.allStarPath(mjd=mjd))
#Add astroNN? astroNN file matched line-by-line to allStar, so match here
# [ages file not matched line-by-line]
if use_astroNN or kwargs.get('astroNN',False) or use_astroNN_abundances:
_warn_astroNN_abundances()
astroNNdata= astroNN()
data= _swap_in_astroNN(data,astroNNdata)
if use_astroNN or kwargs.get('astroNN',False) or use_astroNN_distances:
_warn_astroNN_distances()
astroNNdata= astroNNDistances()
data= _add_astroNN_distances(data,astroNNdata)
if use_astroNN or kwargs.get('astroNN',False) or use_astroNN_ages:
_warn_astroNN_ages()
astroNNdata= astroNNAges()
data= _add_astroNN_ages(data,astroNNdata)
if raw: return data
#Remove duplicates, cache
if rmdups:
dupsFilename= path.allStarPath(mjd=mjd).replace('.fits','-nodups.fits')
if os.path.exists(dupsFilename):
data= fitsread(dupsFilename)
else:
sys.stdout.write('\r'+"Removing duplicates (might take a while) and caching the duplicate-free file ...\r")
sys.stdout.flush()
data= remove_duplicates(data)
#Cache this file for subsequent use of rmdups
fitswrite(dupsFilename,data,clobber=True)
sys.stdout.write('\r'+_ERASESTR+'\r')
sys.stdout.flush()
if not xmatch is None:
from gaia_tools.load import _xmatch_cds
if rmdups:
matchFilePath= dupsFilename
else:
matchFilePath= filePath
if use_astroNN_ages:
matchFilePath= matchFilePath.replace('rc-','rc-astroNN-ages-')
ma,mai= _xmatch_cds(data,xmatch,filePath,**kwargs)
data= data[mai]
#Some cuts
if rmcommissioning:
try:
indx= numpy.array(['apogee.n.c'.encode('utf-8') in s for s in data['APSTAR_ID']])
indx+= numpy.array(['apogee.s.c'.encode('utf-8') in s for s in data['APSTAR_ID']])
except TypeError:
indx= numpy.array(['apogee.n.c' in s for s in data['APSTAR_ID']])
indx+= numpy.array(['apogee.s.c' in s for s in data['APSTAR_ID']])
data= data[True^indx]
if not xmatch is None: ma= ma[True^indx]
if rmnovisits:
indx= numpy.array([s.strip() != '' for s in data['VISITS']])
data= data[indx]
if not xmatch is None: ma= ma[indx]
if main:
indx= mainIndx(data)
data= data[indx]
if not xmatch is None: ma= ma[indx]
if akvers.lower() == 'targ':
aktag= 'AK_TARG'
elif akvers.lower() == 'wise':
aktag= 'AK_WISE'
if ak:
if not xmatch is None: ma= ma[True^numpy.isnan(data[aktag])]
data= data[True^numpy.isnan(data[aktag])]
if not xmatch is None: ma= ma[(data[aktag] > -50.)]
data= data[(data[aktag] > -50.)]
if exclude_star_bad:
if not xmatch is None: ma= ma[(data['ASPCAPFLAG'] & 2**23) == 0]
data= data[(data['ASPCAPFLAG'] & 2**23) == 0]
if exclude_star_warn:
if not xmatch is None: ma= ma[(data['ASPCAPFLAG'] & 2**7) == 0]
data= data[(data['ASPCAPFLAG'] & 2**7) == 0]
#Add dereddened J, H, and Ks
aj= data[aktag]*2.5
ah= data[aktag]*1.55
if _ESUTIL_LOADED:
data= esutil.numpy_util.add_fields(data,[('J0', float),
('H0', float),
('K0', float)])
data['J0']= data['J']-aj
data['H0']= data['H']-ah
data['K0']= data['K']-data[aktag]
data['J0'][(data[aktag] <= -50.)]= -9999.9999
data['H0'][(data[aktag] <= -50.)]= -9999.9999
data['K0'][(data[aktag] <= -50.)]= -9999.9999
else:
warnings.warn("Extinction-corrected J,H,K not added because esutil is not installed",RuntimeWarning)
#Add distances
if adddist and _ESUTIL_LOADED:
dist= fitsread(path.distPath(),1)
h=esutil.htm.HTM()
m1,m2,d12 = h.match(dist['RA'],dist['DEC'],
data['RA'],data['DEC'],
2./3600.,maxmatch=1)
data= data[m2]
if not xmatch is None: ma= ma[m2]
dist= dist[m1]
distredux= path._redux_dr()
if distredux.lower() == 'v302' or distredux.lower() == path._DR10REDUX:
data= esutil.numpy_util.add_fields(data,[('DM05', float),
('DM16', float),
('DM50', float),
('DM84', float),
('DM95', float),
('DMPEAK', float),
('DMAVG', float),
('SIG_DM', float),
('DIST_SOL', float),
('SIG_DISTSOL', float)])
data['DM05']= dist['DM05']
data['DM16']= dist['DM16']
data['DM50']= dist['DM50']
data['DM84']= dist['DM84']
data['DM95']= dist['DM95']
data['DMPEAK']= dist['DMPEAK']
data['DMAVG']= dist['DMAVG']
data['SIG_DM']= dist['SIG_DM']
data['DIST_SOL']= dist['DIST_SOL']/1000.
data['SIG_DISTSOL']= dist['SIG_DISTSOL']/1000.
elif distredux.lower() == path._DR11REDUX:
data= esutil.numpy_util.add_fields(data,[('DISO', float),
('DMASS', float),
('DISO_GAL', float),
('DMASS_GAL', float)])
data['DISO']= dist['DISO'][:,1]
data['DMASS']= dist['DMASS'][:,1]
data['DISO_GAL']= dist['DISO_GAL'][:,1]
data['DMASS_GAL']= dist['DMASS_GAL'][:,1]
elif distredux.lower() == path._DR12REDUX:
data= esutil.numpy_util.add_fields(data,[('HIP_PLX', float),
('HIP_E_PLX', float),
('RC_DIST', float),
('APOKASC_DIST_DIRECT', float),
('BPG_DIST1_MEAN', float),
('HAYDEN_DIST_PEAK', float),
('SCHULTHEIS_DIST', float)])
data['HIP_PLX']= dist['HIP_PLX']
data['HIP_E_PLX']= dist['HIP_E_PLX']
data['RC_DIST']= dist['RC_dist_pc']
data['APOKASC_DIST_DIRECT']= dist['APOKASC_dist_direct_pc']/1000.
data['BPG_DIST1_MEAN']= dist['BPG_dist1_mean']
data['HAYDEN_DIST_PEAK']= 10.**(dist['HAYDEN_distmod_PEAK']/5.-2.)
data['SCHULTHEIS_DIST']= dist['SCHULTHEIS_dist']
elif adddist:
warnings.warn("Distances not added because matching requires the uninstalled esutil module",RuntimeWarning)
if _ESUTIL_LOADED and (path._APOGEE_REDUX.lower() == 'current' \
or 'l3' in path._APOGEE_REDUX.lower() \
or int(path._APOGEE_REDUX[1:]) > 600):
data= esutil.numpy_util.add_fields(data,[('METALS', float),
('ALPHAFE', float)])
data['METALS']= data['PARAM'][:,paramIndx('metals')]
data['ALPHAFE']= data['PARAM'][:,paramIndx('alpha')]
if not xmatch is None:
return (data,ma)
else:
return data
def allStar(rmcommissioning=True,
main=False,
ak=True,
akvers='targ',
rmnovisits=False,
adddist=False,
distredux='v302',
rmdups=True):
"""
NAME:
allStar
PURPOSE:
read the allStar file
INPUT:
rmcommissioning= (default: True) if True, only use data obtained after commissioning
main= (default: False) if True, only select stars in the main survey
ak= (default: True) only use objects for which dereddened mags exist
akvers= 'targ' (default) or 'wise': use target AK (AK_TARG) or AK derived from all-sky WISE (AK_WISE)
rmnovisits= (False) if True, remove stars with no good visits (to go into the combined spectrum); shouldn't be necessary
adddist= (default: False) add distances from Michael Hayden
distredux= (default: v302) reduction on which the distances are based
rmdups= (True) if True, remove duplicates
OUTPUT:
allStar data
HISTORY:
2013-09-06 - Written - Bovy (IAS)
"""
#read allStar file
data= fitsio.read(path.allStarPath())
#Some cuts
if rmcommissioning:
indx= numpy.array(['apogee.n.c' in s for s in data['APSTAR_ID']])
indx+= numpy.array(['apogee.s.c' in s for s in data['APSTAR_ID']])
data= data[True-indx]
if rmnovisits:
indx= numpy.array([s.strip() != '' for s in data['VISITS']])
data= data[indx]
if main:
indx= mainIndx(data)
data= data[indx]
if akvers.lower() == 'targ':
aktag= 'AK_TARG'
elif akvers.lower() == 'wise':
aktag= 'AK_WISE'
if ak:
data= data[True-numpy.isnan(data[aktag])]
data= data[(data[aktag] > -50.)]
if rmdups:
data= remove_duplicates(data)
#Add dereddened J, H, and Ks
aj= data[aktag]*2.5
ah= data[aktag]*1.55
data= esutil.numpy_util.add_fields(data,[('J0', float),
('H0', float),
('K0', float)])
data['J0']= data['J']-aj
data['H0']= data['H']-ah
data['K0']= data['K']-data[aktag]
data['J0'][(data[aktag] <= -50.)]= -9999.9999
data['H0'][(data[aktag] <= -50.)]= -9999.9999
data['K0'][(data[aktag] <= -50.)]= -9999.9999
#Add distances
if adddist:
dist= fitsio.read(path.distPath(redux=distredux))
h=esutil.htm.HTM()
m1,m2,d12 = h.match(dist['RA'],dist['DEC'],
data['RA'],data['DEC'],
2./3600.,maxmatch=1)
data= data[m2]
dist= dist[m1]
if distredux.lower() == 'v302':
data= esutil.numpy_util.add_fields(data,[('DM05', float),
('DM16', float),
('DM50', float),
('DM84', float),
('DM95', float),
('DMPEAK', float),
('DMAVG', float),
('SIG_DM', float),
('DIST_SOL', float),
('SIG_DISTSOL', float)])
data['DM05']= dist['DM05']
data['DM16']= dist['DM16']
data['DM50']= dist['DM50']
data['DM84']= dist['DM84']
data['DM95']= dist['DM95']
data['DMPEAK']= dist['DMPEAK']
data['DMAVG']= dist['DMAVG']
data['SIG_DM']= dist['SIG_DM']
data['DIST_SOL']= dist['DIST_SOL']/1000.
data['SIG_DISTSOL']= dist['SIG_DISTSOL']/1000.
elif distredux.lower() == 'v402':
data= esutil.numpy_util.add_fields(data,[('DISO', float),
('DMASS', float),
('DISO_GAL', float),
('DMASS_GAL', float)])
data['DISO']= dist['DISO'][:,1]
data['DMASS']= dist['DMASS'][:,1]
data['DISO_GAL']= dist['DISO_GAL'][:,1]
data['DMASS_GAL']= dist['DMASS_GAL'][:,1]
return data