16:"FAILFIT" # Fitting for stellar parameters failed on pixel
}
def rgsample(dr='13'):
data= apread.allStar(main=True,exclude_star_bad=True,exclude_star_warn=True,rmdups=True)
jk= data['J0']-data['K0']
z= isodist.FEH2Z(data['METALS'],zsolar=0.017)
z[z > 0.024]= 0.024
logg= data['LOGG']
indx= ((jk >= 0.8)#+(z < rcmodel.jkzcut(jk-0.1,upper=False))
+(logg > rcmodel.loggteffcut(data['TEFF'],z,upper=True)))
rgindx=indx*(data['METALS'] > -.8)
return data[rgindx]
# Chosen set of bits on which to mask
badcombpixmask = bitmask.badpixmask()
badcombpixmask += 2**bitmask.apogee_pixmask_int("SIG_SKYLINE")
elems = ['C','N','O','Na','Mg','Al','Si','S','K','Ca','Ti','V','Mn','Fe','Ni']
# Functions to access particular sample types
readfn = {'clusters' : read_caldata, # Sample of clusters
'OCs': read_caldata, # Sample of open clusters
'GCs': read_caldata, # Sample of globular clusters
'red_clump' : apread.rcsample, # Sample of red clump stars
'red_giant' : rgsample # Sample of red giant stars
}
# List of accepted keys to do slice in
keyList = ['RA','DEC','GLON','GLAT','TEFF','LOGG','TEFF_ERR','LOGG_ERR',
def read_spectra(cluster,
teffmin=4000.,
teffmax=5000.,
cont_type='cannon',
cont_deg=4):
"""
NAME:
read_spectra
PURPOSE:
Read the APOGEE spectra and their errors for stars in a given cluster
INPUT:
cluster - Name of the cluster (name in one of the data files)
teffmin= (4000.) minimum temperature
teffmax= (5000.) maximum temperature
cont_type = ('cannon') type of continuum normalization to perform
cont_deg= (4) degree polynomial to fit for continuum normalization
OUTPUT:
(data, spec, specerr) - (full data structure, spectra [nspec,nlam], spectral uncertainties [nspec,nlam]) nlam=7214 on ASPCAP grid
HISTORY:
2015-08-13 - Written based on some older code - Bovy (UofT)
"""
if cluster.upper() in _GCS:
data = read_meszarosgcdata()
else:
data = read_caldata()
# Cut to just this cluster and temperature range
if 'rc' in cluster.lower():
# Only for NGC 6819
rc = True
cluster = cluster[:-2]
else:
rc = False
data = data[data['CLUSTER'] == cluster.upper()]
data= data[(data['TEFF'] < teffmax)\
*(data['TEFF'] > teffmin)]
if cluster.lower() == 'n6819':
g4CN = good4CN(cluster, data)
g4CN[10] = False # another one, by hand!
if rc:
data = data[True - g4CN] # Just those!
else:
data = data[g4CN] # Just those!
# Load all the spectra
nspec = len(data)
spec = numpy.zeros((nspec, 7214))
specerr = numpy.zeros((nspec, 7214))
# Setup bad pixel mask
badcombpixmask= bitmask.badpixmask()\
+2**bitmask.apogee_pixmask_int("SIG_SKYLINE")
for ii in range(nspec):
sys.stdout.write('\r' + "Loading spectrum %i / %i ...\r" %
(ii + 1, nspec))
sys.stdout.flush()
spec[ii] = apread.apStar(data['LOCATION_ID'][ii],
data['ID'][ii],
ext=1,
header=False,
aspcapWavegrid=True)[_COMBINED_INDEX]
specerr[ii] = apread.apStar(data['LOCATION_ID'][ii],
data['ID'][ii],
ext=2,
header=False,
aspcapWavegrid=True)[_COMBINED_INDEX]
# Inflate uncertainties for bad pixels
mask = apread.apStar(data['LOCATION_ID'][ii],
data['ID'][ii],
ext=3,
header=False,
aspcapWavegrid=True)[_COMBINED_INDEX]
specerr[ii,(mask & (badcombpixmask)) != 0]+=\
100.*numpy.mean(spec[ii,True-numpy.isnan(spec[ii])])
# Also inflate pixels with high SNR to 0.5%
highsnr = spec[ii] / specerr[ii] > 200.
specerr[ii, highsnr] = 0.005 * numpy.fabs(spec[ii, highsnr])
# Continuum-normalize
cont = continuum.fit(spec[ii],
specerr[ii],
type=cont_type,
deg=cont_deg)
spec[ii] /= cont
specerr[ii] /= cont
specerr[ii, highsnr] = 0.005 # like standard APOGEE reduction
sys.stdout.write('\r' + _ERASESTR + '\r')
sys.stdout.flush()
return (data, spec, specerr)
def read_batch_of_spectra(batch_count, batch_size=10000):
'''
Download a bunch of *combined* spectra in one go. Set the uncertainties to a large
value in bad pixels, normalize, and save the batch locally.
'''
# read in the catalog catalog
catalog, fibers = read_apogee_catalog()
catalog = catalog[batch_count * batch_size:(batch_count + 1) * batch_size]
fibers = fibers[batch_count * batch_size:(batch_count + 1) * batch_size]
_COMBINED_INDEX = 1
nspec = len(catalog)
spec = np.zeros((nspec, 7214))
specerr = np.zeros((nspec, 7214))
# Set up bad pixel mask
badcombpixmask = bitmask.badpixmask() + 2**bitmask.apogee_pixmask_int(
"SIG_SKYLINE")
# loop through the individual targets
for ii in range(nspec):
field = catalog['FIELD'][ii].decode()
ap_id = catalog['APOGEE_ID'][ii].decode()
loc_id = catalog['LOCATION_ID'][ii]
print('processing target %d with id %s' % (ii, ap_id))
try:
if loc_id == 1:
temp1 = apread.apStar(field,
ap_id,
ext=1,
header=False,
aspcapWavegrid=True)
temp2 = apread.apStar(field,
ap_id,
ext=2,
header=False,
aspcapWavegrid=True)
temp3 = apread.apStar(field,
ap_id,
ext=3,
header=False,
aspcapWavegrid=True)
else:
temp1 = apread.apStar(loc_id,
ap_id,
ext=1,
header=False,
aspcapWavegrid=True)
temp2 = apread.apStar(loc_id,
ap_id,
ext=2,
header=False,
aspcapWavegrid=True)
temp3 = apread.apStar(loc_id,
ap_id,
ext=3,
header=False,
aspcapWavegrid=True)
if temp1.shape[0] > 6000:
spec[ii] = temp1
specerr[ii] = temp2
mask = temp3
else:
spec[ii] = temp1[_COMBINED_INDEX]
specerr[ii] = temp2[_COMBINED_INDEX]
mask = temp3[_COMBINED_INDEX]
# Inflate uncertainties for bad pixels
specerr[ii, (mask & (badcombpixmask)) != 0] += \
100. * np.mean(spec[ii, np.isfinite(spec[ii])])
# Inflate pixels with high SNR to 0.5
highsnr = spec[ii] / specerr[ii] > 200.
specerr[ii, highsnr] = 0.005 * np.fabs(spec[ii, highsnr])
# Continuum-normalize
cont = utils.get_apogee_continuum(wavelength=wavelength,
spec=spec[ii],
spec_err=specerr[ii],
cont_pixels=cont_pixels)
spec[ii] /= cont
specerr[ii] /= cont
specerr[ii, highsnr] = 0.005
except OSError:
print('target could not be found!')
continue
# save spectra
np.savez('spectra/apogee_all_spectra_' + str(batch_count) + '.npz',
wavelength=wavelength,
spectra=spec,
spec_err=specerr,
apogee_id=np.array(catalog["APOGEE_ID"]),
apogee_fiber_id=fibers)
def get_visit_spectra_individual_object(apogee_id,
allvisit_cat=None,
save_local=False):
'''
Download the visit spectra for an individual object.
Get the v_helios from the allStar catalog, which are more accurate than
the values reported in the visit spectra fits files.
Use the barycentric correction to shift spectra to the heliocentric frame.
Do a preliminary normalization using Bovy's visit spectrum normalization tool.
It's critical that the spectra be renormalized prior to fitting using the
spectral_model.get_apogee_continuum() function for self-consistency.
apogee_id = byte-like object, i.e. b'2M06133561+2433362'
##### NOTE #####
I had to make some minor changes to Bovy's APOGEE package to get it to
properly read in the visit spectra. In particular, after the 3rd line of
the function apVisit() in that package's /apogee/tools/read.py file, I added
the following lines:
if ext == 0:
from astropy.io import fits
hdulist = fits.open(filePath)
header = hdulist[0].header
hdulist.close()
return header
I think this is the only change that was necessary, but it's possible I'm
forgetting another change.
'''
import apogee.tools.read as apread
from apogee.tools import bitmask
from apogee.spec import continuum
import astropy
if allvisit_cat is None:
allvisit_cat = apread.allVisit(rmcommissioning=False, ak=False)
where_visits = np.where(allvisit_cat['APOGEE_ID'] == apogee_id)[0]
plate_ids = np.array([int(i) for i in allvisit_cat[where_visits]['PLATE']])
fiberids = allvisit_cat[where_visits]['FIBERID']
mjds = allvisit_cat[where_visits]['MJD']
JDs = allvisit_cat[where_visits]['JD']
vhelios_accurate = allvisit_cat[where_visits]['VHELIO']
vhelios_synth = allvisit_cat[where_visits]['SYNTHVHELIO']
snrs = allvisit_cat[where_visits]['SNR']
BCs = allvisit_cat[where_visits]['BC']
badcombpixmask = bitmask.badpixmask() + 2**bitmask.apogee_pixmask_int(
"SIG_SKYLINE")
all_spec, all_err, all_snr, all_hjd, all_vhelio = [], [], [], [], []
for i, pid in enumerate(plate_ids):
try:
spec = apread.apVisit(pid,
mjds[i],
fiberids[i],
ext=1,
header=False)
specerr = apread.apVisit(pid,
mjds[i],
fiberids[i],
ext=2,
header=False)
wave = apread.apVisit(pid,
mjds[i],
fiberids[i],
ext=4,
header=False)
mask = apread.apVisit(pid,
mjds[i],
fiberids[i],
ext=3,
header=False)
masterheader = apread.apVisit(pid,
mjds[i],
fiberids[i],
ext=0,
header=True)
badpix = (mask & (badcombpixmask)) != 0
if np.sum(badpix) / len(badpix) > 0.5:
print('too many bad pixels!')
continue # if 50% or more of the pixels are bad, don't bother.
specerr[badpix] = 100 * np.median(spec)
# a small fraction of the visit spectra are on a different wavelength
# grid than normal (maybe commissioning?). In any case, interpolate them
# to the wavelength grid expected by Bovy's visit normalization routine.
if len(wave) != 12288:
print('fixing wavelength...')
standard_grid = utils.load_visit_wavelength()
spec = np.interp(standard_grid, wave, spec)
specerr = np.interp(standard_grid, wave, specerr)
wave = np.copy(standard_grid)
# preliminary normalization using Bovy's visit normalization routine.
cont = continuum.fitApvisit(spec, specerr, wave)
specnorm, errnorm = spec / cont, specerr / cont
# correct for Earth's orbital motion.
spec_shift = utils.doppler_shift(wavelength=wave,
flux=specnorm,
dv=BCs[i])
spec_err_shift = utils.doppler_shift(wavelength=wave,
flux=errnorm,
dv=BCs[i])
# interpolate to the standard wavelength grid we use for combined spectra.
interp_spec = np.interp(wavelength, wave, spec_shift)
interp_err = np.interp(wavelength, wave, spec_err_shift)
# truncate SNR at 200
interp_err[interp_err < 0.005] = 0.005
all_spec.append(interp_spec)
all_err.append(interp_err)
# Ideally, get the Julian date of the observations in the heliocentric frame.
# Sometimes this isn't available; in that case, get the Julian date in Earth's
# frame. These differ by at most 8 minutes, so not a big deal.
try:
all_hjd.append(masterheader['HJD'])
except KeyError:
all_hjd.append(JDs[i])
# There are a few cases where the v_helios from the allvisit catalog are clearly wrong.
if np.abs(vhelios_accurate[i] > 1000) and np.abs(
vhelios_synth[i] < 1000):
vhel = vhelios_synth[i]
else:
vhel = vhelios_accurate[i]
all_snr.append(snrs[i])
all_vhelio.append(vhel)
except astropy.io.fits.verify.VerifyError:
print('there was a verification error')
continue
all_spec, all_err, all_snr, all_hjd, all_vhelio = np.array(all_spec), \
np.array(all_err), np.array(all_snr), np.array(all_hjd), np.array(all_vhelio)
msk = np.argsort(all_hjd)
all_spec, all_err, all_snr, all_hjd, all_vhelio = all_spec[msk], all_err[msk], \
all_snr[msk], all_hjd[msk], all_vhelio[msk]
if save_local:
np.savez('spectra/visit/visit_spectra_ap_id_' +
str(apogee_id.decode()) + '_.npz',
spectra=all_spec,
spec_errs=all_err,
snrs=all_snr,
hjds=all_hjd,
vhelios=all_vhelio)
return all_spec, all_err, all_snr, all_hjd, all_vhelio
def get_combined_spectrum_single_object(apogee_id,
catalog=None,
save_local=False):
'''
apogee_id should be a byte-like object; i.e b'2M13012770+5754582'
This downloads a single combined spectrum and the associated error array,
and it normalizes both.
'''
# read in the allStar catalog if you haven't already
if catalog is None:
catalog, fibers = read_apogee_catalog()
# Set up bad pixel mask
badcombpixmask = bitmask.badpixmask() + 2**bitmask.apogee_pixmask_int(
"SIG_SKYLINE")
_COMBINED_INDEX = 1
msk = np.where(catalog['APOGEE_ID'] == apogee_id)[0]
if not len(msk):
raise ValueError(
'the desired Apogee ID was not found in the allStar catalog.')
field = catalog['FIELD'][msk[0]].decode()
ap_id = apogee_id.decode()
loc_id = catalog['LOCATION_ID'][msk[0]]
if loc_id == 1:
temp1 = apread.apStar(field,
ap_id,
ext=1,
header=False,
aspcapWavegrid=True)
temp2 = apread.apStar(field,
ap_id,
ext=2,
header=False,
aspcapWavegrid=True)
temp3 = apread.apStar(field,
ap_id,
ext=3,
header=False,
aspcapWavegrid=True)
else:
temp1 = apread.apStar(loc_id,
ap_id,
ext=1,
header=False,
aspcapWavegrid=True)
temp2 = apread.apStar(loc_id,
ap_id,
ext=2,
header=False,
aspcapWavegrid=True)
temp3 = apread.apStar(loc_id,
ap_id,
ext=3,
header=False,
aspcapWavegrid=True)
if temp1.shape[0] > 6000:
spec = temp1
specerr = temp2
mask = temp3
else:
spec = temp1[_COMBINED_INDEX]
specerr = temp2[_COMBINED_INDEX]
mask = temp3[_COMBINED_INDEX]
# Inflate uncertainties for bad pixels
specerr[(mask & (badcombpixmask)) != 0] += 100 * np.mean(
spec[np.isfinite(spec)])
# Inflate pixels with high SNR to 0.5
highsnr = spec / specerr > 200.
specerr[highsnr] = 0.005 * np.fabs(spec[highsnr])
# Continuum-normalize
cont = utils.get_apogee_continuum(wavelength=wavelength,
spec=spec,
spec_err=specerr,
cont_pixels=cont_pixels)
spec /= cont
specerr /= cont
specerr[highsnr] = 0.005
if save_local:
np.savez('spectra/combined/spectrum_ap_id_' + str(apogee_id.decode()) +
'_.npz',
spectrum=spec,
spec_err=specerr)
return spec, specerr
def read_spectra(cluster,teffmin=4000.,teffmax=5000.,cont_type='cannon',
cont_deg=4):
"""
NAME:
read_spectra
PURPOSE:
Read the APOGEE spectra and their errors for stars in a given cluster
INPUT:
cluster - Name of the cluster (name in one of the data files)
teffmin= (4000.) minimum temperature
teffmax= (5000.) maximum temperature
cont_type = ('cannon') type of continuum normalization to perform
cont_deg= (4) degree polynomial to fit for continuum normalization
OUTPUT:
(data, spec, specerr) - (full data structure, spectra [nspec,nlam], spectral uncertainties [nspec,nlam]) nlam=7214 on ASPCAP grid
HISTORY:
2015-08-13 - Written based on some older code - Bovy (UofT)
"""
if cluster.upper() in _GCS:
data= read_meszarosgcdata()
else:
data= read_caldata()
# Cut to just this cluster and temperature range
if 'rc' in cluster.lower():
# Only for NGC 6819
rc= True
cluster= cluster[:-2]
else:
rc= False
data= data[data['CLUSTER'] == cluster.upper()]
data= data[(data['TEFF'] < teffmax)\
*(data['TEFF'] > teffmin)]
if cluster.lower() == 'n6819':
g4CN= good4CN(cluster,data)
g4CN[10]= False # another one, by hand!
if rc:
data= data[True-g4CN] # Just those!
else:
data= data[g4CN] # Just those!
# Load all the spectra
nspec= len(data)
spec= numpy.zeros((nspec,7214))
specerr= numpy.zeros((nspec,7214))
# Setup bad pixel mask
badcombpixmask= bitmask.badpixmask()\
+2**bitmask.apogee_pixmask_int("SIG_SKYLINE")
for ii in range(nspec):
sys.stdout.write('\r'+"Loading spectrum %i / %i ...\r" % (ii+1,nspec))
sys.stdout.flush()
spec[ii]= apread.apStar(data['LOCATION_ID'][ii],
data['ID'][ii],
ext=1,header=False,
aspcapWavegrid=True)[_COMBINED_INDEX]
specerr[ii]= apread.apStar(data['LOCATION_ID'][ii],
data['ID'][ii],
ext=2,header=False,
aspcapWavegrid=True)[_COMBINED_INDEX]
# Inflate uncertainties for bad pixels
mask= apread.apStar(data['LOCATION_ID'][ii],
data['ID'][ii],
ext=3,header=False,
aspcapWavegrid=True)[_COMBINED_INDEX]
specerr[ii,(mask & (badcombpixmask)) != 0]+=\
100.*numpy.mean(spec[ii,True-numpy.isnan(spec[ii])])
# Also inflate pixels with high SNR to 0.5%
highsnr= spec[ii]/specerr[ii] > 200.
specerr[ii,highsnr]= 0.005*numpy.fabs(spec[ii,highsnr])
# Continuum-normalize
cont= continuum.fit(spec[ii],specerr[ii],type=cont_type,deg=cont_deg)
spec[ii]/= cont
specerr[ii]/= cont
specerr[ii,highsnr]= 0.005 # like standard APOGEE reduction
sys.stdout.write('\r'+_ERASESTR+'\r')
sys.stdout.flush()
return (data,spec,specerr)
