} def rgsample(dr='13'): data= apread.allStar(main=True,exclude_star_bad=True,exclude_star_warn=True,rmdups=True) jk= data['J0']-data['K0'] z= isodist.FEH2Z(data['METALS'],zsolar=0.017) z[z > 0.024]= 0.024 logg= data['LOGG'] indx= ((jk >= 0.8)#+(z < rcmodel.jkzcut(jk-0.1,upper=False)) +(logg > rcmodel.loggteffcut(data['TEFF'],z,upper=True))) rgindx=indx*(data['METALS'] > -.8) return data[rgindx] # Chosen set of bits on which to mask badcombpixmask = bitmask.badpixmask() badcombpixmask += 2**bitmask.apogee_pixmask_int("SIG_SKYLINE") elems = ['C','N','O','Na','Mg','Al','Si','S','K','Ca','Ti','V','Mn','Fe','Ni'] # Functions to access particular sample types readfn = {'clusters' : read_caldata, # Sample of clusters 'OCs': read_caldata, # Sample of open clusters 'GCs': read_caldata, # Sample of globular clusters 'red_clump' : apread.rcsample, # Sample of red clump stars 'red_giant' : rgsample # Sample of red giant stars } # List of accepted keys to do slice in keyList = ['RA','DEC','GLON','GLAT','TEFF','LOGG','TEFF_ERR','LOGG_ERR', 'AL_H','CA_H','C_H','FE_H','K_H','MG_H','MN_H','NA_H','NI_H',
def read_spectra(cluster, teffmin=4000., teffmax=5000., cont_type='cannon', cont_deg=4): """ NAME: read_spectra PURPOSE: Read the APOGEE spectra and their errors for stars in a given cluster INPUT: cluster - Name of the cluster (name in one of the data files) teffmin= (4000.) minimum temperature teffmax= (5000.) maximum temperature cont_type = ('cannon') type of continuum normalization to perform cont_deg= (4) degree polynomial to fit for continuum normalization OUTPUT: (data, spec, specerr) - (full data structure, spectra [nspec,nlam], spectral uncertainties [nspec,nlam]) nlam=7214 on ASPCAP grid HISTORY: 2015-08-13 - Written based on some older code - Bovy (UofT) """ if cluster.upper() in _GCS: data = read_meszarosgcdata() else: data = read_caldata() # Cut to just this cluster and temperature range if 'rc' in cluster.lower(): # Only for NGC 6819 rc = True cluster = cluster[:-2] else: rc = False data = data[data['CLUSTER'] == cluster.upper()] data= data[(data['TEFF'] < teffmax)\ *(data['TEFF'] > teffmin)] if cluster.lower() == 'n6819': g4CN = good4CN(cluster, data) g4CN[10] = False # another one, by hand! if rc: data = data[True - g4CN] # Just those! else: data = data[g4CN] # Just those! # Load all the spectra nspec = len(data) spec = numpy.zeros((nspec, 7214)) specerr = numpy.zeros((nspec, 7214)) # Setup bad pixel mask badcombpixmask= bitmask.badpixmask()\ +2**bitmask.apogee_pixmask_int("SIG_SKYLINE") for ii in range(nspec): sys.stdout.write('\r' + "Loading spectrum %i / %i ...\r" % (ii + 1, nspec)) sys.stdout.flush() spec[ii] = apread.apStar(data['LOCATION_ID'][ii], data['ID'][ii], ext=1, header=False, aspcapWavegrid=True)[_COMBINED_INDEX] specerr[ii] = apread.apStar(data['LOCATION_ID'][ii], data['ID'][ii], ext=2, header=False, aspcapWavegrid=True)[_COMBINED_INDEX] # Inflate uncertainties for bad pixels mask = apread.apStar(data['LOCATION_ID'][ii], data['ID'][ii], ext=3, header=False, aspcapWavegrid=True)[_COMBINED_INDEX] specerr[ii,(mask & (badcombpixmask)) != 0]+=\ 100.*numpy.mean(spec[ii,True-numpy.isnan(spec[ii])]) # Also inflate pixels with high SNR to 0.5% highsnr = spec[ii] / specerr[ii] > 200. specerr[ii, highsnr] = 0.005 * numpy.fabs(spec[ii, highsnr]) # Continuum-normalize cont = continuum.fit(spec[ii], specerr[ii], type=cont_type, deg=cont_deg) spec[ii] /= cont specerr[ii] /= cont specerr[ii, highsnr] = 0.005 # like standard APOGEE reduction sys.stdout.write('\r' + _ERASESTR + '\r') sys.stdout.flush() return (data, spec, specerr)
def read_batch_of_spectra(batch_count, batch_size=10000): ''' Download a bunch of *combined* spectra in one go. Set the uncertainties to a large value in bad pixels, normalize, and save the batch locally. ''' # read in the catalog catalog catalog, fibers = read_apogee_catalog() catalog = catalog[batch_count * batch_size:(batch_count + 1) * batch_size] fibers = fibers[batch_count * batch_size:(batch_count + 1) * batch_size] _COMBINED_INDEX = 1 nspec = len(catalog) spec = np.zeros((nspec, 7214)) specerr = np.zeros((nspec, 7214)) # Set up bad pixel mask badcombpixmask = bitmask.badpixmask() + 2**bitmask.apogee_pixmask_int( "SIG_SKYLINE") # loop through the individual targets for ii in range(nspec): field = catalog['FIELD'][ii].decode() ap_id = catalog['APOGEE_ID'][ii].decode() loc_id = catalog['LOCATION_ID'][ii] print('processing target %d with id %s' % (ii, ap_id)) try: if loc_id == 1: temp1 = apread.apStar(field, ap_id, ext=1, header=False, aspcapWavegrid=True) temp2 = apread.apStar(field, ap_id, ext=2, header=False, aspcapWavegrid=True) temp3 = apread.apStar(field, ap_id, ext=3, header=False, aspcapWavegrid=True) else: temp1 = apread.apStar(loc_id, ap_id, ext=1, header=False, aspcapWavegrid=True) temp2 = apread.apStar(loc_id, ap_id, ext=2, header=False, aspcapWavegrid=True) temp3 = apread.apStar(loc_id, ap_id, ext=3, header=False, aspcapWavegrid=True) if temp1.shape[0] > 6000: spec[ii] = temp1 specerr[ii] = temp2 mask = temp3 else: spec[ii] = temp1[_COMBINED_INDEX] specerr[ii] = temp2[_COMBINED_INDEX] mask = temp3[_COMBINED_INDEX] # Inflate uncertainties for bad pixels specerr[ii, (mask & (badcombpixmask)) != 0] += \ 100. * np.mean(spec[ii, np.isfinite(spec[ii])]) # Inflate pixels with high SNR to 0.5 highsnr = spec[ii] / specerr[ii] > 200. specerr[ii, highsnr] = 0.005 * np.fabs(spec[ii, highsnr]) # Continuum-normalize cont = utils.get_apogee_continuum(wavelength=wavelength, spec=spec[ii], spec_err=specerr[ii], cont_pixels=cont_pixels) spec[ii] /= cont specerr[ii] /= cont specerr[ii, highsnr] = 0.005 except OSError: print('target could not be found!') continue # save spectra np.savez('spectra/apogee_all_spectra_' + str(batch_count) + '.npz', wavelength=wavelength, spectra=spec, spec_err=specerr, apogee_id=np.array(catalog["APOGEE_ID"]), apogee_fiber_id=fibers)
def get_visit_spectra_individual_object(apogee_id, allvisit_cat=None, save_local=False): ''' Download the visit spectra for an individual object. Get the v_helios from the allStar catalog, which are more accurate than the values reported in the visit spectra fits files. Use the barycentric correction to shift spectra to the heliocentric frame. Do a preliminary normalization using Bovy's visit spectrum normalization tool. It's critical that the spectra be renormalized prior to fitting using the spectral_model.get_apogee_continuum() function for self-consistency. apogee_id = byte-like object, i.e. b'2M06133561+2433362' ##### NOTE ##### I had to make some minor changes to Bovy's APOGEE package to get it to properly read in the visit spectra. In particular, after the 3rd line of the function apVisit() in that package's /apogee/tools/read.py file, I added the following lines: if ext == 0: from astropy.io import fits hdulist = fits.open(filePath) header = hdulist[0].header hdulist.close() return header I think this is the only change that was necessary, but it's possible I'm forgetting another change. ''' import apogee.tools.read as apread from apogee.tools import bitmask from apogee.spec import continuum import astropy if allvisit_cat is None: allvisit_cat = apread.allVisit(rmcommissioning=False, ak=False) where_visits = np.where(allvisit_cat['APOGEE_ID'] == apogee_id)[0] plate_ids = np.array([int(i) for i in allvisit_cat[where_visits]['PLATE']]) fiberids = allvisit_cat[where_visits]['FIBERID'] mjds = allvisit_cat[where_visits]['MJD'] JDs = allvisit_cat[where_visits]['JD'] vhelios_accurate = allvisit_cat[where_visits]['VHELIO'] vhelios_synth = allvisit_cat[where_visits]['SYNTHVHELIO'] snrs = allvisit_cat[where_visits]['SNR'] BCs = allvisit_cat[where_visits]['BC'] badcombpixmask = bitmask.badpixmask() + 2**bitmask.apogee_pixmask_int( "SIG_SKYLINE") all_spec, all_err, all_snr, all_hjd, all_vhelio = [], [], [], [], [] for i, pid in enumerate(plate_ids): try: spec = apread.apVisit(pid, mjds[i], fiberids[i], ext=1, header=False) specerr = apread.apVisit(pid, mjds[i], fiberids[i], ext=2, header=False) wave = apread.apVisit(pid, mjds[i], fiberids[i], ext=4, header=False) mask = apread.apVisit(pid, mjds[i], fiberids[i], ext=3, header=False) masterheader = apread.apVisit(pid, mjds[i], fiberids[i], ext=0, header=True) badpix = (mask & (badcombpixmask)) != 0 if np.sum(badpix) / len(badpix) > 0.5: print('too many bad pixels!') continue # if 50% or more of the pixels are bad, don't bother. specerr[badpix] = 100 * np.median(spec) # a small fraction of the visit spectra are on a different wavelength # grid than normal (maybe commissioning?). In any case, interpolate them # to the wavelength grid expected by Bovy's visit normalization routine. if len(wave) != 12288: print('fixing wavelength...') standard_grid = utils.load_visit_wavelength() spec = np.interp(standard_grid, wave, spec) specerr = np.interp(standard_grid, wave, specerr) wave = np.copy(standard_grid) # preliminary normalization using Bovy's visit normalization routine. cont = continuum.fitApvisit(spec, specerr, wave) specnorm, errnorm = spec / cont, specerr / cont # correct for Earth's orbital motion. spec_shift = utils.doppler_shift(wavelength=wave, flux=specnorm, dv=BCs[i]) spec_err_shift = utils.doppler_shift(wavelength=wave, flux=errnorm, dv=BCs[i]) # interpolate to the standard wavelength grid we use for combined spectra. interp_spec = np.interp(wavelength, wave, spec_shift) interp_err = np.interp(wavelength, wave, spec_err_shift) # truncate SNR at 200 interp_err[interp_err < 0.005] = 0.005 all_spec.append(interp_spec) all_err.append(interp_err) # Ideally, get the Julian date of the observations in the heliocentric frame. # Sometimes this isn't available; in that case, get the Julian date in Earth's # frame. These differ by at most 8 minutes, so not a big deal. try: all_hjd.append(masterheader['HJD']) except KeyError: all_hjd.append(JDs[i]) # There are a few cases where the v_helios from the allvisit catalog are clearly wrong. if np.abs(vhelios_accurate[i] > 1000) and np.abs( vhelios_synth[i] < 1000): vhel = vhelios_synth[i] else: vhel = vhelios_accurate[i] all_snr.append(snrs[i]) all_vhelio.append(vhel) except astropy.io.fits.verify.VerifyError: print('there was a verification error') continue all_spec, all_err, all_snr, all_hjd, all_vhelio = np.array(all_spec), \ np.array(all_err), np.array(all_snr), np.array(all_hjd), np.array(all_vhelio) msk = np.argsort(all_hjd) all_spec, all_err, all_snr, all_hjd, all_vhelio = all_spec[msk], all_err[msk], \ all_snr[msk], all_hjd[msk], all_vhelio[msk] if save_local: np.savez('spectra/visit/visit_spectra_ap_id_' + str(apogee_id.decode()) + '_.npz', spectra=all_spec, spec_errs=all_err, snrs=all_snr, hjds=all_hjd, vhelios=all_vhelio) return all_spec, all_err, all_snr, all_hjd, all_vhelio
def get_combined_spectrum_single_object(apogee_id, catalog=None, save_local=False): ''' apogee_id should be a byte-like object; i.e b'2M13012770+5754582' This downloads a single combined spectrum and the associated error array, and it normalizes both. ''' # read in the allStar catalog if you haven't already if catalog is None: catalog, fibers = read_apogee_catalog() # Set up bad pixel mask badcombpixmask = bitmask.badpixmask() + 2**bitmask.apogee_pixmask_int( "SIG_SKYLINE") _COMBINED_INDEX = 1 msk = np.where(catalog['APOGEE_ID'] == apogee_id)[0] if not len(msk): raise ValueError( 'the desired Apogee ID was not found in the allStar catalog.') field = catalog['FIELD'][msk[0]].decode() ap_id = apogee_id.decode() loc_id = catalog['LOCATION_ID'][msk[0]] if loc_id == 1: temp1 = apread.apStar(field, ap_id, ext=1, header=False, aspcapWavegrid=True) temp2 = apread.apStar(field, ap_id, ext=2, header=False, aspcapWavegrid=True) temp3 = apread.apStar(field, ap_id, ext=3, header=False, aspcapWavegrid=True) else: temp1 = apread.apStar(loc_id, ap_id, ext=1, header=False, aspcapWavegrid=True) temp2 = apread.apStar(loc_id, ap_id, ext=2, header=False, aspcapWavegrid=True) temp3 = apread.apStar(loc_id, ap_id, ext=3, header=False, aspcapWavegrid=True) if temp1.shape[0] > 6000: spec = temp1 specerr = temp2 mask = temp3 else: spec = temp1[_COMBINED_INDEX] specerr = temp2[_COMBINED_INDEX] mask = temp3[_COMBINED_INDEX] # Inflate uncertainties for bad pixels specerr[(mask & (badcombpixmask)) != 0] += 100 * np.mean( spec[np.isfinite(spec)]) # Inflate pixels with high SNR to 0.5 highsnr = spec / specerr > 200. specerr[highsnr] = 0.005 * np.fabs(spec[highsnr]) # Continuum-normalize cont = utils.get_apogee_continuum(wavelength=wavelength, spec=spec, spec_err=specerr, cont_pixels=cont_pixels) spec /= cont specerr /= cont specerr[highsnr] = 0.005 if save_local: np.savez('spectra/combined/spectrum_ap_id_' + str(apogee_id.decode()) + '_.npz', spectrum=spec, spec_err=specerr) return spec, specerr
def read_spectra(cluster,teffmin=4000.,teffmax=5000.,cont_type='cannon', cont_deg=4): """ NAME: read_spectra PURPOSE: Read the APOGEE spectra and their errors for stars in a given cluster INPUT: cluster - Name of the cluster (name in one of the data files) teffmin= (4000.) minimum temperature teffmax= (5000.) maximum temperature cont_type = ('cannon') type of continuum normalization to perform cont_deg= (4) degree polynomial to fit for continuum normalization OUTPUT: (data, spec, specerr) - (full data structure, spectra [nspec,nlam], spectral uncertainties [nspec,nlam]) nlam=7214 on ASPCAP grid HISTORY: 2015-08-13 - Written based on some older code - Bovy (UofT) """ if cluster.upper() in _GCS: data= read_meszarosgcdata() else: data= read_caldata() # Cut to just this cluster and temperature range if 'rc' in cluster.lower(): # Only for NGC 6819 rc= True cluster= cluster[:-2] else: rc= False data= data[data['CLUSTER'] == cluster.upper()] data= data[(data['TEFF'] < teffmax)\ *(data['TEFF'] > teffmin)] if cluster.lower() == 'n6819': g4CN= good4CN(cluster,data) g4CN[10]= False # another one, by hand! if rc: data= data[True-g4CN] # Just those! else: data= data[g4CN] # Just those! # Load all the spectra nspec= len(data) spec= numpy.zeros((nspec,7214)) specerr= numpy.zeros((nspec,7214)) # Setup bad pixel mask badcombpixmask= bitmask.badpixmask()\ +2**bitmask.apogee_pixmask_int("SIG_SKYLINE") for ii in range(nspec): sys.stdout.write('\r'+"Loading spectrum %i / %i ...\r" % (ii+1,nspec)) sys.stdout.flush() spec[ii]= apread.apStar(data['LOCATION_ID'][ii], data['ID'][ii], ext=1,header=False, aspcapWavegrid=True)[_COMBINED_INDEX] specerr[ii]= apread.apStar(data['LOCATION_ID'][ii], data['ID'][ii], ext=2,header=False, aspcapWavegrid=True)[_COMBINED_INDEX] # Inflate uncertainties for bad pixels mask= apread.apStar(data['LOCATION_ID'][ii], data['ID'][ii], ext=3,header=False, aspcapWavegrid=True)[_COMBINED_INDEX] specerr[ii,(mask & (badcombpixmask)) != 0]+=\ 100.*numpy.mean(spec[ii,True-numpy.isnan(spec[ii])]) # Also inflate pixels with high SNR to 0.5% highsnr= spec[ii]/specerr[ii] > 200. specerr[ii,highsnr]= 0.005*numpy.fabs(spec[ii,highsnr]) # Continuum-normalize cont= continuum.fit(spec[ii],specerr[ii],type=cont_type,deg=cont_deg) spec[ii]/= cont specerr[ii]/= cont specerr[ii,highsnr]= 0.005 # like standard APOGEE reduction sys.stdout.write('\r'+_ERASESTR+'\r') sys.stdout.flush() return (data,spec,specerr)