def test_isStdStar(self):
"""test isStdStar works for cmx, main, and sv1 fibermaps"""
from desispec.fluxcalibration import isStdStar
from desitarget.targetmask import desi_mask, mws_mask
from desitarget.sv1.sv1_targetmask import desi_mask as sv1_desi_mask
from desitarget.sv1.sv1_targetmask import mws_mask as sv1_mws_mask
from desitarget.cmx.cmx_targetmask import cmx_mask
from desitarget.targets import main_cmx_or_sv
from astropy.table import Table
#- CMX
fm = Table()
fm['CMX_TARGET'] = np.zeros(10, dtype=int)
fm['CMX_TARGET'][0:2] = cmx_mask.STD_FAINT
fm['CMX_TARGET'][2:4] = cmx_mask.SV0_STD_FAINT
self.assertEqual(main_cmx_or_sv(fm)[2], 'cmx')
self.assertEqual(np.count_nonzero(isStdStar(fm)), 4)
#- SV1
fm = Table()
fm['SV1_DESI_TARGET'] = np.zeros(10, dtype=int)
fm['SV1_MWS_TARGET'] = np.zeros(10, dtype=int)
fm['SV1_DESI_TARGET'][0:2] = sv1_desi_mask.STD_FAINT
fm['SV1_MWS_TARGET'][2:4] = sv1_mws_mask.GAIA_STD_FAINT
self.assertEqual(main_cmx_or_sv(fm)[2], 'sv1')
self.assertEqual(np.count_nonzero(isStdStar(fm)), 4)
#- Main
fm = Table()
fm['DESI_TARGET'] = np.zeros(10, dtype=int)
fm['MWS_TARGET'] = np.zeros(10, dtype=int)
fm['DESI_TARGET'][0:2] = desi_mask.STD_FAINT
fm['DESI_TARGET'][2:4] = desi_mask.STD_BRIGHT
fm['DESI_TARGET'][4:6] |= desi_mask.MWS_ANY
fm['MWS_TARGET'][4:6] = sv1_mws_mask.GAIA_STD_FAINT
self.assertEqual(main_cmx_or_sv(fm)[2], 'main')
self.assertEqual(np.count_nonzero(isStdStar(fm)), 6)
self.assertEqual(np.count_nonzero(isStdStar(fm, bright=False)), 4)
self.assertEqual(np.count_nonzero(isStdStar(fm, bright=True)), 2)
def select_stdstars(data, fibermap, snrcut=10, verbose=False):
"""Select spectra based on S/N and targeting bit.
"""
from astropy.table import Table
from desitarget.targets import main_cmx_or_sv
night, expid = data['NIGHT'][0], data['EXPID'][0]
out_fibermap = fibermap.copy()
# Pre-select standards
istd = np.where(isStdStar(out_fibermap))[0]
target_colnames, target_masks, survey = main_cmx_or_sv(out_fibermap)
target_col, target_mask = target_colnames[0], target_masks[
0] # CMX_TARGET, CMX_MASK for commissioning
#istd = np.where((out_fibermap[target_col] & target_mask.mask('STD_BRIGHT') != 0))[0]
# For each standard, apply a minimum S/N cut (in any camera) to
# subselect the fibers that were reasonably centered on the fiber.
reject = np.ones(len(istd), dtype=bool)
#if len(istd) == 0:
# pdb.set_trace()
if len(istd) > 0:
for ii, fiber in enumerate(out_fibermap['FIBER'][istd]):
wspec = np.where(
(data['EXPID'] == expid) * (data['FIBER'] == fiber))[0]
if len(wspec) > 0:
isnr = np.where(data['MEDIAN_CALIB_SNR'][wspec] > snrcut)[0]
if len(isnr) > 0:
reject[ii] = False
if np.sum(~reject) > 0:
istd_keep = istd[~reject]
else:
istd_keep = []
if np.sum(reject) > 0:
istd_reject = istd[reject]
else:
istd_reject = []
# Hack! Set the targeting bit for the failed standards to zero!
if len(istd_reject) > 0:
out_fibermap[target_col][istd_reject] = 0
if verbose:
print('EXPID={}, NSTD={}/{}'.format(expid, len(istd_keep), len(istd)))
return out_fibermap
def qa_throughput(night, verbose=False, overwrite=False):
"""Make some QAplots of the throughput.
"""
import astropy.units as u
from astropy import constants as const
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(context='talk', style='ticks') #, font_scale=1.0)
import desimodel.io
pngfile = os.path.join(outdir, 'qa',
'qa-throughput-{}.png'.format(str(night)))
if os.path.isfile(pngfile) and not overwrite:
print('File exists {}'.format(pngfile))
return
desi = desimodel.io.load_desiparams()
area = (desi['area']['geometric_area'] * u.m**2).to(u.cm**2)
thru = dict()
for camera in ('b', 'r', 'z'):
thru[camera] = desimodel.io.load_throughput(camera)
#import specter.throughput
#thrufile = '/global/u2/i/ioannis/repos/desihub/desimodel/data/throughput/thru-{}.fits'.format(camera)
#thru[camera] = specter.throughput.load_throughput(thrufile)
fig, ax = plt.subplots(figsize=(8, 6))
allexpiddir = sorted(
glob(os.path.join(outdir, 'exposures', str(night), '????????')))
colors = iter(plt.cm.rainbow(np.linspace(0, 1, len(allexpiddir))))
nstar = 0
for expiddir in allexpiddir:
expid = os.path.basename(expiddir)
for spectro in np.arange(10):
#calibfiles = desispec.io.findfile(
# 'fluxcalib', night=night, expid=int(expid), spectrograph=spectro,
# camera=camera, specprod_dir=outdir)
calibfiles = glob(
os.path.join(expiddir,
'fluxcalib-*{}-{}.fits'.format(spectro, expid)))
if len(calibfiles) == 3:
#print(calibfile, os.path.isfile(calibfile))
col = next(colors)
for camera in ('b', 'r', 'z'):
cframefile = os.path.join(
expiddir,
'cframe-{}{}-{}.fits'.format(camera, spectro, expid))
calibfile = os.path.join(
expiddir, 'fluxcalib-{}{}-{}.fits'.format(
camera, spectro, expid))
info = fitsio.FITS(calibfile)
hdr = info['FLUXCALIB'].read_header()
exptime = hdr['EXPTIME'] * u.s
wave = info['WAVELENGTH'].read() * u.angstrom
flux = info['FLUXCALIB'].read()
ivar = info['IVAR'].read()
specthru = flux * 1e17 * (u.electron / u.angstrom) / (
u.erg / u.s / u.cm / u.cm / u.angstrom)
specthru *= const.h.to(u.erg * u.s) * const.c.to(
u.angstrom /
u.s) / exptime / area / wave # [electron/photon]
# Plot just the standards--
fibermap = desispec.io.read_fibermap(cframefile)
istd = np.where(isStdStar(fibermap))[0]
for ii in istd:
if camera == 'b':
label = '{}-{}-{}'.format(int(expid), str(spectro),
fibermap['FIBER'][ii])
else:
label = None
if np.max(specthru[ii, :].value) > 1:
print(
'Weird star in EXPID={}, camera={}, fiber={}'.
format(expid, camera, fibermap['FIBER'][ii]))
else:
if camera == 'b':
nstar += 1
ax.plot(wave.value,
specthru[ii, :].value,
label=label,
alpha=0.8,
color=col)
for camera in ('b', 'r', 'z'):
ax.plot(thru[camera]._wave,
thru[camera].thru(thru[camera]._wave),
color='k') #, alpha=0.8)
ax.set_xlabel(r'Wavelength ($\AA$)')
ax.set_ylabel('Throughput (electron / photon)')
ax.set_ylim(0, 1)
ax.set_title('Night {}, N={}'.format(night, nstar))
ax.legend(loc='upper right', ncol=2, fontsize=12)
fig.subplots_adjust(left=0.14, bottom=0.16)
print('Writing {}'.format(pngfile))
fig.savefig(pngfile)
def update_frame_fibermaps(night, verbose=False, overwrite=False):
"""Update the fibermaps in each frame file with the necessary information on
targeted standard stars.
"""
from astropy.table import Table
from desitarget.targets import main_cmx_or_sv
# Build the stacked QA and fiberassign "map" files.
data, fiberassignmap = gather_nightwatch_qa(night,
verbose=verbose,
overwrite=overwrite)
if data is None:
return
# For each EXPID, find the fibers with standard stars *and* good spectra.
fibermapdict = dict()
for expid in set(data['EXPID']):
strexpid = '{:08d}'.format(expid)
framefiles = sorted(
glob(
desispec.io.findfile('frame',
night=night,
expid=expid,
specprod_dir=specprod_dir,
camera='*')))
# Figure out which framefiles need to be (re)processed.
if overwrite:
need_framefiles = framefiles
else:
need_framefiles = []
for framefile in framefiles:
outframefile = framefile.replace(specprod_dir, outdir)
if not os.path.isfile(outframefile):
need_framefiles.append(framefile)
need_framefiles = np.array(need_framefiles)
if len(need_framefiles) == 0:
print('All done with NIGHT={}, EXPID={}'.format(night, strexpid))
continue
# Read and cache the fibermap.
thismap = np.where((fiberassignmap['EXPID'] == strexpid) *
(fiberassignmap['NIGHT'] == str(night)))[0]
tileid, fibermapfile = fiberassignmap['TILEID'][thismap][
0], fiberassignmap['FIBERASSIGNFILE'][thismap][0]
strtileid = str(tileid)
if strtileid in fibermapdict.keys():
print('Using cached fibermap for TILEID={}'.format(strtileid))
fullfibermap = fibermapdict[strtileid]
else:
fibermapfile = os.path.join(rawdata_dir, str(night), fibermapfile)
if verbose:
print('Reading and caching {}'.format(fibermapfile))
fullfibermap = Table(
fitsio.read(fibermapfile)) #, columns=['CMX_TARGET']))
fibermapdict[strtileid] = fullfibermap
# Select standard stars
thisspec = np.where(
np.logical_and(data['EXPID'] == expid, data['NIGHT'] == night))[0]
if len(thisspec) == 0:
print('This should never happen!')
pdb.set_trace()
use_fullfibermap = select_stdstars(data[thisspec],
fullfibermap,
verbose=verbose)
#check = np.where(isStdStar(use_fullfibermap))[0]
#if len(check) == 0:
# pdb.set_trace()
# Finally process every frame containing at least one standard star.
for framefile in framefiles:
outframefile = framefile.replace(specprod_dir, outdir)
# Shouldn't need this bit of code because we checked for existing
# files above.
if os.path.isfile(outframefile) and not overwrite:
print('File exists {}'.format(outframefile))
continue
fibers = fitsio.read(framefile, 'FIBERMAP', columns='FIBER')
fibermap = use_fullfibermap[np.isin(use_fullfibermap['FIBER'],
fibers)]
#fibermap = fullfibermap[np.isin(fullfibermap['FIBER'], fr.fibermap['FIBER'])]
# Require at least one standard star on this petal.
#target_colnames, target_masks, survey = main_cmx_or_sv(fibermap)
#target_col, target_mask = target_colnames[0], target_masks[0] # CMX_TARGET, CMX_MASK for commissioning
#istd = np.where((fibermap[target_col] & target_mask.mask('STD_BRIGHT') != 0))[0]
istd = np.where(isStdStar(fibermap))[0]
if len(istd) > 0:
# Read and copy all the columns!
#if verbose:
# print('Reading {}'.format(framefile))
fr = desispec.io.read_frame(framefile)
#import matplotlib.pyplot as plt
#plt.plot(fr.wave, fr.flux[istd[0], :]) ; plt.show()
#pdb.set_trace()
# Fragile. desi_proc should be smarter about how it initializes
# the dummy fibermap and/or we should be smarter about what
# columns we assume that desi_proc has added...
isky = np.where(fr.fibermap['OBJTYPE'] == 'SKY')[0]
if len(isky) == 0:
print('No sky fibers found -- this should not happen!')
continue
fibermap['OBJTYPE'][isky] = fr.fibermap['OBJTYPE'][
isky] # fragile!
fibermap['DESI_TARGET'][isky] |= fr.fibermap['DESI_TARGET'][
isky] # fragile!
fr.fibermap = fibermap
if verbose:
print('Writing {}'.format(outframefile))
desispec.io.write_frame(outframefile, fr)
# Also create a soft link to the skymodel file so we can run pipeline QA.
skymodelfile = desispec.io.findfile(
'sky',
night=night,
expid=expid,
camera=fr.meta['CAMERA'],
specprod_dir=specprod_dir) # note: specprod_dir!
cmd = 'ln -sf {} {}'.format(
skymodelfile, skymodelfile.replace(specprod_dir, outdir))
print(cmd)
os.system(cmd)
if False:
print('Returning after first EXPID - fix me.')
return
def main(args) :
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
log = get_logger()
log.info("mag delta %s = %f (for the pre-selection of stellar models)"%(args.color,args.delta_color))
frames={}
flats={}
skies={}
spectrograph=None
starfibers=None
starindices=None
fibermap=None
# READ DATA
############################################
for filename in args.frames :
log.info("reading %s"%filename)
frame=io.read_frame(filename)
header=fits.getheader(filename, 0)
frame_fibermap = frame.fibermap
frame_starindices = np.where(isStdStar(frame_fibermap['DESI_TARGET']))[0]
#- Confirm that all fluxes have entries but trust targeting bits
#- to get basic magnitude range correct
keep = np.ones(len(frame_starindices), dtype=bool)
for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']: #- and W1 and W2?
keep &= frame_fibermap[colname][frame_starindices] > 10**((22.5-30)/2.5)
keep &= frame_fibermap[colname][frame_starindices] < 10**((22.5-0)/2.5)
frame_starindices = frame_starindices[keep]
camera=safe_read_key(header,"CAMERA").strip().lower()
if spectrograph is None :
spectrograph = frame.spectrograph
fibermap = frame_fibermap
starindices=frame_starindices
starfibers=fibermap["FIBER"][starindices]
elif spectrograph != frame.spectrograph :
log.error("incompatible spectrographs %d != %d"%(spectrograph,frame.spectrograph))
raise ValueError("incompatible spectrographs %d != %d"%(spectrograph,frame.spectrograph))
elif starindices.size != frame_starindices.size or np.sum(starindices!=frame_starindices)>0 :
log.error("incompatible fibermap")
raise ValueError("incompatible fibermap")
if not camera in frames :
frames[camera]=[]
frames[camera].append(frame)
for filename in args.skymodels :
log.info("reading %s"%filename)
sky=io.read_sky(filename)
header=fits.getheader(filename, 0)
camera=safe_read_key(header,"CAMERA").strip().lower()
if not camera in skies :
skies[camera]=[]
skies[camera].append(sky)
for filename in args.fiberflats :
log.info("reading %s"%filename)
header=fits.getheader(filename, 0)
flat=io.read_fiberflat(filename)
camera=safe_read_key(header,"CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if camera in flats:
log.warning("cannot handle several flats of same camera (%s), will use only the first one"%camera)
#raise ValueError("cannot handle several flats of same camera (%s)"%camera)
else :
flats[camera]=flat
if starindices.size == 0 :
log.error("no STD star found in fibermap")
raise ValueError("no STD star found in fibermap")
log.info("found %d STD stars"%starindices.size)
log.warning("Not using flux errors for Standard Star fits!")
# DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
############################################
for cam in frames :
if not cam in skies:
log.warning("Missing sky for %s"%cam)
frames.pop(cam)
continue
if not cam in flats:
log.warning("Missing flat for %s"%cam)
frames.pop(cam)
continue
flat=flats[cam]
for frame,sky in zip(frames[cam],skies[cam]) :
frame.flux = frame.flux[starindices]
frame.ivar = frame.ivar[starindices]
frame.ivar *= (frame.mask[starindices] == 0)
frame.ivar *= (sky.ivar[starindices] != 0)
frame.ivar *= (sky.mask[starindices] == 0)
frame.ivar *= (flat.ivar[starindices] != 0)
frame.ivar *= (flat.mask[starindices] == 0)
frame.flux *= ( frame.ivar > 0) # just for clean plots
for star in range(frame.flux.shape[0]) :
ok=np.where((frame.ivar[star]>0)&(flat.fiberflat[star]!=0))[0]
if ok.size > 0 :
frame.flux[star] = frame.flux[star]/flat.fiberflat[star] - sky.flux[star]
frame.resolution_data = frame.resolution_data[starindices]
nstars = starindices.size
fibermap = Table(fibermap[starindices])
# READ MODELS
############################################
log.info("reading star models in %s"%args.starmodels)
stdwave,stdflux,templateid,teff,logg,feh=io.read_stdstar_templates(args.starmodels)
# COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
############################################
#- Support older fibermaps
if 'PHOTSYS' not in fibermap.colnames:
log.warning('Old fibermap format; using defaults for missing columns')
log.warning(" PHOTSYS = 'S'")
log.warning(" MW_TRANSMISSION_G/R/Z = 1.0")
log.warning(" EBV = 0.0")
fibermap['PHOTSYS'] = 'S'
fibermap['MW_TRANSMISSION_G'] = 1.0
fibermap['MW_TRANSMISSION_R'] = 1.0
fibermap['MW_TRANSMISSION_Z'] = 1.0
fibermap['EBV'] = 0.0
model_filters = dict()
if 'S' in fibermap['PHOTSYS']:
for filter_name in ['DECAM_G', 'DECAM_R', 'DECAM_Z']:
model_filters[filter_name] = load_filter(filter_name)
if 'N' in fibermap['PHOTSYS']:
for filter_name in ['BASS_G', 'BASS_R', 'MZLS_Z']:
model_filters[filter_name] = load_filter(filter_name)
if len(model_filters) == 0:
raise ValueError("No filters loaded; neither 'N' nor 'S' in PHOTSYS?")
log.info("computing model mags for %s"%sorted(model_filters.keys()))
model_mags = dict()
fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
for filter_name, filter_response in model_filters.items():
model_mags[filter_name] = filter_response.get_ab_magnitude(stdflux*fluxunits,stdwave)
log.info("done computing model mags")
# LOOP ON STARS TO FIND BEST MODEL
############################################
linear_coefficients=np.zeros((nstars,stdflux.shape[0]))
chi2dof=np.zeros((nstars))
redshift=np.zeros((nstars))
normflux=[]
star_mags = dict()
star_unextincted_mags = dict()
for band in ['G', 'R', 'Z']:
star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_'+band])
star_unextincted_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_'+band] / fibermap['MW_TRANSMISSION_'+band])
star_colors = dict()
star_colors['G-R'] = star_mags['G'] - star_mags['R']
star_colors['R-Z'] = star_mags['R'] - star_mags['Z']
star_unextincted_colors = dict()
star_unextincted_colors['G-R'] = star_unextincted_mags['G'] - star_unextincted_mags['R']
star_unextincted_colors['R-Z'] = star_unextincted_mags['R'] - star_unextincted_mags['Z']
fitted_model_colors = np.zeros(nstars)
for star in range(nstars) :
log.info("finding best model for observed star #%d"%star)
# np.array of wave,flux,ivar,resol
wave = {}
flux = {}
ivar = {}
resolution_data = {}
for camera in frames :
for i,frame in enumerate(frames[camera]) :
identifier="%s-%d"%(camera,i)
wave[identifier]=frame.wave
flux[identifier]=frame.flux[star]
ivar[identifier]=frame.ivar[star]
resolution_data[identifier]=frame.resolution_data[star]
# preselect models based on magnitudes
if fibermap['PHOTSYS'][star] == 'N':
if args.color == 'G-R':
model_colors = model_mags['BASS_G'] - model_mags['BASS_R']
elif args.color == 'R-Z':
model_colors = model_mags['BASS_R'] - model_mags['MZLS_Z']
else:
raise ValueError('Unknown color {}'.format(args.color))
else:
if args.color == 'G-R':
model_colors = model_mags['DECAM_G'] - model_mags['DECAM_R']
elif args.color == 'R-Z':
model_colors = model_mags['DECAM_R'] - model_mags['DECAM_Z']
else:
raise ValueError('Unknown color {}'.format(args.color))
color_diff = model_colors - star_unextincted_colors[args.color][star]
selection = np.abs(color_diff) < args.delta_color
# smallest cube in parameter space including this selection (needed for interpolation)
new_selection = (teff>=np.min(teff[selection]))&(teff<=np.max(teff[selection]))
new_selection &= (logg>=np.min(logg[selection]))&(logg<=np.max(logg[selection]))
new_selection &= (feh>=np.min(feh[selection]))&(feh<=np.max(feh[selection]))
selection = np.where(new_selection)[0]
log.info("star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d"%(
star, starfibers[star], args.color, star_unextincted_colors[args.color][star],
selection.size, stdflux.shape[0]))
# Match unextincted standard stars to data
coefficients, redshift[star], chi2dof[star] = match_templates(
wave, flux, ivar, resolution_data,
stdwave, stdflux[selection],
teff[selection], logg[selection], feh[selection],
ncpu=args.ncpu, z_max=args.z_max, z_res=args.z_res,
template_error=args.template_error
)
linear_coefficients[star,selection] = coefficients
log.info('Star Fiber: {0}; TEFF: {1}; LOGG: {2}; FEH: {3}; Redshift: {4}; Chisq/dof: {5}'.format(
starfibers[star],
np.inner(teff,linear_coefficients[star]),
np.inner(logg,linear_coefficients[star]),
np.inner(feh,linear_coefficients[star]),
redshift[star],
chi2dof[star])
)
# Apply redshift to original spectrum at full resolution
model=np.zeros(stdwave.size)
redshifted_stdwave = stdwave*(1+redshift[star])
for i,c in enumerate(linear_coefficients[star]) :
if c != 0 :
model += c*np.interp(stdwave,redshifted_stdwave,stdflux[i])
# Apply dust extinction to the model
model *= dust_transmission(stdwave, fibermap['EBV'][star])
# Compute final color of dust-extincted model
if fibermap['PHOTSYS'][star] == 'N':
if args.color == 'G-R':
model_mag1 = model_filters['BASS_G'].get_ab_magnitude(model*fluxunits, stdwave)
model_mag2 = model_filters['BASS_R'].get_ab_magnitude(model*fluxunits, stdwave)
model_magr = model_mag2
elif args.color == 'R-Z':
model_mag1 = model_filters['BASS_R'].get_ab_magnitude(model*fluxunits, stdwave)
model_mag2 = model_filters['MZLS_Z'].get_ab_magnitude(model*fluxunits, stdwave)
model_magr = model_mag1
else:
raise ValueError('Unknown color {}'.format(args.color))
else:
if args.color == 'G-R':
model_mag1 = model_filters['DECAM_G'].get_ab_magnitude(model*fluxunits, stdwave)
model_mag2 = model_filters['DECAM_R'].get_ab_magnitude(model*fluxunits, stdwave)
model_magr = model_mag2
elif args.color == 'R-Z':
model_mag1 = model_filters['DECAM_R'].get_ab_magnitude(model*fluxunits, stdwave)
model_mag2 = model_filters['DECAM_Z'].get_ab_magnitude(model*fluxunits, stdwave)
model_magr = model_mag1
else:
raise ValueError('Unknown color {}'.format(args.color))
fitted_model_colors[star] = model_mag1 - model_mag2
#- TODO: move this back into normalize_templates, at the cost of
#- recalculating a model magnitude?
# Normalize the best model using reported magnitude
scalefac=10**((model_magr - star_mags['R'][star])/2.5)
log.info('scaling R mag {} to {} using scale {}'.format(model_magr, star_mags['R'][star], scalefac))
normflux.append(model*scalefac)
# Now write the normalized flux for all best models to a file
normflux=np.array(normflux)
data={}
data['LOGG']=linear_coefficients.dot(logg)
data['TEFF']= linear_coefficients.dot(teff)
data['FEH']= linear_coefficients.dot(feh)
data['CHI2DOF']=chi2dof
data['REDSHIFT']=redshift
data['COEFF']=linear_coefficients
data['DATA_%s'%args.color]=star_colors[args.color]
data['MODEL_%s'%args.color]=fitted_model_colors
io.write_stdstar_models(args.outfile,normflux,stdwave,starfibers,data)
def SNRFit(frame,
night,
camera,
expid,
params,
fidboundary=None,
offline=False):
"""
Signal vs. Noise With fitting
Take flux and inverse variance arrays and calculate S/N for individual
targets (ELG, LRG, QSO, STD) and for each amplifier of the camera.
then fit the log(snr)=a+b*mag or log(snr)=poly(mag)
see http://arXiv.org/abs/0706.1062v2 for proper fitting of power-law distributions
it is not implemented here!
qadict has the following data model
"MAGNITUDES" : ndarray - Depends on camera (DECAM_G, DECAM_R, DECAM_Z)
"MEDIAN_SNR" : ndarray (nfiber)
"NUM_NEGATIVE_SNR" : int
"SNR_MAG_TGT"
"FITCOEFF_TGT" : list
"SNR_RESID" : list, can be trimmed down during the fitting
"FIDSNR_TGT"
"RA" : ndarray (nfiber)
"DEC" : ndarray (nfiber)
"OBJLIST" : list - Save a copy to make sense of the list order later
"EXPTIME" : float
"FIT_FILTER" : str
"r2" : float - Fitting parameter
Args:
frame: desispec.Frame object
night :
camera :
expid : int
params: parameters dictionary
{
"Func": "linear", # Fit function type one of ["linear","poly","astro"]
"FIDMAG": 22.0, # magnitude to evaluate the fit
"Filter":"DECAM_R", #filter name
}
fidboundary : list of slices indicating where to select in fiber
and wavelength directions for each amp (output of slice_fidboundary function)
offline: bool, optional
If True, save things differently for offline
Returns:
qadict : dict
"""
#- Get imaging magnitudes and calculate SNR
fmag = 22.0
if "FIDMAG" in params:
fmag = params["FIDMAG"]
mediansnr = SN_ratio(frame.flux, frame.ivar)
qadict = {"MEDIAN_SNR": mediansnr}
exptime = frame.meta["EXPTIME"]
ivar = frame.ivar
if "Filter" in params:
thisfilter = params["Filter"]
elif camera[0] == 'b':
thisfilter = 'DECAM_G'
elif camera[0] == 'r':
thisfilter = 'DECAM_R'
else:
thisfilter = 'DECAM_Z'
qadict["FIT_FILTER"] = thisfilter
qadict["EXPTIME"] = exptime
if thisfilter in ('DECAM_G', 'BASS_G'):
photflux = frame.fibermap['FLUX_G']
elif thisfilter in ('DECAM_R', 'BASS_R'):
photflux = frame.fibermap['FLUX_R']
elif thisfilter in ('DECAM_Z', 'MZLS_Z'):
photflux = frame.fibermap['FLUX_Z']
else:
raise ValueError('Unknown filter {}'.format(thisfilter))
mag_grz = np.zeros((3, frame.nspec)) + 99.0
for i, colname in enumerate(['FLUX_G', 'FLUX_R', 'FLUX_Z']):
ok = frame.fibermap[colname] > 0
mag_grz[i, ok] = 22.5 - 2.5 * np.log10(frame.fibermap[colname][ok])
qadict["FILTERS"] = ['G', 'R', 'Z']
#qadict["OBJLIST"]=list(objlist)
#- Set up fit of SNR vs. Magnitude
# RS: commenting this until we have flux calibration
# try:
# #- Get read noise from Get_RMS TODO: use header information for this
# rfile=findfile('ql_getrms_file',int(night),int(expid),camera,specprod_dir=os.environ['QL_SPEC_REDUX'])
# with open(rfile) as rf:
# rmsfile=yaml.load(rf)
# rmsval=rmsfile["METRICS"]["NOISE"]
# #- The factor of 1e-3 is a very basic (and temporary!!) flux calibration
# #- used to convert read noise to proper flux units
# r2=1e-3*rmsval**2
# except:
# log.info("Was not able to obtain read noise from prior knowledge, fitting B+R**2...")
# Use astronomically motivated function for SNR fit
funcMap = s2n_funcs(exptime=exptime)
fit = funcMap['astro']
# Use median inverse variance of each fiber for chi2 minimization
var = []
for i in range(len(ivar)):
var.append(1 / np.median(ivar[i]))
neg_snr_tot = []
#- neg_snr_tot counts the number of times a fiber has a negative median SNR. This should
#- not happen for non-sky fibers with actual flux in them. However, it does happen rarely
#- in sims. To avoid this, we omit such fibers in the fit, but keep count for diagnostic
#- purposes.
#- Loop over each target type, and associate SNR and image magnitudes for each type.
resid_snr = []
fidsnr_tgt = []
fitcoeff = []
fitcovar = []
snrmag = []
fitsnr = []
fitT = []
elgfibers = np.where(
(frame.fibermap['DESI_TARGET'] & desi_mask.ELG) != 0)[0]
lrgfibers = np.where(
(frame.fibermap['DESI_TARGET'] & desi_mask.LRG) != 0)[0]
qsofibers = np.where(
(frame.fibermap['DESI_TARGET'] & desi_mask.QSO) != 0)[0]
bgsfibers = np.where(
(frame.fibermap['DESI_TARGET'] & desi_mask.BGS_ANY) != 0)[0]
mwsfibers = np.where(
(frame.fibermap['DESI_TARGET'] & desi_mask.MWS_ANY) != 0)[0]
stdfibers = np.where(isStdStar(frame.fibermap))[0]
for T, fibers in (
['ELG', elgfibers],
['LRG', lrgfibers],
['QSO', qsofibers],
['BGS', bgsfibers],
['MWS', mwsfibers],
['STAR', stdfibers],
):
if len(fibers) == 0:
pass
else:
objvar = np.array(var)[fibers]
medsnr = mediansnr[fibers]
all_medsnr = medsnr.copy() # In case any are cut below
mags = np.zeros(medsnr.shape)
ok = (photflux[fibers] > 0)
mags[ok] = 22.5 - 2.5 * np.log10(photflux[fibers][ok])
try:
#- Determine negative SNR and mag values and remove
neg_snr = len(np.where(medsnr <= 0.0)[0])
neg_snr_tot.append(neg_snr)
xs = mags.argsort()
#- Convert magnitudes to flux
x = 10**(-0.4 * (mags[xs] - 22.5))
med_snr = medsnr[xs]
y = med_snr
#- Fit SNR vs. Magnitude using chi squared minimization,
#- evaluate at fiducial magnitude, and store results in METRICS
#- Set high minimum initally chi2 value to be overwritten when fitting
minchi2 = 1e10
for a in range(100):
for b in range(100):
guess = [0.01 * a, 0.1 * b]
fitdata = fit(x, guess[0], guess[1])
totchi2 = []
for k in range(len(x)):
singlechi2 = ((y[k] - fitdata[k]) / objvar[k])**2
totchi2.append(singlechi2)
chi2 = np.sum(totchi2)
if chi2 <= minchi2:
minchi2 = chi2
fita = guess[0]
fitb = guess[1]
#- Increase granualarity of 'a' by a factor of 10
for c in range(100):
for d in range(100):
guess = [fita - 0.05 + 0.001 * c, 0.1 * d]
fitdata = fit(x, guess[0], guess[1])
totchi2 = []
for k in range(len(x)):
singlechi2 = ((y[k] - fitdata[k]) / objvar[k])**2
totchi2.append(singlechi2)
chi2 = np.sum(totchi2)
if chi2 <= minchi2:
minchi2 = chi2
fitc = guess[0]
fitd = guess[1]
#- Increase granualarity of 'a' by another factor of 10
for e in range(100):
for f in range(100):
guess = [fitc - 0.005 + 0.0001 * e, 0.1 * f]
fitdata = fit(x, guess[0], guess[1])
totchi2 = []
for k in range(len(x)):
singlechi2 = ((y[k] - fitdata[k]) / objvar[k])**2
totchi2.append(singlechi2)
chi2 = np.sum(totchi2)
if chi2 <= minchi2:
minchi2 = chi2
fite = guess[0]
fitf = guess[1]
# Save
fitcoeff.append([fite, fitf])
fidsnr_tgt.append(fit(10**(-0.4 * (fmag - 22.5)), fita, fitb))
fitT.append(T)
except RuntimeError:
log.warning("In fit of {}, Fit minimization failed!".format(T))
fitcoeff.append(np.nan)
fidsnr_tgt.append(np.nan)
qadict["{:s}_FIBERID".format(T)] = fibers.tolist()
if offline:
snr_mag = [medsnr, mags]
snrmag.append(snr_mag)
else:
snr_mag = [all_medsnr, mags]
snrmag.append(snr_mag)
#- Calculate residual SNR for focal plane plots
if not offline:
fit_snr = fit(x, fite, fitf)
fitsnr.append(fit_snr)
resid = (med_snr - fit_snr) / fit_snr
resid_snr += resid.tolist()
else:
x = 10**(-0.4 * (mags - 22.5))
fit_snr = fit(x, fite, fitf)
fitsnr.append(fit_snr)
resid = (all_medsnr - fit_snr) / fit_snr
resid_snr += resid.tolist()
qadict["NUM_NEGATIVE_SNR"] = sum(neg_snr_tot)
qadict["SNR_MAG_TGT"] = snrmag
qadict["FITCOEFF_TGT"] = fitcoeff
qadict["SNR_RESID"] = resid_snr
qadict["FIDSNR_TGT"] = fidsnr_tgt
qadict["OBJLIST"] = fitT
qadict["RA"] = frame.fibermap['TARGET_RA']
qadict["DEC"] = frame.fibermap['TARGET_DEC']
return qadict, fitsnr
def SignalVsNoise(frame, params, fidboundary=None):
"""
Signal vs. Noise
Take flux and inverse variance arrays and calculate S/N for individual
targets (ELG, LRG, QSO, STD) and for each amplifier of the camera.
Args:
flux (array): 2d [nspec,nwave] the signal (typically for spectra,
this comes from frame object
ivar (array): 2d [nspec,nwave] corresponding inverse variance
fidboundary : list of slices indicating where to select in fiber
and wavelength directions for each amp (output of slice_fidboundary function)
"""
mags = _get_mags(frame)
medsnr = SN_ratio(frame.flux, frame.ivar)
#- Calculate median SNR per bin and associate with imaging Mag. for ELG fibers
elgfibers = np.where(
(frame.fibermap['DESI_TARGET'] & desi_mask.ELG) != 0)[0]
elg_medsnr = medsnr[elgfibers]
elg_mag = mags[elgfibers]
elg_snr_mag = np.array((elg_medsnr, elg_mag)) #- not storing fiber number
#- Calculate median SNR, associate with imaging Mag for LRGs
lrgfibers = np.where(
(frame.fibermap['DESI_TARGET'] & desi_mask.LRG) != 0)[0]
lrg_medsnr = medsnr[lrgfibers]
lrg_mag = mags[lrgfibers]
lrg_snr_mag = np.array((lrg_medsnr, lrg_mag))
#- Calculate median SNR, associate with imaging Mag. for QSOs
qsofibers = np.where(
(frame.fibermap['DESI_TARGET'] & desi_mask.QSO) != 0)[0]
qso_medsnr = medsnr[qsofibers]
qso_mag = mags[qsofibers]
qso_snr_mag = np.array((qso_medsnr, qso_mag))
#- Calculate median SNR, associate with Mag. for STD stars
stdfibers = np.where(isStdStar(frame.fibermap))[0]
std_medsnr = medsnr[stdfibers]
std_mag = mags[stdfibers]
std_snr_mag = np.array((std_medsnr, std_mag))
#- Median S/N for different amp zones.
average_amp = None
if fidboundary is not None:
averages = []
for ii in range(4):
if fidboundary[ii][
0].start is not None: #- have fibers in this amp?
medsnramp = SN_ratio(frame.flux[fidboundary[ii]],
frame.ivar[fidboundary[ii]])
averages.append(np.mean(medsnramp))
else:
averages.append(None)
average_amp = np.array(averages)
elg_fidmag_snr = []
star_fidmag_snr = []
ra = frame.fibermap['TARGET_RA']
dec = frame.fibermap['TARGET_DEC']
#- fill QA dict with metrics:
qadict = {
"RA": ra,
"DEC": dec,
"MEDIAN_SNR": medsnr,
"MEDIAN_AMP_SNR": average_amp,
"ELG_FIBERID": elgfibers.tolist(),
"ELG_SNR_MAG": elg_snr_mag,
"LRG_FIBERID": lrgfibers.tolist(),
"LRG_SNR_MAG": lrg_snr_mag,
"QSO_FIBERID": qsofibers.tolist(),
"QSO_SNR_MAG": qso_snr_mag,
"STAR_FIBERID": stdfibers.tolist(),
"STAR_SNR_MAG": std_snr_mag,
"ELG_FIDMAG_SNR": elg_fidmag_snr,
"STAR_FIDMAG_SNR": star_fidmag_snr
}
return qadict
def main(args):
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
log = get_logger()
log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
(args.color, args.delta_color))
frames = {}
flats = {}
skies = {}
spectrograph = None
starfibers = None
starindices = None
fibermap = None
# READ DATA
############################################
for filename in args.frames:
log.info("reading %s" % filename)
frame = io.read_frame(filename)
header = fits.getheader(filename, 0)
frame_fibermap = frame.fibermap
frame_starindices = np.where(isStdStar(frame_fibermap))[0]
#- Confirm that all fluxes have entries but trust targeting bits
#- to get basic magnitude range correct
keep = np.ones(len(frame_starindices), dtype=bool)
for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']: #- and W1 and W2?
keep &= frame_fibermap[colname][frame_starindices] > 10**(
(22.5 - 30) / 2.5)
keep &= frame_fibermap[colname][frame_starindices] < 10**(
(22.5 - 0) / 2.5)
frame_starindices = frame_starindices[keep]
camera = safe_read_key(header, "CAMERA").strip().lower()
if spectrograph is None:
spectrograph = frame.spectrograph
fibermap = frame_fibermap
starindices = frame_starindices
starfibers = fibermap["FIBER"][starindices]
elif spectrograph != frame.spectrograph:
log.error("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
raise ValueError("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
elif starindices.size != frame_starindices.size or np.sum(
starindices != frame_starindices) > 0:
log.error("incompatible fibermap")
raise ValueError("incompatible fibermap")
if not camera in frames:
frames[camera] = []
frames[camera].append(frame)
for filename in args.skymodels:
log.info("reading %s" % filename)
sky = io.read_sky(filename)
header = fits.getheader(filename, 0)
camera = safe_read_key(header, "CAMERA").strip().lower()
if not camera in skies:
skies[camera] = []
skies[camera].append(sky)
for filename in args.fiberflats:
log.info("reading %s" % filename)
header = fits.getheader(filename, 0)
flat = io.read_fiberflat(filename)
camera = safe_read_key(header, "CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if camera in flats:
log.warning(
"cannot handle several flats of same camera (%s), will use only the first one"
% camera)
#raise ValueError("cannot handle several flats of same camera (%s)"%camera)
else:
flats[camera] = flat
if starindices.size == 0:
log.error("no STD star found in fibermap")
raise ValueError("no STD star found in fibermap")
log.info("found %d STD stars" % starindices.size)
log.warning("Not using flux errors for Standard Star fits!")
# DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
############################################
for cam in frames:
if not cam in skies:
log.warning("Missing sky for %s" % cam)
frames.pop(cam)
continue
if not cam in flats:
log.warning("Missing flat for %s" % cam)
frames.pop(cam)
continue
flat = flats[cam]
for frame, sky in zip(frames[cam], skies[cam]):
frame.flux = frame.flux[starindices]
frame.ivar = frame.ivar[starindices]
frame.ivar *= (frame.mask[starindices] == 0)
frame.ivar *= (sky.ivar[starindices] != 0)
frame.ivar *= (sky.mask[starindices] == 0)
frame.ivar *= (flat.ivar[starindices] != 0)
frame.ivar *= (flat.mask[starindices] == 0)
frame.flux *= (frame.ivar > 0) # just for clean plots
for star in range(frame.flux.shape[0]):
ok = np.where((frame.ivar[star] > 0)
& (flat.fiberflat[star] != 0))[0]
if ok.size > 0:
frame.flux[star] = frame.flux[star] / flat.fiberflat[
star] - sky.flux[star]
frame.resolution_data = frame.resolution_data[starindices]
nstars = starindices.size
fibermap = Table(fibermap[starindices])
# READ MODELS
############################################
log.info("reading star models in %s" % args.starmodels)
stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
args.starmodels)
# COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
############################################
#- Support older fibermaps
if 'PHOTSYS' not in fibermap.colnames:
log.warning('Old fibermap format; using defaults for missing columns')
log.warning(" PHOTSYS = 'S'")
log.warning(" MW_TRANSMISSION_G/R/Z = 1.0")
log.warning(" EBV = 0.0")
fibermap['PHOTSYS'] = 'S'
fibermap['MW_TRANSMISSION_G'] = 1.0
fibermap['MW_TRANSMISSION_R'] = 1.0
fibermap['MW_TRANSMISSION_Z'] = 1.0
fibermap['EBV'] = 0.0
model_filters = dict()
if 'S' in fibermap['PHOTSYS']:
for filter_name in ['DECAM_G', 'DECAM_R', 'DECAM_Z']:
model_filters[filter_name] = load_filter(filter_name)
if 'N' in fibermap['PHOTSYS']:
for filter_name in ['BASS_G', 'BASS_R', 'MZLS_Z']:
model_filters[filter_name] = load_filter(filter_name)
if len(model_filters) == 0:
raise ValueError("No filters loaded; neither 'N' nor 'S' in PHOTSYS?")
log.info("computing model mags for %s" % sorted(model_filters.keys()))
model_mags = dict()
fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
for filter_name, filter_response in model_filters.items():
model_mags[filter_name] = filter_response.get_ab_magnitude(
stdflux * fluxunits, stdwave)
log.info("done computing model mags")
# LOOP ON STARS TO FIND BEST MODEL
############################################
linear_coefficients = np.zeros((nstars, stdflux.shape[0]))
chi2dof = np.zeros((nstars))
redshift = np.zeros((nstars))
normflux = []
star_mags = dict()
star_unextincted_mags = dict()
for band in ['G', 'R', 'Z']:
star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_' + band])
star_unextincted_mags[band] = 22.5 - 2.5 * np.log10(
fibermap['FLUX_' + band] / fibermap['MW_TRANSMISSION_' + band])
star_colors = dict()
star_colors['G-R'] = star_mags['G'] - star_mags['R']
star_colors['R-Z'] = star_mags['R'] - star_mags['Z']
star_unextincted_colors = dict()
star_unextincted_colors[
'G-R'] = star_unextincted_mags['G'] - star_unextincted_mags['R']
star_unextincted_colors[
'R-Z'] = star_unextincted_mags['R'] - star_unextincted_mags['Z']
fitted_model_colors = np.zeros(nstars)
for star in range(nstars):
log.info("finding best model for observed star #%d" % star)
# np.array of wave,flux,ivar,resol
wave = {}
flux = {}
ivar = {}
resolution_data = {}
for camera in frames:
for i, frame in enumerate(frames[camera]):
identifier = "%s-%d" % (camera, i)
wave[identifier] = frame.wave
flux[identifier] = frame.flux[star]
ivar[identifier] = frame.ivar[star]
resolution_data[identifier] = frame.resolution_data[star]
# preselect models based on magnitudes
if fibermap['PHOTSYS'][star] == 'N':
if args.color == 'G-R':
model_colors = model_mags['BASS_G'] - model_mags['BASS_R']
elif args.color == 'R-Z':
model_colors = model_mags['BASS_R'] - model_mags['MZLS_Z']
else:
raise ValueError('Unknown color {}'.format(args.color))
else:
if args.color == 'G-R':
model_colors = model_mags['DECAM_G'] - model_mags['DECAM_R']
elif args.color == 'R-Z':
model_colors = model_mags['DECAM_R'] - model_mags['DECAM_Z']
else:
raise ValueError('Unknown color {}'.format(args.color))
color_diff = model_colors - star_unextincted_colors[args.color][star]
selection = np.abs(color_diff) < args.delta_color
# smallest cube in parameter space including this selection (needed for interpolation)
new_selection = (teff >= np.min(teff[selection])) & (teff <= np.max(
teff[selection]))
new_selection &= (logg >= np.min(logg[selection])) & (logg <= np.max(
logg[selection]))
new_selection &= (feh >= np.min(feh[selection])) & (feh <= np.max(
feh[selection]))
selection = np.where(new_selection)[0]
log.info(
"star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d"
% (star, starfibers[star], args.color,
star_unextincted_colors[args.color][star], selection.size,
stdflux.shape[0]))
# Match unextincted standard stars to data
coefficients, redshift[star], chi2dof[star] = match_templates(
wave,
flux,
ivar,
resolution_data,
stdwave,
stdflux[selection],
teff[selection],
logg[selection],
feh[selection],
ncpu=args.ncpu,
z_max=args.z_max,
z_res=args.z_res,
template_error=args.template_error)
linear_coefficients[star, selection] = coefficients
log.info(
'Star Fiber: {0}; TEFF: {1}; LOGG: {2}; FEH: {3}; Redshift: {4}; Chisq/dof: {5}'
.format(starfibers[star], np.inner(teff,
linear_coefficients[star]),
np.inner(logg, linear_coefficients[star]),
np.inner(feh, linear_coefficients[star]), redshift[star],
chi2dof[star]))
# Apply redshift to original spectrum at full resolution
model = np.zeros(stdwave.size)
redshifted_stdwave = stdwave * (1 + redshift[star])
for i, c in enumerate(linear_coefficients[star]):
if c != 0:
model += c * np.interp(stdwave, redshifted_stdwave, stdflux[i])
# Apply dust extinction to the model
model *= dust_transmission(stdwave, fibermap['EBV'][star])
# Compute final color of dust-extincted model
if fibermap['PHOTSYS'][star] == 'N':
if args.color == 'G-R':
model_mag1 = model_filters['BASS_G'].get_ab_magnitude(
model * fluxunits, stdwave)
model_mag2 = model_filters['BASS_R'].get_ab_magnitude(
model * fluxunits, stdwave)
model_magr = model_mag2
elif args.color == 'R-Z':
model_mag1 = model_filters['BASS_R'].get_ab_magnitude(
model * fluxunits, stdwave)
model_mag2 = model_filters['MZLS_Z'].get_ab_magnitude(
model * fluxunits, stdwave)
model_magr = model_mag1
else:
raise ValueError('Unknown color {}'.format(args.color))
else:
if args.color == 'G-R':
model_mag1 = model_filters['DECAM_G'].get_ab_magnitude(
model * fluxunits, stdwave)
model_mag2 = model_filters['DECAM_R'].get_ab_magnitude(
model * fluxunits, stdwave)
model_magr = model_mag2
elif args.color == 'R-Z':
model_mag1 = model_filters['DECAM_R'].get_ab_magnitude(
model * fluxunits, stdwave)
model_mag2 = model_filters['DECAM_Z'].get_ab_magnitude(
model * fluxunits, stdwave)
model_magr = model_mag1
else:
raise ValueError('Unknown color {}'.format(args.color))
fitted_model_colors[star] = model_mag1 - model_mag2
#- TODO: move this back into normalize_templates, at the cost of
#- recalculating a model magnitude?
# Normalize the best model using reported magnitude
scalefac = 10**((model_magr - star_mags['R'][star]) / 2.5)
log.info('scaling R mag {} to {} using scale {}'.format(
model_magr, star_mags['R'][star], scalefac))
normflux.append(model * scalefac)
# Now write the normalized flux for all best models to a file
normflux = np.array(normflux)
data = {}
data['LOGG'] = linear_coefficients.dot(logg)
data['TEFF'] = linear_coefficients.dot(teff)
data['FEH'] = linear_coefficients.dot(feh)
data['CHI2DOF'] = chi2dof
data['REDSHIFT'] = redshift
data['COEFF'] = linear_coefficients
data['DATA_%s' % args.color] = star_colors[args.color]
data['MODEL_%s' % args.color] = fitted_model_colors
io.write_stdstar_models(args.outfile, normflux, stdwave, starfibers, data)
def main(args) :
log=get_logger()
cmd = ['desi_compute_fluxcalibration',]
for key, value in args.__dict__.items():
if value is not None:
cmd += ['--'+key, str(value)]
cmd = ' '.join(cmd)
log.info(cmd)
log.info("read frame")
# read frame
frame = read_frame(args.infile)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel=read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux,model_wave,model_fibers,model_metadata=read_stdstar_models(args.models)
if args.chi2cut > 0 :
ok = np.where(model_metadata["CHI2DOF"]<args.chi2cut)[0]
if ok.size == 0 :
log.error("chi2cut has discarded all stars")
sys.exit(12)
nstars=model_flux.shape[0]
nbad=nstars-ok.size
if nbad>0 :
log.warning("discarding %d star(s) out of %d because of chi2cut"%(nbad,nstars))
model_flux=model_flux[ok]
model_fibers=model_fibers[ok]
model_metadata=model_metadata[:][ok]
if args.delta_color_cut > 0 :
ok = np.where(np.abs(model_metadata["MODEL_G-R"]-model_metadata["DATA_G-R"])<args.delta_color_cut)[0]
nstars=model_flux.shape[0]
nbad=nstars-ok.size
if nbad>0 :
log.warning("discarding %d star(s) out of %d because |delta_color|>%f"%(nbad,nstars,args.delta_color_cut))
model_flux=model_flux[ok]
model_fibers=model_fibers[ok]
model_metadata=model_metadata[:][ok]
# automatically reject stars that ar chi2 outliers
if args.chi2cut_nsig > 0 :
mchi2=np.median(model_metadata["CHI2DOF"])
rmschi2=np.std(model_metadata["CHI2DOF"])
maxchi2=mchi2+args.chi2cut_nsig*rmschi2
ok=np.where(model_metadata["CHI2DOF"]<=maxchi2)[0]
nstars=model_flux.shape[0]
nbad=nstars-ok.size
if nbad>0 :
log.warning("discarding %d star(s) out of %d because reduced chi2 outliers (at %d sigma, giving rchi2<%f )"%(nbad,nstars,args.chi2cut_nsig,maxchi2))
model_flux=model_flux[ok]
model_fibers=model_fibers[ok]
model_metadata=model_metadata[:][ok]
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
## check whether star fibers from args.models are consistent with fibers from fibermap
## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
fibermap_std_indices = np.where(isStdStar(fibermap['DESI_TARGET']))[0]
if np.any(~np.in1d(model_fibers%500, fibermap_std_indices)):
for i in model_fibers%500:
log.error("inconsistency with spectrum {}, OBJTYPE='{}', DESI_TARGET={} in fibermap".format(
(i, fibermap["OBJTYPE"][i], fibermap["DESI_TARGET"][i])))
sys.exit(12)
fluxcalib = compute_flux_calibration(frame, model_wave, model_flux, model_fibers%500)
# QA
if (args.qafile is not None):
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR'])
# Run
#import pdb; pdb.set_trace()
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s"%args.outfile)
def main(args):
log = get_logger()
cmd = [
'desi_compute_fluxcalibration',
]
for key, value in args.__dict__.items():
if value is not None:
cmd += ['--' + key, str(value)]
cmd = ' '.join(cmd)
log.info(cmd)
log.info("read frame")
# read frame
frame = read_frame(args.infile)
# Set fibermask flagged spectra to have 0 flux and variance
frame = get_fiberbitmasked_frame(frame,
bitmask='flux',
ivar_framemask=True)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel = read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models(
args.models)
ok = np.ones(len(model_metadata), dtype=bool)
if args.chi2cut > 0:
log.info("Apply cut CHI2DOF<{}".format(args.chi2cut))
ok &= (model_metadata["CHI2DOF"] < args.chi2cut)
if args.delta_color_cut > 0:
log.info("Apply cut |delta color|<{}".format(args.delta_color_cut))
ok &= (np.abs(model_metadata["MODEL_G-R"] - model_metadata["DATA_G-R"])
< args.delta_color_cut)
if args.min_color is not None:
log.info("Apply cut DATA_G-R>{}".format(args.min_color))
ok &= (model_metadata["DATA_G-R"] > args.min_color)
if args.chi2cut_nsig > 0:
# automatically reject stars that ar chi2 outliers
mchi2 = np.median(model_metadata["CHI2DOF"])
rmschi2 = np.std(model_metadata["CHI2DOF"])
maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2
log.info("Apply cut CHI2DOF<{} based on chi2cut_nsig={}".format(
maxchi2, args.chi2cut_nsig))
ok &= (model_metadata["CHI2DOF"] <= maxchi2)
ok = np.where(ok)[0]
if ok.size == 0:
log.error("cuts discarded all stars")
sys.exit(12)
nstars = model_flux.shape[0]
nbad = nstars - ok.size
if nbad > 0:
log.warning("discarding %d star(s) out of %d because of cuts" %
(nbad, nstars))
model_flux = model_flux[ok]
model_fibers = model_fibers[ok]
model_metadata = model_metadata[:][ok]
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
## check whether star fibers from args.models are consistent with fibers from fibermap
## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
fibermap_std_indices = np.where(isStdStar(fibermap))[0]
if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)):
target_colnames, target_masks, survey = main_cmx_or_sv(fibermap)
colname = target_colnames[0]
for i in model_fibers % 500:
log.error(
"inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap"
.format(i, fibermap["OBJTYPE"][i], colname,
fibermap[colname][i]))
sys.exit(12)
# Make sure the fibers of interest aren't entirely masked.
if np.sum(
np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) ==
frame.nwave) == len(model_fibers):
log.warning('All standard-star spectra are masked!')
return
fluxcalib = compute_flux_calibration(
frame,
model_wave,
model_flux,
model_fibers % 500,
highest_throughput_nstars=args.highest_throughput)
# QA
if (args.qafile is not None):
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame_meta=frame.meta,
flavor=frame.meta['FLAVOR'])
# Run
#import pdb; pdb.set_trace()
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s" % args.outfile)
def main(args):
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
log = get_logger()
log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
(args.color, args.delta_color))
log.info('multiprocess parallelizing with {} processes'.format(args.ncpu))
# READ DATA
############################################
# First loop through and group by exposure and spectrograph
frames_by_expid = {}
for filename in args.frames:
log.info("reading %s" % filename)
frame = io.read_frame(filename)
expid = safe_read_key(frame.meta, "EXPID")
camera = safe_read_key(frame.meta, "CAMERA").strip().lower()
spec = camera[1]
uniq_key = (expid, spec)
if uniq_key in frames_by_expid.keys():
frames_by_expid[uniq_key][camera] = frame
else:
frames_by_expid[uniq_key] = {camera: frame}
frames = {}
flats = {}
skies = {}
spectrograph = None
starfibers = None
starindices = None
fibermap = None
# For each unique expid,spec pair, get the logical OR of the FIBERSTATUS for all
# cameras and then proceed with extracting the frame information
# once we modify the fibermap FIBERSTATUS
for (expid, spec), camdict in frames_by_expid.items():
fiberstatus = None
for frame in camdict.values():
if fiberstatus is None:
fiberstatus = frame.fibermap['FIBERSTATUS'].data.copy()
else:
fiberstatus |= frame.fibermap['FIBERSTATUS']
for camera, frame in camdict.items():
frame.fibermap['FIBERSTATUS'] |= fiberstatus
# Set fibermask flagged spectra to have 0 flux and variance
frame = get_fiberbitmasked_frame(frame,
bitmask='stdstars',
ivar_framemask=True)
frame_fibermap = frame.fibermap
frame_starindices = np.where(isStdStar(frame_fibermap))[0]
#- Confirm that all fluxes have entries but trust targeting bits
#- to get basic magnitude range correct
keep = np.ones(len(frame_starindices), dtype=bool)
for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']: #- and W1 and W2?
keep &= frame_fibermap[colname][frame_starindices] > 10**(
(22.5 - 30) / 2.5)
keep &= frame_fibermap[colname][frame_starindices] < 10**(
(22.5 - 0) / 2.5)
frame_starindices = frame_starindices[keep]
if spectrograph is None:
spectrograph = frame.spectrograph
fibermap = frame_fibermap
starindices = frame_starindices
starfibers = fibermap["FIBER"][starindices]
elif spectrograph != frame.spectrograph:
log.error("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
raise ValueError("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
elif starindices.size != frame_starindices.size or np.sum(
starindices != frame_starindices) > 0:
log.error("incompatible fibermap")
raise ValueError("incompatible fibermap")
if not camera in frames:
frames[camera] = []
frames[camera].append(frame)
# possibly cleanup memory
del frames_by_expid
for filename in args.skymodels:
log.info("reading %s" % filename)
sky = io.read_sky(filename)
camera = safe_read_key(sky.header, "CAMERA").strip().lower()
if not camera in skies:
skies[camera] = []
skies[camera].append(sky)
for filename in args.fiberflats:
log.info("reading %s" % filename)
flat = io.read_fiberflat(filename)
camera = safe_read_key(flat.header, "CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if camera in flats:
log.warning(
"cannot handle several flats of same camera (%s), will use only the first one"
% camera)
#raise ValueError("cannot handle several flats of same camera (%s)"%camera)
else:
flats[camera] = flat
if starindices.size == 0:
log.error("no STD star found in fibermap")
raise ValueError("no STD star found in fibermap")
log.info("found %d STD stars" % starindices.size)
log.warning("Not using flux errors for Standard Star fits!")
# DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
############################################
# since poping dict, we need to copy keys to iterate over to avoid
# RuntimeError due to changing dict
frame_cams = list(frames.keys())
for cam in frame_cams:
if not cam in skies:
log.warning("Missing sky for %s" % cam)
frames.pop(cam)
continue
if not cam in flats:
log.warning("Missing flat for %s" % cam)
frames.pop(cam)
continue
flat = flats[cam]
for frame, sky in zip(frames[cam], skies[cam]):
frame.flux = frame.flux[starindices]
frame.ivar = frame.ivar[starindices]
frame.ivar *= (frame.mask[starindices] == 0)
frame.ivar *= (sky.ivar[starindices] != 0)
frame.ivar *= (sky.mask[starindices] == 0)
frame.ivar *= (flat.ivar[starindices] != 0)
frame.ivar *= (flat.mask[starindices] == 0)
frame.flux *= (frame.ivar > 0) # just for clean plots
for star in range(frame.flux.shape[0]):
ok = np.where((frame.ivar[star] > 0)
& (flat.fiberflat[star] != 0))[0]
if ok.size > 0:
frame.flux[star] = frame.flux[star] / flat.fiberflat[
star] - sky.flux[star]
frame.resolution_data = frame.resolution_data[starindices]
# CHECK S/N
############################################
# for each band in 'brz', record quadratic sum of median S/N across wavelength
snr = dict()
for band in ['b', 'r', 'z']:
snr[band] = np.zeros(starindices.size)
for cam in frames:
band = cam[0].lower()
for frame in frames[cam]:
msnr = np.median(frame.flux * np.sqrt(frame.ivar) /
np.sqrt(np.gradient(frame.wave)),
axis=1) # median SNR per sqrt(A.)
msnr *= (msnr > 0)
snr[band] = np.sqrt(snr[band]**2 + msnr**2)
log.info("SNR(B) = {}".format(snr['b']))
###############################
max_number_of_stars = 50
min_blue_snr = 4.
###############################
indices = np.argsort(snr['b'])[::-1][:max_number_of_stars]
validstars = np.where(snr['b'][indices] > min_blue_snr)[0]
#- TODO: later we filter on models based upon color, thus throwing
#- away very blue stars for which we don't have good models.
log.info("Number of stars with median stacked blue S/N > {} /sqrt(A) = {}".
format(min_blue_snr, validstars.size))
if validstars.size == 0:
log.error("No valid star")
sys.exit(12)
validstars = indices[validstars]
for band in ['b', 'r', 'z']:
snr[band] = snr[band][validstars]
log.info("BLUE SNR of selected stars={}".format(snr['b']))
for cam in frames:
for frame in frames[cam]:
frame.flux = frame.flux[validstars]
frame.ivar = frame.ivar[validstars]
frame.resolution_data = frame.resolution_data[validstars]
starindices = starindices[validstars]
starfibers = starfibers[validstars]
nstars = starindices.size
fibermap = Table(fibermap[starindices])
# MASK OUT THROUGHPUT DIP REGION
############################################
mask_throughput_dip_region = True
if mask_throughput_dip_region:
wmin = 4300.
wmax = 4500.
log.warning(
"Masking out the wavelength region [{},{}]A in the standard star fit"
.format(wmin, wmax))
for cam in frames:
for frame in frames[cam]:
ii = np.where((frame.wave >= wmin) & (frame.wave <= wmax))[0]
if ii.size > 0:
frame.ivar[:, ii] = 0
# READ MODELS
############################################
log.info("reading star models in %s" % args.starmodels)
stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
args.starmodels)
# COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
############################################
#- Support older fibermaps
if 'PHOTSYS' not in fibermap.colnames:
log.warning('Old fibermap format; using defaults for missing columns')
log.warning(" PHOTSYS = 'S'")
log.warning(" MW_TRANSMISSION_G/R/Z = 1.0")
log.warning(" EBV = 0.0")
fibermap['PHOTSYS'] = 'S'
fibermap['MW_TRANSMISSION_G'] = 1.0
fibermap['MW_TRANSMISSION_R'] = 1.0
fibermap['MW_TRANSMISSION_Z'] = 1.0
fibermap['EBV'] = 0.0
model_filters = dict()
for band in ["G", "R", "Z"]:
for photsys in np.unique(fibermap['PHOTSYS']):
model_filters[band + photsys] = load_legacy_survey_filter(
band=band, photsys=photsys)
log.info("computing model mags for %s" % sorted(model_filters.keys()))
model_mags = dict()
fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
for filter_name, filter_response in model_filters.items():
model_mags[filter_name] = filter_response.get_ab_magnitude(
stdflux * fluxunits, stdwave)
log.info("done computing model mags")
# LOOP ON STARS TO FIND BEST MODEL
############################################
linear_coefficients = np.zeros((nstars, stdflux.shape[0]))
chi2dof = np.zeros((nstars))
redshift = np.zeros((nstars))
normflux = []
star_mags = dict()
star_unextincted_mags = dict()
for band in ['G', 'R', 'Z']:
star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_' + band])
star_unextincted_mags[band] = 22.5 - 2.5 * np.log10(
fibermap['FLUX_' + band] / fibermap['MW_TRANSMISSION_' + band])
star_colors = dict()
star_colors['G-R'] = star_mags['G'] - star_mags['R']
star_colors['R-Z'] = star_mags['R'] - star_mags['Z']
star_unextincted_colors = dict()
star_unextincted_colors[
'G-R'] = star_unextincted_mags['G'] - star_unextincted_mags['R']
star_unextincted_colors[
'R-Z'] = star_unextincted_mags['R'] - star_unextincted_mags['Z']
fitted_model_colors = np.zeros(nstars)
for star in range(nstars):
log.info("finding best model for observed star #%d" % star)
# np.array of wave,flux,ivar,resol
wave = {}
flux = {}
ivar = {}
resolution_data = {}
for camera in frames:
for i, frame in enumerate(frames[camera]):
identifier = "%s-%d" % (camera, i)
wave[identifier] = frame.wave
flux[identifier] = frame.flux[star]
ivar[identifier] = frame.ivar[star]
resolution_data[identifier] = frame.resolution_data[star]
# preselect models based on magnitudes
photsys = fibermap['PHOTSYS'][star]
if not args.color in ['G-R', 'R-Z']:
raise ValueError('Unknown color {}'.format(args.color))
bands = args.color.split("-")
model_colors = model_mags[bands[0] + photsys] - model_mags[bands[1] +
photsys]
color_diff = model_colors - star_unextincted_colors[args.color][star]
selection = np.abs(color_diff) < args.delta_color
if np.sum(selection) == 0:
log.warning("no model in the selected color range for this star")
continue
# smallest cube in parameter space including this selection (needed for interpolation)
new_selection = (teff >= np.min(teff[selection])) & (teff <= np.max(
teff[selection]))
new_selection &= (logg >= np.min(logg[selection])) & (logg <= np.max(
logg[selection]))
new_selection &= (feh >= np.min(feh[selection])) & (feh <= np.max(
feh[selection]))
selection = np.where(new_selection)[0]
log.info(
"star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d"
% (star, starfibers[star], args.color,
star_unextincted_colors[args.color][star], selection.size,
stdflux.shape[0]))
# Match unextincted standard stars to data
coefficients, redshift[star], chi2dof[star] = match_templates(
wave,
flux,
ivar,
resolution_data,
stdwave,
stdflux[selection],
teff[selection],
logg[selection],
feh[selection],
ncpu=args.ncpu,
z_max=args.z_max,
z_res=args.z_res,
template_error=args.template_error)
linear_coefficients[star, selection] = coefficients
log.info(
'Star Fiber: {}; TEFF: {:.3f}; LOGG: {:.3f}; FEH: {:.3f}; Redshift: {:g}; Chisq/dof: {:.3f}'
.format(starfibers[star], np.inner(teff,
linear_coefficients[star]),
np.inner(logg, linear_coefficients[star]),
np.inner(feh, linear_coefficients[star]), redshift[star],
chi2dof[star]))
# Apply redshift to original spectrum at full resolution
model = np.zeros(stdwave.size)
redshifted_stdwave = stdwave * (1 + redshift[star])
for i, c in enumerate(linear_coefficients[star]):
if c != 0:
model += c * np.interp(stdwave, redshifted_stdwave, stdflux[i])
# Apply dust extinction to the model
log.info("Applying MW dust extinction to star {} with EBV = {}".format(
star, fibermap['EBV'][star]))
model *= dust_transmission(stdwave, fibermap['EBV'][star])
# Compute final color of dust-extincted model
photsys = fibermap['PHOTSYS'][star]
if not args.color in ['G-R', 'R-Z']:
raise ValueError('Unknown color {}'.format(args.color))
bands = args.color.split("-")
model_mag1 = model_filters[bands[0] + photsys].get_ab_magnitude(
model * fluxunits, stdwave)
model_mag2 = model_filters[bands[1] + photsys].get_ab_magnitude(
model * fluxunits, stdwave)
fitted_model_colors[star] = model_mag1 - model_mag2
if bands[0] == "R":
model_magr = model_mag1
elif bands[1] == "R":
model_magr = model_mag2
#- TODO: move this back into normalize_templates, at the cost of
#- recalculating a model magnitude?
# Normalize the best model using reported magnitude
scalefac = 10**((model_magr - star_mags['R'][star]) / 2.5)
log.info('scaling R mag {:.3f} to {:.3f} using scale {}'.format(
model_magr, star_mags['R'][star], scalefac))
normflux.append(model * scalefac)
# Now write the normalized flux for all best models to a file
normflux = np.array(normflux)
fitted_stars = np.where(chi2dof != 0)[0]
if fitted_stars.size == 0:
log.error("No star has been fit.")
sys.exit(12)
data = {}
data['LOGG'] = linear_coefficients[fitted_stars, :].dot(logg)
data['TEFF'] = linear_coefficients[fitted_stars, :].dot(teff)
data['FEH'] = linear_coefficients[fitted_stars, :].dot(feh)
data['CHI2DOF'] = chi2dof[fitted_stars]
data['REDSHIFT'] = redshift[fitted_stars]
data['COEFF'] = linear_coefficients[fitted_stars, :]
data['DATA_%s' % args.color] = star_colors[args.color][fitted_stars]
data['MODEL_%s' % args.color] = fitted_model_colors[fitted_stars]
data['BLUE_SNR'] = snr['b'][fitted_stars]
data['RED_SNR'] = snr['r'][fitted_stars]
data['NIR_SNR'] = snr['z'][fitted_stars]
io.write_stdstar_models(args.outfile, normflux, stdwave,
starfibers[fitted_stars], data)
def main(args):
log = get_logger()
cmd = [
'desi_compute_fluxcalibration',
]
for key, value in args.__dict__.items():
if value is not None:
cmd += ['--' + key, str(value)]
cmd = ' '.join(cmd)
log.info(cmd)
log.info("read frame")
# read frame
frame = read_frame(args.infile)
# Set fibermask flagged spectra to have 0 flux and variance
frame = get_fiberbitmasked_frame(frame,
bitmask='flux',
ivar_framemask=True)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel = read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models(
args.models)
ok = np.ones(len(model_metadata), dtype=bool)
if args.chi2cut > 0:
log.info("apply cut CHI2DOF<{}".format(args.chi2cut))
good = (model_metadata["CHI2DOF"] < args.chi2cut)
bad = ~good
ok &= good
if np.any(bad):
log.info(" discard {} stars with CHI2DOF= {}".format(
np.sum(bad), list(model_metadata["CHI2DOF"][bad])))
legacy_filters = ('G-R', 'R-Z')
gaia_filters = ('GAIA-BP-RP', 'GAIA-G-RP')
model_column_list = model_metadata.columns.names
if args.color is None:
if 'MODEL_G-R' in model_column_list:
color = 'G-R'
elif 'MODEL_GAIA-BP-RP' in model_column_list:
log.info('Using Gaia filters')
color = 'GAIA-BP-RP'
else:
log.error(
"Can't find either G-R or BP-RP color in the model file.")
sys.exit(15)
else:
if args.color not in legacy_filters and args.color not in gaia_filters:
log.error(
'Color name {} is not allowed, must be one of {} {}'.format(
args.color, legacy_filters, gaia_filters))
sys.exit(14)
color = args.color
if color not in model_column_list:
# This should't happen
log.error(
'The color {} was not computed in the models'.format(color))
sys.exit(16)
if args.delta_color_cut > 0:
log.info("apply cut |delta color|<{}".format(args.delta_color_cut))
good = (np.abs(model_metadata["MODEL_" + color] -
model_metadata["DATA_" + color]) < args.delta_color_cut)
bad = ok & (~good)
ok &= good
if np.any(bad):
vals = model_metadata["MODEL_" +
color][bad] - model_metadata["DATA_" +
color][bad]
log.info(" discard {} stars with dcolor= {}".format(
np.sum(bad), list(vals)))
if args.min_color is not None:
log.info("apply cut DATA_{}>{}".format(color, args.min_color))
good = (model_metadata["DATA_{}".format(color)] > args.min_color)
bad = ok & (~good)
ok &= good
if np.any(bad):
vals = model_metadata["DATA_{}".format(color)][bad]
log.info(" discard {} stars with {}= {}".format(
np.sum(bad), color, list(vals)))
if args.chi2cut_nsig > 0:
# automatically reject stars that ar chi2 outliers
mchi2 = np.median(model_metadata["CHI2DOF"])
rmschi2 = np.std(model_metadata["CHI2DOF"])
maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2
log.info("apply cut CHI2DOF<{} based on chi2cut_nsig={}".format(
maxchi2, args.chi2cut_nsig))
good = (model_metadata["CHI2DOF"] <= maxchi2)
bad = ok & (~good)
ok &= good
if np.any(bad):
log.info(" discard {} stars with CHI2DOF={}".format(
np.sum(bad), list(model_metadata["CHI2DOF"][bad])))
ok = np.where(ok)[0]
if ok.size == 0:
log.error("selection cuts discarded all stars")
sys.exit(12)
nstars = model_flux.shape[0]
nbad = nstars - ok.size
if nbad > 0:
log.warning("discarding %d star(s) out of %d because of cuts" %
(nbad, nstars))
model_flux = model_flux[ok]
model_fibers = model_fibers[ok]
model_metadata = model_metadata[:][ok]
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
## check whether star fibers from args.models are consistent with fibers from fibermap
## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
fibermap_std_indices = np.where(isStdStar(fibermap))[0]
if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)):
target_colnames, target_masks, survey = main_cmx_or_sv(fibermap)
colname = target_colnames[0]
for i in model_fibers % 500:
log.error(
"inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap"
.format(i, fibermap["OBJTYPE"][i], colname,
fibermap[colname][i]))
sys.exit(12)
# Make sure the fibers of interest aren't entirely masked.
if np.sum(
np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) ==
frame.nwave) == len(model_fibers):
log.warning('All standard-star spectra are masked!')
return
fluxcalib = compute_flux_calibration(
frame,
model_wave,
model_flux,
model_fibers % 500,
highest_throughput_nstars=args.highest_throughput,
exposure_seeing_fwhm=args.seeing_fwhm)
# QA
if (args.qafile is not None):
from desispec.io import write_qa_frame
from desispec.io.qa import load_qa_frame
from desispec.qa import qa_plots
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame_meta=frame.meta,
flavor=frame.meta['FLAVOR'])
# Run
#import pdb; pdb.set_trace()
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)
# record inputs
frame.meta['IN_FRAME'] = shorten_filename(args.infile)
frame.meta['IN_SKY'] = shorten_filename(args.sky)
frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat)
frame.meta['STDMODEL'] = shorten_filename(args.models)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s" % args.outfile)
def main(args):
log = get_logger()
cmd = [
'desi_compute_fluxcalibration',
]
for key, value in args.__dict__.items():
if value is not None:
cmd += ['--' + key, str(value)]
cmd = ' '.join(cmd)
log.info(cmd)
log.info("read frame")
# read frame
frame = read_frame(args.infile)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel = read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models(
args.models)
if args.chi2cut > 0:
ok = np.where(model_metadata["CHI2DOF"] < args.chi2cut)[0]
if ok.size == 0:
log.error("chi2cut has discarded all stars")
sys.exit(12)
nstars = model_flux.shape[0]
nbad = nstars - ok.size
if nbad > 0:
log.warning("discarding %d star(s) out of %d because of chi2cut" %
(nbad, nstars))
model_flux = model_flux[ok]
model_fibers = model_fibers[ok]
model_metadata = model_metadata[:][ok]
if args.delta_color_cut > 0:
ok = np.where(
np.abs(model_metadata["MODEL_G-R"] -
model_metadata["DATA_G-R"]) < args.delta_color_cut)[0]
nstars = model_flux.shape[0]
nbad = nstars - ok.size
if nbad > 0:
log.warning(
"discarding %d star(s) out of %d because |delta_color|>%f" %
(nbad, nstars, args.delta_color_cut))
model_flux = model_flux[ok]
model_fibers = model_fibers[ok]
model_metadata = model_metadata[:][ok]
# automatically reject stars that ar chi2 outliers
if args.chi2cut_nsig > 0:
mchi2 = np.median(model_metadata["CHI2DOF"])
rmschi2 = np.std(model_metadata["CHI2DOF"])
maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2
ok = np.where(model_metadata["CHI2DOF"] <= maxchi2)[0]
nstars = model_flux.shape[0]
nbad = nstars - ok.size
if nbad > 0:
log.warning(
"discarding %d star(s) out of %d because reduced chi2 outliers (at %d sigma, giving rchi2<%f )"
% (nbad, nstars, args.chi2cut_nsig, maxchi2))
model_flux = model_flux[ok]
model_fibers = model_fibers[ok]
model_metadata = model_metadata[:][ok]
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
## check whether star fibers from args.models are consistent with fibers from fibermap
## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
fibermap_std_indices = np.where(isStdStar(fibermap))[0]
if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)):
if 'DESI_TARGET' in fibermap:
colname = 'DESI_TARGET'
else:
colname = 'SV1_DESI_TARGET' #- TODO: could become SV2_DESI_TARGET
for i in model_fibers % 500:
log.error(
"inconsistency with spectrum {}, OBJTYPE='{}', {}={} in fibermap"
.format((i, fibermap["OBJTYPE"][i], colname,
fibermap[colname][i])))
sys.exit(12)
fluxcalib = compute_flux_calibration(frame, model_wave, model_flux,
model_fibers % 500)
# QA
if (args.qafile is not None):
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame,
flavor=frame.meta['FLAVOR'])
# Run
#import pdb; pdb.set_trace()
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s" % args.outfile)
def s2nfit(frame, camera, params):
"""
Signal vs. Noise With fitting
Take flux and inverse variance arrays and calculate S/N for individual
targets (ELG, LRG, QSO, STD) and for each amplifier of the camera.
then fit snr=A*mag/sqrt(A*mag+B)
see http://arXiv.org/abs/0706.1062v2 for proper fitting of power-law distributions
it is not implemented here!
Instead we use scipy.optimize.curve_fit
Args:
frame: desispec.Frame object
camera: str, name of the camera
params: parameters dictionary for S/N
Returns:
qadict : dict
MEDIAN_SNR (ndarray, nfiber): Median S/N of light in each fiber
FIT_FILTER (str): Filter used for the fluxes
EXPTIME (float): Exposure time
XXX_FIBERID (list): Fibers matching ELG, LRG, BGS, etc.
SNR_MAG_TGT (list): List of lists with S/N and mag of ELG, LRG, BGS, etc.
FITCOEFF_TGT (list): List of fitted coefficients. Junk fits have np.nan
OBJLIST (list): List of object types analyzed (1 or more fiber)
"""
# Median snr
snr = frame.flux * np.sqrt(frame.ivar)
mediansnr = np.median(snr, axis=1)
qadict = {"MEDIAN_SNR": mediansnr}
exptime = frame.meta["EXPTIME"]
# Parse filters
if "Filter" in params:
thisfilter = params["Filter"]
elif camera[0] == 'b':
thisfilter = 'DECAM_G'
elif camera[0] == 'r':
thisfilter = 'DECAM_R'
else:
thisfilter = 'DECAM_Z'
qadict["FIT_FILTER"] = thisfilter
qadict["EXPTIME"] = exptime
if thisfilter in ('DECAM_G', 'BASS_G'):
photflux = frame.fibermap['FLUX_G']
elif thisfilter in ('DECAM_R', 'BASS_R'):
photflux = frame.fibermap['FLUX_R']
elif thisfilter in ('DECAM_Z', 'MZLS_Z'):
photflux = frame.fibermap['FLUX_Z']
else:
raise ValueError('Unknown filter {}'.format(thisfilter))
# - Loop over each target type, and associate SNR and image magnitudes for each type.
fitcoeff = []
snrmag = []
fitsnr = []
fitT = []
elgfibers = np.where((frame.fibermap['DESI_TARGET'] & desi_mask.ELG) != 0)[0]
lrgfibers = np.where((frame.fibermap['DESI_TARGET'] & desi_mask.LRG) != 0)[0]
qsofibers = np.where((frame.fibermap['DESI_TARGET'] & desi_mask.QSO) != 0)[0]
bgsfibers = np.where((frame.fibermap['DESI_TARGET'] & desi_mask.BGS_ANY) != 0)[0]
mwsfibers = np.where((frame.fibermap['DESI_TARGET'] & desi_mask.MWS_ANY) != 0)[0]
stdfibers = np.where(isStdStar(frame.fibermap))[0]
for T, fibers in (
['ELG', elgfibers],
['LRG', lrgfibers],
['QSO', qsofibers],
['BGS', bgsfibers],
['MWS', mwsfibers],
['STAR', stdfibers],
):
if len(fibers) == 0:
continue
# S/N of the fibers
medsnr = mediansnr[fibers]
mags = np.zeros(medsnr.shape)
fit_these = photflux[fibers] > 0
mags[fit_these] = 22.5 - 2.5 * np.log10(photflux[fibers][fit_these])
# Fit
try:
popt, pcov = optimize.curve_fit(s2n_flux_astro, photflux[fibers][fit_these].data,
medsnr[fit_these]/exptime**(1/2), p0=(0.02, 1.))
except RuntimeError:
fitcoeff.append([np.nan, np.nan])
else:
fitcoeff.append([popt[0], popt[1]])
# Save
fitT.append(T)
qadict["{:s}_FIBERID".format(T)] = fibers.tolist()
snr_mag = [medsnr.tolist(), mags.tolist()]
snrmag.append(snr_mag)
# Save
qadict["SNR_MAG_TGT"] = snrmag
qadict["FITCOEFF_TGT"] = fitcoeff
qadict["OBJLIST"] = fitT
# Return
return qadict, fitsnr
def main(args, comm=None):
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
log = get_logger()
log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
(args.color, args.delta_color))
if args.mpi or comm is not None:
from mpi4py import MPI
if comm is None:
comm = MPI.COMM_WORLD
size = comm.Get_size()
rank = comm.Get_rank()
if rank == 0:
log.info('mpi parallelizing with {} ranks'.format(size))
else:
comm = None
rank = 0
size = 1
# disable multiprocess by forcing ncpu = 1 when using MPI
if comm is not None:
ncpu = 1
if rank == 0:
log.info('disabling multiprocess (forcing ncpu = 1)')
else:
ncpu = args.ncpu
if ncpu > 1:
if rank == 0:
log.info(
'multiprocess parallelizing with {} processes'.format(ncpu))
if args.ignore_gpu and desispec.fluxcalibration.use_gpu:
# Opt-out of GPU usage
desispec.fluxcalibration.use_gpu = False
if rank == 0:
log.info('ignoring GPU')
elif desispec.fluxcalibration.use_gpu:
# Nothing to do here, GPU is used by default if available
if rank == 0:
log.info('using GPU')
else:
if rank == 0:
log.info('GPU not available')
std_targetids = None
if args.std_targetids is not None:
std_targetids = args.std_targetids
# READ DATA
############################################
# First loop through and group by exposure and spectrograph
frames_by_expid = {}
rows = list()
for filename in args.frames:
log.info("reading %s" % filename)
frame = io.read_frame(filename)
night = safe_read_key(frame.meta, "NIGHT")
expid = safe_read_key(frame.meta, "EXPID")
camera = safe_read_key(frame.meta, "CAMERA").strip().lower()
rows.append((night, expid, camera))
spec = camera[1]
uniq_key = (expid, spec)
if uniq_key in frames_by_expid.keys():
frames_by_expid[uniq_key][camera] = frame
else:
frames_by_expid[uniq_key] = {camera: frame}
input_frames_table = Table(rows=rows, names=('NIGHT', 'EXPID', 'TILEID'))
frames = {}
flats = {}
skies = {}
spectrograph = None
starfibers = None
starindices = None
fibermap = None
# For each unique expid,spec pair, get the logical OR of the FIBERSTATUS for all
# cameras and then proceed with extracting the frame information
# once we modify the fibermap FIBERSTATUS
for (expid, spec), camdict in frames_by_expid.items():
fiberstatus = None
for frame in camdict.values():
if fiberstatus is None:
fiberstatus = frame.fibermap['FIBERSTATUS'].data.copy()
else:
fiberstatus |= frame.fibermap['FIBERSTATUS']
for camera, frame in camdict.items():
frame.fibermap['FIBERSTATUS'] |= fiberstatus
# Set fibermask flagged spectra to have 0 flux and variance
frame = get_fiberbitmasked_frame(frame,
bitmask='stdstars',
ivar_framemask=True)
frame_fibermap = frame.fibermap
if std_targetids is None:
frame_starindices = np.where(isStdStar(frame_fibermap))[0]
else:
frame_starindices = np.nonzero(
np.isin(frame_fibermap['TARGETID'], std_targetids))[0]
#- Confirm that all fluxes have entries but trust targeting bits
#- to get basic magnitude range correct
keep_legacy = np.ones(len(frame_starindices), dtype=bool)
for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']: #- and W1 and W2?
keep_legacy &= frame_fibermap[colname][
frame_starindices] > 10**((22.5 - 30) / 2.5)
keep_legacy &= frame_fibermap[colname][
frame_starindices] < 10**((22.5 - 0) / 2.5)
keep_gaia = np.ones(len(frame_starindices), dtype=bool)
for colname in ['G', 'BP', 'RP']: #- and W1 and W2?
keep_gaia &= frame_fibermap[
'GAIA_PHOT_' + colname +
'_MEAN_MAG'][frame_starindices] > 10
keep_gaia &= frame_fibermap[
'GAIA_PHOT_' + colname +
'_MEAN_MAG'][frame_starindices] < 20
n_legacy_std = keep_legacy.sum()
n_gaia_std = keep_gaia.sum()
keep = keep_legacy | keep_gaia
# accept both types of standards for the time being
# keep the indices for gaia/legacy subsets
gaia_indices = keep_gaia[keep]
legacy_indices = keep_legacy[keep]
frame_starindices = frame_starindices[keep]
if spectrograph is None:
spectrograph = frame.spectrograph
fibermap = frame_fibermap
starindices = frame_starindices
starfibers = fibermap["FIBER"][starindices]
elif spectrograph != frame.spectrograph:
log.error("incompatible spectrographs {} != {}".format(
spectrograph, frame.spectrograph))
raise ValueError("incompatible spectrographs {} != {}".format(
spectrograph, frame.spectrograph))
elif starindices.size != frame_starindices.size or np.sum(
starindices != frame_starindices) > 0:
log.error("incompatible fibermap")
raise ValueError("incompatible fibermap")
if not camera in frames:
frames[camera] = []
frames[camera].append(frame)
# possibly cleanup memory
del frames_by_expid
for filename in args.skymodels:
log.info("reading %s" % filename)
sky = io.read_sky(filename)
camera = safe_read_key(sky.header, "CAMERA").strip().lower()
if not camera in skies:
skies[camera] = []
skies[camera].append(sky)
for filename in args.fiberflats:
log.info("reading %s" % filename)
flat = io.read_fiberflat(filename)
camera = safe_read_key(flat.header, "CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if camera in flats:
log.warning(
"cannot handle several flats of same camera (%s), will use only the first one"
% camera)
#raise ValueError("cannot handle several flats of same camera (%s)"%camera)
else:
flats[camera] = flat
# if color is not specified we decide on the fly
color = args.color
if color is not None:
if color[:4] == 'GAIA':
legacy_color = False
gaia_color = True
else:
legacy_color = True
gaia_color = False
if n_legacy_std == 0 and legacy_color:
raise Exception(
'Specified Legacy survey color, but no legacy standards')
if n_gaia_std == 0 and gaia_color:
raise Exception('Specified gaia color, but no gaia stds')
if starindices.size == 0:
log.error("no STD star found in fibermap")
raise ValueError("no STD star found in fibermap")
log.info("found %d STD stars" % starindices.size)
if n_legacy_std == 0:
gaia_std = True
if color is None:
color = 'GAIA-BP-RP'
else:
gaia_std = False
if color is None:
color = 'G-R'
if n_gaia_std > 0:
log.info('Gaia standards found but not used')
if gaia_std:
# The name of the reference filter to which we normalize the flux
ref_mag_name = 'GAIA-G'
color_band1, color_band2 = ['GAIA-' + _ for _ in color[5:].split('-')]
log.info(
"Using Gaia standards with color {} and normalizing to {}".format(
color, ref_mag_name))
# select appropriate subset of standards
starindices = starindices[gaia_indices]
starfibers = starfibers[gaia_indices]
else:
ref_mag_name = 'R'
color_band1, color_band2 = color.split('-')
log.info("Using Legacy standards with color {} and normalizing to {}".
format(color, ref_mag_name))
# select appropriate subset of standards
starindices = starindices[legacy_indices]
starfibers = starfibers[legacy_indices]
# excessive check but just in case
if not color in ['G-R', 'R-Z', 'GAIA-BP-RP', 'GAIA-G-RP']:
raise ValueError('Unknown color {}'.format(color))
# log.warning("Not using flux errors for Standard Star fits!")
# DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
############################################
# since poping dict, we need to copy keys to iterate over to avoid
# RuntimeError due to changing dict
frame_cams = list(frames.keys())
for cam in frame_cams:
if not cam in skies:
log.warning("Missing sky for %s" % cam)
frames.pop(cam)
continue
if not cam in flats:
log.warning("Missing flat for %s" % cam)
frames.pop(cam)
continue
flat = flats[cam]
for frame, sky in zip(frames[cam], skies[cam]):
frame.flux = frame.flux[starindices]
frame.ivar = frame.ivar[starindices]
frame.ivar *= (frame.mask[starindices] == 0)
frame.ivar *= (sky.ivar[starindices] != 0)
frame.ivar *= (sky.mask[starindices] == 0)
frame.ivar *= (flat.ivar[starindices] != 0)
frame.ivar *= (flat.mask[starindices] == 0)
frame.flux *= (frame.ivar > 0) # just for clean plots
for star in range(frame.flux.shape[0]):
ok = np.where((frame.ivar[star] > 0)
& (flat.fiberflat[star] != 0))[0]
if ok.size > 0:
frame.flux[star] = frame.flux[star] / flat.fiberflat[
star] - sky.flux[star]
frame.resolution_data = frame.resolution_data[starindices]
nframes = len(frames[cam])
if nframes > 1:
# optimal weights for the coaddition = ivar*throughput, not directly ivar,
# we estimate the relative throughput with median fluxes at this stage
medflux = np.zeros(nframes)
for i, frame in enumerate(frames[cam]):
if np.sum(frame.ivar > 0) == 0:
log.error(
"ivar=0 for all std star spectra in frame {}-{:08d}".
format(cam, frame.meta["EXPID"]))
else:
medflux[i] = np.median(frame.flux[frame.ivar > 0])
log.debug("medflux = {}".format(medflux))
medflux *= (medflux > 0)
if np.sum(medflux > 0) == 0:
log.error(
"mean median flux = 0, for all stars in fibers {}".format(
list(frames[cam][0].fibermap["FIBER"][starindices])))
sys.exit(12)
mmedflux = np.mean(medflux[medflux > 0])
weights = medflux / mmedflux
log.info("coadding {} exposures in cam {}, w={}".format(
nframes, cam, weights))
sw = np.zeros(frames[cam][0].flux.shape)
swf = np.zeros(frames[cam][0].flux.shape)
swr = np.zeros(frames[cam][0].resolution_data.shape)
for i, frame in enumerate(frames[cam]):
sw += weights[i] * frame.ivar
swf += weights[i] * frame.ivar * frame.flux
swr += weights[i] * frame.ivar[:,
None, :] * frame.resolution_data
coadded_frame = frames[cam][0]
coadded_frame.ivar = sw
coadded_frame.flux = swf / (sw + (sw == 0))
coadded_frame.resolution_data = swr / ((sw +
(sw == 0))[:, None, :])
frames[cam] = [coadded_frame]
# CHECK S/N
############################################
# for each band in 'brz', record quadratic sum of median S/N across wavelength
snr = dict()
for band in ['b', 'r', 'z']:
snr[band] = np.zeros(starindices.size)
for cam in frames:
band = cam[0].lower()
for frame in frames[cam]:
msnr = np.median(frame.flux * np.sqrt(frame.ivar) /
np.sqrt(np.gradient(frame.wave)),
axis=1) # median SNR per sqrt(A.)
msnr *= (msnr > 0)
snr[band] = np.sqrt(snr[band]**2 + msnr**2)
log.info("SNR(B) = {}".format(snr['b']))
###############################
max_number_of_stars = 50
min_blue_snr = 4.
###############################
indices = np.argsort(snr['b'])[::-1][:max_number_of_stars]
validstars = np.where(snr['b'][indices] > min_blue_snr)[0]
#- TODO: later we filter on models based upon color, thus throwing
#- away very blue stars for which we don't have good models.
log.info("Number of stars with median stacked blue S/N > {} /sqrt(A) = {}".
format(min_blue_snr, validstars.size))
if validstars.size == 0:
log.error("No valid star")
sys.exit(12)
validstars = indices[validstars]
for band in ['b', 'r', 'z']:
snr[band] = snr[band][validstars]
log.info("BLUE SNR of selected stars={}".format(snr['b']))
for cam in frames:
for frame in frames[cam]:
frame.flux = frame.flux[validstars]
frame.ivar = frame.ivar[validstars]
frame.resolution_data = frame.resolution_data[validstars]
starindices = starindices[validstars]
starfibers = starfibers[validstars]
nstars = starindices.size
fibermap = Table(fibermap[starindices])
# MASK OUT THROUGHPUT DIP REGION
############################################
mask_throughput_dip_region = True
if mask_throughput_dip_region:
wmin = 4300.
wmax = 4500.
log.warning(
"Masking out the wavelength region [{},{}]A in the standard star fit"
.format(wmin, wmax))
for cam in frames:
for frame in frames[cam]:
ii = np.where((frame.wave >= wmin) & (frame.wave <= wmax))[0]
if ii.size > 0:
frame.ivar[:, ii] = 0
# READ MODELS
############################################
log.info("reading star models in %s" % args.starmodels)
stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
args.starmodels)
# COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
############################################
#- Support older fibermaps
if 'PHOTSYS' not in fibermap.colnames:
log.warning('Old fibermap format; using defaults for missing columns')
log.warning(" PHOTSYS = 'S'")
log.warning(" EBV = 0.0")
fibermap['PHOTSYS'] = 'S'
fibermap['EBV'] = 0.0
if not np.in1d(np.unique(fibermap['PHOTSYS']), ['', 'N', 'S', 'G']).all():
log.error('Unknown PHOTSYS found')
raise Exception('Unknown PHOTSYS found')
# Fetching Filter curves
model_filters = dict()
for band in ["G", "R", "Z"]:
for photsys in np.unique(fibermap['PHOTSYS']):
if photsys in ['N', 'S']:
model_filters[band + photsys] = load_legacy_survey_filter(
band=band, photsys=photsys)
if len(model_filters) == 0:
log.info('No Legacy survey photometry identified in fibermap')
# I will always load gaia data even if we are fitting LS standards only
for band in ["G", "BP", "RP"]:
model_filters["GAIA-" + band] = load_gaia_filter(band=band, dr=2)
# Compute model mags on rank 0 and bcast result to other ranks
# This sidesteps an OOM event on Cori Haswell with "-c 2"
model_mags = None
if rank == 0:
log.info("computing model mags for %s" % sorted(model_filters.keys()))
model_mags = dict()
for filter_name in model_filters.keys():
model_mags[filter_name] = get_magnitude(stdwave, stdflux,
model_filters, filter_name)
log.info("done computing model mags")
if comm is not None:
model_mags = comm.bcast(model_mags, root=0)
# LOOP ON STARS TO FIND BEST MODEL
############################################
star_mags = dict()
star_unextincted_mags = dict()
if gaia_std and (fibermap['EBV'] == 0).all():
log.info("Using E(B-V) from SFD rather than FIBERMAP")
# when doing gaia standards, on old tiles the
# EBV is not set so we fetch from SFD (in original SFD scaling)
ebv = SFDMap(scaling=1).ebv(
acoo.SkyCoord(ra=fibermap['TARGET_RA'] * units.deg,
dec=fibermap['TARGET_DEC'] * units.deg))
else:
ebv = fibermap['EBV']
photometric_systems = np.unique(fibermap['PHOTSYS'])
if not gaia_std:
for band in ['G', 'R', 'Z']:
star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_' + band])
star_unextincted_mags[band] = np.zeros(star_mags[band].shape)
for photsys in photometric_systems:
r_band = extinction_total_to_selective_ratio(
band, photsys) # dimensionless
# r_band = a_band / E(B-V)
# E(B-V) is a difference of magnitudes (dimensionless)
# a_band = -2.5*log10(effective dust transmission) , dimensionless
# effective dust transmission =
# integral( SED(lambda) * filter_transmission(lambda,band) * dust_transmission(lambda,E(B-V)) dlamdba)
# / integral( SED(lambda) * filter_transmission(lambda,band) dlamdba)
selection = (fibermap['PHOTSYS'] == photsys)
a_band = r_band * ebv[selection] # dimensionless
star_unextincted_mags[band][selection] = 22.5 - 2.5 * np.log10(
fibermap['FLUX_' + band][selection]) - a_band
for band in ['G', 'BP', 'RP']:
star_mags['GAIA-' + band] = fibermap['GAIA_PHOT_' + band + '_MEAN_MAG']
for band, extval in gaia_extinction(star_mags['GAIA-G'],
star_mags['GAIA-BP'],
star_mags['GAIA-RP'], ebv).items():
star_unextincted_mags['GAIA-' +
band] = star_mags['GAIA-' + band] - extval
star_colors = dict()
star_unextincted_colors = dict()
# compute the colors and define the unextincted colors
# the unextincted colors are filled later
if not gaia_std:
for c1, c2 in ['GR', 'RZ']:
star_colors[c1 + '-' + c2] = star_mags[c1] - star_mags[c2]
star_unextincted_colors[c1 + '-' +
c2] = (star_unextincted_mags[c1] -
star_unextincted_mags[c2])
for c1, c2 in [('BP', 'RP'), ('G', 'RP')]:
star_colors['GAIA-' + c1 + '-' + c2] = (star_mags['GAIA-' + c1] -
star_mags['GAIA-' + c2])
star_unextincted_colors['GAIA-' + c1 + '-' +
c2] = (star_unextincted_mags['GAIA-' + c1] -
star_unextincted_mags['GAIA-' + c2])
linear_coefficients = np.zeros((nstars, stdflux.shape[0]))
chi2dof = np.zeros((nstars))
redshift = np.zeros((nstars))
normflux = np.zeros((nstars, stdwave.size))
fitted_model_colors = np.zeros(nstars)
local_comm, head_comm = None, None
if comm is not None:
# All ranks in local_comm work on the same stars
local_comm = comm.Split(rank % nstars, rank)
# The color 1 in head_comm contains all ranks that are have rank 0 in local_comm
head_comm = comm.Split(rank < nstars, rank)
for star in range(rank % nstars, nstars, size):
log.info("rank %d: finding best model for observed star #%d" %
(rank, star))
# np.array of wave,flux,ivar,resol
wave = {}
flux = {}
ivar = {}
resolution_data = {}
for camera in frames:
for i, frame in enumerate(frames[camera]):
identifier = "%s-%d" % (camera, i)
wave[identifier] = frame.wave
flux[identifier] = frame.flux[star]
ivar[identifier] = frame.ivar[star]
resolution_data[identifier] = frame.resolution_data[star]
# preselect models based on magnitudes
photsys = fibermap['PHOTSYS'][star]
if gaia_std:
model_colors = model_mags[color_band1] - model_mags[color_band2]
else:
model_colors = model_mags[color_band1 +
photsys] - model_mags[color_band2 +
photsys]
color_diff = model_colors - star_unextincted_colors[color][star]
selection = np.abs(color_diff) < args.delta_color
if np.sum(selection) == 0:
log.warning("no model in the selected color range for this star")
continue
# smallest cube in parameter space including this selection (needed for interpolation)
new_selection = (teff >= np.min(teff[selection])) & (teff <= np.max(
teff[selection]))
new_selection &= (logg >= np.min(logg[selection])) & (logg <= np.max(
logg[selection]))
new_selection &= (feh >= np.min(feh[selection])) & (feh <= np.max(
feh[selection]))
selection = np.where(new_selection)[0]
log.info(
"star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d"
% (star, starfibers[star], color,
star_unextincted_colors[color][star], selection.size,
stdflux.shape[0]))
# Match unextincted standard stars to data
match_templates_result = match_templates(
wave,
flux,
ivar,
resolution_data,
stdwave,
stdflux[selection],
teff[selection],
logg[selection],
feh[selection],
ncpu=ncpu,
z_max=args.z_max,
z_res=args.z_res,
template_error=args.template_error,
comm=local_comm)
# Only local rank 0 can perform the remaining work
if local_comm is not None and local_comm.Get_rank() != 0:
continue
coefficients, redshift[star], chi2dof[star] = match_templates_result
linear_coefficients[star, selection] = coefficients
log.info(
'Star Fiber: {}; TEFF: {:.3f}; LOGG: {:.3f}; FEH: {:.3f}; Redshift: {:g}; Chisq/dof: {:.3f}'
.format(starfibers[star], np.inner(teff,
linear_coefficients[star]),
np.inner(logg, linear_coefficients[star]),
np.inner(feh, linear_coefficients[star]), redshift[star],
chi2dof[star]))
# Apply redshift to original spectrum at full resolution
model = np.zeros(stdwave.size)
redshifted_stdwave = stdwave * (1 + redshift[star])
for i, c in enumerate(linear_coefficients[star]):
if c != 0:
model += c * np.interp(stdwave, redshifted_stdwave, stdflux[i])
# Apply dust extinction to the model
log.info("Applying MW dust extinction to star {} with EBV = {}".format(
star, ebv[star]))
model *= dust_transmission(stdwave, ebv[star])
# Compute final color of dust-extincted model
photsys = fibermap['PHOTSYS'][star]
if not gaia_std:
model_mag1, model_mag2 = [
get_magnitude(stdwave, model, model_filters, _ + photsys)
for _ in [color_band1, color_band2]
]
else:
model_mag1, model_mag2 = [
get_magnitude(stdwave, model, model_filters, _)
for _ in [color_band1, color_band2]
]
if color_band1 == ref_mag_name:
model_magr = model_mag1
elif color_band2 == ref_mag_name:
model_magr = model_mag2
else:
# if the reference magnitude is not among colours
# I'm fetching it separately. This will happen when
# colour is BP-RP and ref magnitude is G
if gaia_std:
model_magr = get_magnitude(stdwave, model, model_filters,
ref_mag_name)
else:
model_magr = get_magnitude(stdwave, model, model_filters,
ref_mag_name + photsys)
fitted_model_colors[star] = model_mag1 - model_mag2
#- TODO: move this back into normalize_templates, at the cost of
#- recalculating a model magnitude?
cur_refmag = star_mags[ref_mag_name][star]
# Normalize the best model using reported magnitude
scalefac = 10**((model_magr - cur_refmag) / 2.5)
log.info('scaling {} mag {:.3f} to {:.3f} using scale {}'.format(
ref_mag_name, model_magr, cur_refmag, scalefac))
normflux[star] = model * scalefac
if head_comm is not None and rank < nstars: # head_comm color is 1
linear_coefficients = head_comm.reduce(linear_coefficients,
op=MPI.SUM,
root=0)
redshift = head_comm.reduce(redshift, op=MPI.SUM, root=0)
chi2dof = head_comm.reduce(chi2dof, op=MPI.SUM, root=0)
fitted_model_colors = head_comm.reduce(fitted_model_colors,
op=MPI.SUM,
root=0)
normflux = head_comm.reduce(normflux, op=MPI.SUM, root=0)
# Check at least one star was fit. The check is peformed on rank 0 and
# the result is bcast to other ranks so that all ranks exit together if
# the check fails.
atleastonestarfit = False
if rank == 0:
fitted_stars = np.where(chi2dof != 0)[0]
atleastonestarfit = fitted_stars.size > 0
if comm is not None:
atleastonestarfit = comm.bcast(atleastonestarfit, root=0)
if not atleastonestarfit:
log.error("No star has been fit.")
sys.exit(12)
# Now write the normalized flux for all best models to a file
if rank == 0:
# get the fibermap from any input frame for the standard stars
fibermap = Table(frame.fibermap)
keep = np.isin(fibermap['FIBER'], starfibers[fitted_stars])
fibermap = fibermap[keep]
# drop fibermap columns specific to exposures instead of targets
for col in [
'DELTA_X', 'DELTA_Y', 'EXPTIME', 'NUM_ITER', 'FIBER_RA',
'FIBER_DEC', 'FIBER_X', 'FIBER_Y'
]:
if col in fibermap.colnames:
fibermap.remove_column(col)
data = {}
data['LOGG'] = linear_coefficients[fitted_stars, :].dot(logg)
data['TEFF'] = linear_coefficients[fitted_stars, :].dot(teff)
data['FEH'] = linear_coefficients[fitted_stars, :].dot(feh)
data['CHI2DOF'] = chi2dof[fitted_stars]
data['REDSHIFT'] = redshift[fitted_stars]
data['COEFF'] = linear_coefficients[fitted_stars, :]
data['DATA_%s' % color] = star_colors[color][fitted_stars]
data['MODEL_%s' % color] = fitted_model_colors[fitted_stars]
data['BLUE_SNR'] = snr['b'][fitted_stars]
data['RED_SNR'] = snr['r'][fitted_stars]
data['NIR_SNR'] = snr['z'][fitted_stars]
io.write_stdstar_models(args.outfile, normflux, stdwave,
starfibers[fitted_stars], data, fibermap,
input_frames_table)