def main(args):
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
log = get_logger()
log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
(args.color, args.delta_color))
frames = {}
flats = {}
skies = {}
spectrograph = None
starfibers = None
starindices = None
fibermap = None
# READ DATA
############################################
for filename in args.frames:
log.info("reading %s" % filename)
frame = io.read_frame(filename)
header = fits.getheader(filename, 0)
frame_fibermap = frame.fibermap
frame_starindices = np.where(frame_fibermap["OBJTYPE"] == "STD")[0]
camera = safe_read_key(header, "CAMERA").strip().lower()
if spectrograph is None:
spectrograph = frame.spectrograph
fibermap = frame_fibermap
starindices = frame_starindices
starfibers = fibermap["FIBER"][starindices]
elif spectrograph != frame.spectrograph:
log.error("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
raise ValueError("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
elif starindices.size != frame_starindices.size or np.sum(
starindices != frame_starindices) > 0:
log.error("incompatible fibermap")
raise ValueError("incompatible fibermap")
if frames.has_key(camera):
log.error(
"cannot handle for now several frame of same camera (%s)" %
camera)
raise ValueError(
"cannot handle for now several frame of same camera (%s)" %
camera)
frames[camera] = frame
for filename in args.skymodels:
log.info("reading %s" % filename)
sky = io.read_sky(filename)
header = fits.getheader(filename, 0)
camera = safe_read_key(header, "CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if skies.has_key(camera):
log.error("cannot handle several skymodels of same camera (%s)" %
camera)
raise ValueError(
"cannot handle several skymodels of same camera (%s)" % camera)
skies[camera] = sky
for filename in args.fiberflats:
log.info("reading %s" % filename)
header = fits.getheader(filename, 0)
flat = io.read_fiberflat(filename)
camera = safe_read_key(header, "CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if flats.has_key(camera):
log.error("cannot handle several flats of same camera (%s)" %
camera)
raise ValueError(
"cannot handle several flats of same camera (%s)" % camera)
flats[camera] = flat
if starindices.size == 0:
log.error("no STD star found in fibermap")
raise ValueError("no STD star found in fibermap")
log.info("found %d STD stars" % starindices.size)
imaging_filters = fibermap["FILTER"][starindices]
imaging_mags = fibermap["MAG"][starindices]
log.warning(
"NO MAG ERRORS IN FIBERMAP, I AM IGNORING MEASUREMENT ERRORS !!")
log.warning(
"NO EXTINCTION VALUES IN FIBERMAP, I AM IGNORING THIS FOR NOW !!")
# DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
############################################
for cam in frames:
if not skies.has_key(cam):
log.warning("Missing sky for %s" % cam)
frames.pop(cam)
continue
if not flats.has_key(cam):
log.warning("Missing flat for %s" % cam)
frames.pop(cam)
continue
frames[cam].flux = frames[cam].flux[starindices]
frames[cam].ivar = frames[cam].ivar[starindices]
frames[cam].ivar *= (frames[cam].mask[starindices] == 0)
frames[cam].ivar *= (skies[cam].ivar[starindices] != 0)
frames[cam].ivar *= (skies[cam].mask[starindices] == 0)
frames[cam].ivar *= (flats[cam].ivar[starindices] != 0)
frames[cam].ivar *= (flats[cam].mask[starindices] == 0)
frames[cam].flux *= (frames[cam].ivar > 0) # just for clean plots
for star in range(frames[cam].flux.shape[0]):
ok = np.where((frames[cam].ivar[star] > 0)
& (flats[cam].fiberflat[star] != 0))[0]
if ok.size > 0:
frames[cam].flux[star] = frames[cam].flux[star] / flats[
cam].fiberflat[star] - skies[cam].flux[star]
nstars = starindices.size
starindices = None # we don't need this anymore
# READ MODELS
############################################
log.info("reading star models in %s" % args.starmodels)
stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
args.starmodels)
# COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
############################################
model_filters = []
for tmp in np.unique(imaging_filters):
if len(tmp) > 0: # can be one empty entry
model_filters.append(tmp)
log.info("computing model mags %s" % model_filters)
model_mags = np.zeros((stdflux.shape[0], len(model_filters)))
fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
for index in range(len(model_filters)):
filter_response = load_filter(model_filters[index])
for m in range(stdflux.shape[0]):
model_mags[m, index] = filter_response.get_ab_magnitude(
stdflux[m] * fluxunits, stdwave)
log.info("done computing model mags")
# LOOP ON STARS TO FIND BEST MODEL
############################################
bestModelIndex = np.arange(nstars)
templateID = np.arange(nstars)
chi2dof = np.zeros((nstars))
redshift = np.zeros((nstars))
normflux = []
for star in range(nstars):
log.info("finding best model for observed star #%d" % star)
# np.array of wave,flux,ivar,resol
wave = {}
flux = {}
ivar = {}
resolution_data = {}
for camera in frames:
band = camera[0]
wave[band] = frames[camera].wave
flux[band] = frames[camera].flux[star]
ivar[band] = frames[camera].ivar[star]
resolution_data[band] = frames[camera].resolution_data[star]
# preselec models based on magnitudes
# compute star color
index1, index2 = get_color_filter_indices(imaging_filters[star],
args.color)
if index1 < 0 or index2 < 0:
log.error("cannot compute '%s' color from %s" %
(color_name, filters))
filter1 = imaging_filters[star][index1]
filter2 = imaging_filters[star][index2]
star_color = imaging_mags[star][index1] - imaging_mags[star][index2]
# compute models color
model_index1 = -1
model_index2 = -1
for i, fname in enumerate(model_filters):
if fname == filter1:
model_index1 = i
elif fname == filter2:
model_index2 = i
if model_index1 < 0 or model_index2 < 0:
log.error("cannot compute '%s' model color from %s" %
(color_name, filters))
model_colors = model_mags[:, model_index1] - model_mags[:,
model_index2]
# selection
selection = np.where(
np.abs(model_colors - star_color) < args.delta_color)[0]
log.info(
"star#%d fiber #%d, %s = %s-%s = %f, number of pre-selected models = %d/%d"
% (star, starfibers[star], args.color, filter1, filter2,
star_color, selection.size, stdflux.shape[0]))
index_in_selection, redshift[star], chi2dof[star] = match_templates(
wave,
flux,
ivar,
resolution_data,
stdwave,
stdflux[selection],
teff[selection],
logg[selection],
feh[selection],
ncpu=args.ncpu,
z_max=args.z_max,
z_res=args.z_res)
bestModelIndex[star] = selection[index_in_selection]
log.info(
'Star Fiber: {0}; TemplateID: {1}; Redshift: {2}; Chisq/dof: {3}'.
format(starfibers[star], bestModelIndex[star], redshift[star],
chi2dof[star]))
# Apply redshift to original spectrum at full resolution
tmp = np.interp(stdwave, stdwave / (1 + redshift[star]),
stdflux[bestModelIndex[star]])
# Normalize the best model using reported magnitude
normalizedflux = normalize_templates(stdwave, tmp, imaging_mags[star],
imaging_filters[star])
normflux.append(normalizedflux)
# Now write the normalized flux for all best models to a file
normflux = np.array(normflux)
data = {}
data['BESTMODEL'] = bestModelIndex
data['TEMPLATEID'] = bestModelIndex # IS THAT IT?
data['CHI2DOF'] = chi2dof
data['REDSHIFT'] = redshift
norm_model_file = args.outfile
io.write_stdstar_models(args.outfile, normflux, stdwave, starfibers, data)
def main() :
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--fiberflatexpid', type = int, help = 'fiberflat exposure ID')
parser.add_argument('--fibermap', type = str, help = 'path of fibermap file')
parser.add_argument('--models', type = str, help = 'path of spectro-photometric stellar spectra fits')
parser.add_argument('--spectrograph', type = int, default = 0, help = 'spectrograph number, can go 0-9')
parser.add_argument('--outfile', type = str, help = 'output file for normalized stdstar model flux')
args = parser.parse_args()
log = get_logger()
# Call necessary environment variables. No need if add argument to give full file path.
if 'DESI_SPECTRO_REDUX' not in os.environ:
raise RuntimeError('Set environment DESI_SPECTRO_REDUX. It is needed to read the needed datafiles')
DESI_SPECTRO_REDUX=os.environ['DESI_SPECTRO_REDUX']
PRODNAME=os.environ['PRODNAME']
if 'DESISIM' not in os.environ:
raise RuntimeError('Set environment DESISIM. It will be neede to read the filter transmission files for calibration')
DESISIM=os.environ['DESISIM'] # to read the filter transmission files
if args.fibermap is None or args.models is None or \
args.spectrograph is None or args.outfile is None or \
args.fiberflatexpid is None:
log.critical('Missing a required argument')
parser.print_help()
sys.exit(12)
# read Standard Stars from the fibermap file
# returns the Fiber id, filter names and mags for the standard stars
fiber_tbdata,fiber_header=io.read_fibermap(args.fibermap, header=True)
#- Trim to just fibers on this spectrograph
ii = (500*args.spectrograph <= fiber_tbdata["FIBER"])
ii &= (fiber_tbdata["FIBER"] < 500*(args.spectrograph+1))
fiber_tbdata = fiber_tbdata[ii]
#- Get info for the standard stars
refStarIdx=np.where(fiber_tbdata["OBJTYPE"]=="STD")
refFibers=fiber_tbdata["FIBER"][refStarIdx]
refFilters=fiber_tbdata["FILTER"][refStarIdx]
refMags=fiber_tbdata["MAG"]
fibers={"FIBER":refFibers,"FILTER":refFilters,"MAG":refMags}
NIGHT=fiber_header['NIGHT']
EXPID=fiber_header['EXPID']
filters=fibers["FILTER"]
if 'DESISIM' not in os.environ:
raise RuntimeError('Set environment DESISIM. Can not find filter response files')
basepath=DESISIM+"/data/"
#now load all the skyfiles, framefiles, fiberflatfiles etc
# all three channels files are simultaneously treated for model fitting
skyfile={}
framefile={}
fiberflatfile={}
for i in ["b","r","z"]:
camera = i+str(args.spectrograph)
skyfile[i] = io.findfile('sky', NIGHT, EXPID, camera)
framefile[i] = io.findfile('frame', NIGHT, EXPID, camera)
fiberflatfile[i] = io.findfile('fiberflat', NIGHT, args.fiberflatexpid, camera)
#Read Frames, Flats and Sky files
frameFlux={}
frameIvar={}
frameWave={}
frameResolution={}
framehdr={}
fiberFlat={}
ivarFlat={}
maskFlat={}
meanspecFlat={}
waveFlat={}
headerFlat={}
sky={}
skyivar={}
skymask={}
skywave={}
skyhdr={}
for i in ["b","r","z"]:
#arg=(night,expid,'%s%s'%(i,spectrograph))
#- minimal code change for refactored I/O, while not taking advantage of simplified structure
frame = io.read_frame(framefile[i])
frameFlux[i] = frame.flux
frameIvar[i] = frame.ivar
frameWave[i] = frame.wave
frameResolution[i] = frame.resolution_data
framehdr[i] = frame.header
ff = io.read_fiberflat(fiberflatfile[i])
fiberFlat[i] = ff.fiberflat
ivarFlat[i] = ff.ivar
maskFlat[i] = ff.mask
meanspecFlat[i] = ff.meanspec
waveFlat[i] = ff.wave
headerFlat[i] = ff.header
skymodel = io.read_sky(skyfile[i])
sky[i] = skymodel.flux
skyivar[i] = skymodel.ivar
skymask[i] = skymodel.mask
skywave[i] = skymodel.wave
skyhdr[i] = skymodel.header
# Convolve Sky with Detector Resolution, so as to subtract from data. Convolve for all 500 specs. Subtracting sky this way should be equivalent to sky_subtract
convolvedsky={"b":sky["b"], "r":sky["r"], "z":sky["z"]}
# Read the standard Star data and divide by flat and subtract sky
stars=[]
ivars=[]
for i in fibers["FIBER"]:
#flat and sky should have same wavelength binning as data, otherwise should be rebinned.
stars.append((i,{"b":[frameFlux["b"][i]/fiberFlat["b"][i]-convolvedsky["b"][i],frameWave["b"]],
"r":[frameFlux["r"][i]/fiberFlat["r"][i]-convolvedsky["r"][i],frameWave["r"]],
"z":[frameFlux["z"][i]/fiberFlat["z"][i]-convolvedsky["z"][i],frameWave]},fibers["MAG"][i]))
ivars.append((i,{"b":[frameIvar["b"][i]],"r":[frameIvar["r"][i,:]],"z":[frameIvar["z"][i,:]]}))
stdwave,stdflux,templateid=io.read_stdstar_templates(args.models)
#- Trim standard star wavelengths to just the range we need
minwave = min([min(w) for w in frameWave.values()])
maxwave = max([max(w) for w in frameWave.values()])
ii = (minwave-10 < stdwave) & (stdwave < maxwave+10)
stdwave = stdwave[ii]
stdflux = stdflux[:, ii]
log.info('Number of Standard Stars in this frame: {0:d}'.format(len(stars)))
if len(stars) == 0:
log.critical("No standard stars! Exiting")
sys.exit(1)
# Now for each star, find the best model and normalize.
normflux=[]
bestModelIndex=np.arange(len(stars))
templateID=np.arange(len(stars))
chi2dof=np.zeros(len(stars))
#- TODO: don't use 'l' as a variable name. Can look like a '1'
for k,l in enumerate(stars):
log.info("checking best model for star {0}".format(l[0]))
starindex=l[0]
mags=l[2]
filters=fibers["FILTER"][k]
rflux=stars[k][1]["r"][0]
bflux=stars[k][1]["b"][0]
zflux=stars[k][1]["z"][0]
flux={"b":bflux,"r":rflux,"z":zflux}
#print ivars
rivar=ivars[k][1]["r"][0]
bivar=ivars[k][1]["b"][0]
zivar=ivars[k][1]["z"][0]
ivar={"b":bivar,"r":rivar,"z":zivar}
resol_star={"r":frameResolution["r"][l[0]],"b":frameResolution["b"][l[0]],"z":frameResolution["z"][l[0]]}
# Now find the best Model
bestModelIndex[k],bestmodelWave,bestModelFlux,chi2dof[k]=match_templates(frameWave,flux,ivar,resol_star,stdwave,stdflux)
log.info('Star Fiber: {0}; Best Model Fiber: {1}; TemplateID: {2}; Chisq/dof: {3}'.format(l[0],bestModelIndex[k],templateid[bestModelIndex[k]],chi2dof[k]))
# Normalize the best model using reported magnitude
modelwave,normalizedflux=normalize_templates(stdwave,stdflux[bestModelIndex[k]],mags,filters,basepath)
normflux.append(normalizedflux)
# Now write the normalized flux for all best models to a file
normflux=np.array(normflux)
stdfibers=fibers["FIBER"]
data={}
data['BESTMODEL']=bestModelIndex
data['CHI2DOF']=chi2dof
data['TEMPLATEID']=templateid[bestModelIndex]
norm_model_file=args.outfile
io.write_stdstar_model(norm_model_file,normflux,stdwave,stdfibers,data)
def main(args) :
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
log = get_logger()
log.info("mag delta %s = %f (for the pre-selection of stellar models)"%(args.color,args.delta_color))
frames={}
flats={}
skies={}
spectrograph=None
starfibers=None
starindices=None
fibermap=None
# READ DATA
############################################
for filename in args.frames :
log.info("reading %s"%filename)
frame=io.read_frame(filename)
header=fits.getheader(filename, 0)
frame_fibermap = frame.fibermap
frame_starindices = np.where(isStdStar(frame_fibermap['DESI_TARGET']))[0]
#- Confirm that all fluxes have entries but trust targeting bits
#- to get basic magnitude range correct
keep = np.ones(len(frame_starindices), dtype=bool)
for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']: #- and W1 and W2?
keep &= frame_fibermap[colname][frame_starindices] > 10**((22.5-30)/2.5)
keep &= frame_fibermap[colname][frame_starindices] < 10**((22.5-0)/2.5)
frame_starindices = frame_starindices[keep]
camera=safe_read_key(header,"CAMERA").strip().lower()
if spectrograph is None :
spectrograph = frame.spectrograph
fibermap = frame_fibermap
starindices=frame_starindices
starfibers=fibermap["FIBER"][starindices]
elif spectrograph != frame.spectrograph :
log.error("incompatible spectrographs %d != %d"%(spectrograph,frame.spectrograph))
raise ValueError("incompatible spectrographs %d != %d"%(spectrograph,frame.spectrograph))
elif starindices.size != frame_starindices.size or np.sum(starindices!=frame_starindices)>0 :
log.error("incompatible fibermap")
raise ValueError("incompatible fibermap")
if not camera in frames :
frames[camera]=[]
frames[camera].append(frame)
for filename in args.skymodels :
log.info("reading %s"%filename)
sky=io.read_sky(filename)
header=fits.getheader(filename, 0)
camera=safe_read_key(header,"CAMERA").strip().lower()
if not camera in skies :
skies[camera]=[]
skies[camera].append(sky)
for filename in args.fiberflats :
log.info("reading %s"%filename)
header=fits.getheader(filename, 0)
flat=io.read_fiberflat(filename)
camera=safe_read_key(header,"CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if camera in flats:
log.warning("cannot handle several flats of same camera (%s), will use only the first one"%camera)
#raise ValueError("cannot handle several flats of same camera (%s)"%camera)
else :
flats[camera]=flat
if starindices.size == 0 :
log.error("no STD star found in fibermap")
raise ValueError("no STD star found in fibermap")
log.info("found %d STD stars"%starindices.size)
log.warning("Not using flux errors for Standard Star fits!")
# DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
############################################
for cam in frames :
if not cam in skies:
log.warning("Missing sky for %s"%cam)
frames.pop(cam)
continue
if not cam in flats:
log.warning("Missing flat for %s"%cam)
frames.pop(cam)
continue
flat=flats[cam]
for frame,sky in zip(frames[cam],skies[cam]) :
frame.flux = frame.flux[starindices]
frame.ivar = frame.ivar[starindices]
frame.ivar *= (frame.mask[starindices] == 0)
frame.ivar *= (sky.ivar[starindices] != 0)
frame.ivar *= (sky.mask[starindices] == 0)
frame.ivar *= (flat.ivar[starindices] != 0)
frame.ivar *= (flat.mask[starindices] == 0)
frame.flux *= ( frame.ivar > 0) # just for clean plots
for star in range(frame.flux.shape[0]) :
ok=np.where((frame.ivar[star]>0)&(flat.fiberflat[star]!=0))[0]
if ok.size > 0 :
frame.flux[star] = frame.flux[star]/flat.fiberflat[star] - sky.flux[star]
frame.resolution_data = frame.resolution_data[starindices]
nstars = starindices.size
fibermap = Table(fibermap[starindices])
# READ MODELS
############################################
log.info("reading star models in %s"%args.starmodels)
stdwave,stdflux,templateid,teff,logg,feh=io.read_stdstar_templates(args.starmodels)
# COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
############################################
#- Support older fibermaps
if 'PHOTSYS' not in fibermap.colnames:
log.warning('Old fibermap format; using defaults for missing columns')
log.warning(" PHOTSYS = 'S'")
log.warning(" MW_TRANSMISSION_G/R/Z = 1.0")
log.warning(" EBV = 0.0")
fibermap['PHOTSYS'] = 'S'
fibermap['MW_TRANSMISSION_G'] = 1.0
fibermap['MW_TRANSMISSION_R'] = 1.0
fibermap['MW_TRANSMISSION_Z'] = 1.0
fibermap['EBV'] = 0.0
model_filters = dict()
if 'S' in fibermap['PHOTSYS']:
for filter_name in ['DECAM_G', 'DECAM_R', 'DECAM_Z']:
model_filters[filter_name] = load_filter(filter_name)
if 'N' in fibermap['PHOTSYS']:
for filter_name in ['BASS_G', 'BASS_R', 'MZLS_Z']:
model_filters[filter_name] = load_filter(filter_name)
if len(model_filters) == 0:
raise ValueError("No filters loaded; neither 'N' nor 'S' in PHOTSYS?")
log.info("computing model mags for %s"%sorted(model_filters.keys()))
model_mags = dict()
fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
for filter_name, filter_response in model_filters.items():
model_mags[filter_name] = filter_response.get_ab_magnitude(stdflux*fluxunits,stdwave)
log.info("done computing model mags")
# LOOP ON STARS TO FIND BEST MODEL
############################################
linear_coefficients=np.zeros((nstars,stdflux.shape[0]))
chi2dof=np.zeros((nstars))
redshift=np.zeros((nstars))
normflux=[]
star_mags = dict()
star_unextincted_mags = dict()
for band in ['G', 'R', 'Z']:
star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_'+band])
star_unextincted_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_'+band] / fibermap['MW_TRANSMISSION_'+band])
star_colors = dict()
star_colors['G-R'] = star_mags['G'] - star_mags['R']
star_colors['R-Z'] = star_mags['R'] - star_mags['Z']
star_unextincted_colors = dict()
star_unextincted_colors['G-R'] = star_unextincted_mags['G'] - star_unextincted_mags['R']
star_unextincted_colors['R-Z'] = star_unextincted_mags['R'] - star_unextincted_mags['Z']
fitted_model_colors = np.zeros(nstars)
for star in range(nstars) :
log.info("finding best model for observed star #%d"%star)
# np.array of wave,flux,ivar,resol
wave = {}
flux = {}
ivar = {}
resolution_data = {}
for camera in frames :
for i,frame in enumerate(frames[camera]) :
identifier="%s-%d"%(camera,i)
wave[identifier]=frame.wave
flux[identifier]=frame.flux[star]
ivar[identifier]=frame.ivar[star]
resolution_data[identifier]=frame.resolution_data[star]
# preselect models based on magnitudes
if fibermap['PHOTSYS'][star] == 'N':
if args.color == 'G-R':
model_colors = model_mags['BASS_G'] - model_mags['BASS_R']
elif args.color == 'R-Z':
model_colors = model_mags['BASS_R'] - model_mags['MZLS_Z']
else:
raise ValueError('Unknown color {}'.format(args.color))
else:
if args.color == 'G-R':
model_colors = model_mags['DECAM_G'] - model_mags['DECAM_R']
elif args.color == 'R-Z':
model_colors = model_mags['DECAM_R'] - model_mags['DECAM_Z']
else:
raise ValueError('Unknown color {}'.format(args.color))
color_diff = model_colors - star_unextincted_colors[args.color][star]
selection = np.abs(color_diff) < args.delta_color
# smallest cube in parameter space including this selection (needed for interpolation)
new_selection = (teff>=np.min(teff[selection]))&(teff<=np.max(teff[selection]))
new_selection &= (logg>=np.min(logg[selection]))&(logg<=np.max(logg[selection]))
new_selection &= (feh>=np.min(feh[selection]))&(feh<=np.max(feh[selection]))
selection = np.where(new_selection)[0]
log.info("star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d"%(
star, starfibers[star], args.color, star_unextincted_colors[args.color][star],
selection.size, stdflux.shape[0]))
# Match unextincted standard stars to data
coefficients, redshift[star], chi2dof[star] = match_templates(
wave, flux, ivar, resolution_data,
stdwave, stdflux[selection],
teff[selection], logg[selection], feh[selection],
ncpu=args.ncpu, z_max=args.z_max, z_res=args.z_res,
template_error=args.template_error
)
linear_coefficients[star,selection] = coefficients
log.info('Star Fiber: {0}; TEFF: {1}; LOGG: {2}; FEH: {3}; Redshift: {4}; Chisq/dof: {5}'.format(
starfibers[star],
np.inner(teff,linear_coefficients[star]),
np.inner(logg,linear_coefficients[star]),
np.inner(feh,linear_coefficients[star]),
redshift[star],
chi2dof[star])
)
# Apply redshift to original spectrum at full resolution
model=np.zeros(stdwave.size)
redshifted_stdwave = stdwave*(1+redshift[star])
for i,c in enumerate(linear_coefficients[star]) :
if c != 0 :
model += c*np.interp(stdwave,redshifted_stdwave,stdflux[i])
# Apply dust extinction to the model
model *= dust_transmission(stdwave, fibermap['EBV'][star])
# Compute final color of dust-extincted model
if fibermap['PHOTSYS'][star] == 'N':
if args.color == 'G-R':
model_mag1 = model_filters['BASS_G'].get_ab_magnitude(model*fluxunits, stdwave)
model_mag2 = model_filters['BASS_R'].get_ab_magnitude(model*fluxunits, stdwave)
model_magr = model_mag2
elif args.color == 'R-Z':
model_mag1 = model_filters['BASS_R'].get_ab_magnitude(model*fluxunits, stdwave)
model_mag2 = model_filters['MZLS_Z'].get_ab_magnitude(model*fluxunits, stdwave)
model_magr = model_mag1
else:
raise ValueError('Unknown color {}'.format(args.color))
else:
if args.color == 'G-R':
model_mag1 = model_filters['DECAM_G'].get_ab_magnitude(model*fluxunits, stdwave)
model_mag2 = model_filters['DECAM_R'].get_ab_magnitude(model*fluxunits, stdwave)
model_magr = model_mag2
elif args.color == 'R-Z':
model_mag1 = model_filters['DECAM_R'].get_ab_magnitude(model*fluxunits, stdwave)
model_mag2 = model_filters['DECAM_Z'].get_ab_magnitude(model*fluxunits, stdwave)
model_magr = model_mag1
else:
raise ValueError('Unknown color {}'.format(args.color))
fitted_model_colors[star] = model_mag1 - model_mag2
#- TODO: move this back into normalize_templates, at the cost of
#- recalculating a model magnitude?
# Normalize the best model using reported magnitude
scalefac=10**((model_magr - star_mags['R'][star])/2.5)
log.info('scaling R mag {} to {} using scale {}'.format(model_magr, star_mags['R'][star], scalefac))
normflux.append(model*scalefac)
# Now write the normalized flux for all best models to a file
normflux=np.array(normflux)
data={}
data['LOGG']=linear_coefficients.dot(logg)
data['TEFF']= linear_coefficients.dot(teff)
data['FEH']= linear_coefficients.dot(feh)
data['CHI2DOF']=chi2dof
data['REDSHIFT']=redshift
data['COEFF']=linear_coefficients
data['DATA_%s'%args.color]=star_colors[args.color]
data['MODEL_%s'%args.color]=fitted_model_colors
io.write_stdstar_models(args.outfile,normflux,stdwave,starfibers,data)
def main(args):
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
log = get_logger()
log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
(args.color, args.delta_color))
frames = {}
flats = {}
skies = {}
spectrograph = None
starfibers = None
starindices = None
fibermap = None
# READ DATA
############################################
for filename in args.frames:
log.info("reading %s" % filename)
frame = io.read_frame(filename)
header = fits.getheader(filename, 0)
frame_fibermap = frame.fibermap
frame_starindices = np.where(isStdStar(frame_fibermap))[0]
#- Confirm that all fluxes have entries but trust targeting bits
#- to get basic magnitude range correct
keep = np.ones(len(frame_starindices), dtype=bool)
for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']: #- and W1 and W2?
keep &= frame_fibermap[colname][frame_starindices] > 10**(
(22.5 - 30) / 2.5)
keep &= frame_fibermap[colname][frame_starindices] < 10**(
(22.5 - 0) / 2.5)
frame_starindices = frame_starindices[keep]
camera = safe_read_key(header, "CAMERA").strip().lower()
if spectrograph is None:
spectrograph = frame.spectrograph
fibermap = frame_fibermap
starindices = frame_starindices
starfibers = fibermap["FIBER"][starindices]
elif spectrograph != frame.spectrograph:
log.error("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
raise ValueError("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
elif starindices.size != frame_starindices.size or np.sum(
starindices != frame_starindices) > 0:
log.error("incompatible fibermap")
raise ValueError("incompatible fibermap")
if not camera in frames:
frames[camera] = []
frames[camera].append(frame)
for filename in args.skymodels:
log.info("reading %s" % filename)
sky = io.read_sky(filename)
header = fits.getheader(filename, 0)
camera = safe_read_key(header, "CAMERA").strip().lower()
if not camera in skies:
skies[camera] = []
skies[camera].append(sky)
for filename in args.fiberflats:
log.info("reading %s" % filename)
header = fits.getheader(filename, 0)
flat = io.read_fiberflat(filename)
camera = safe_read_key(header, "CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if camera in flats:
log.warning(
"cannot handle several flats of same camera (%s), will use only the first one"
% camera)
#raise ValueError("cannot handle several flats of same camera (%s)"%camera)
else:
flats[camera] = flat
if starindices.size == 0:
log.error("no STD star found in fibermap")
raise ValueError("no STD star found in fibermap")
log.info("found %d STD stars" % starindices.size)
log.warning("Not using flux errors for Standard Star fits!")
# DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
############################################
for cam in frames:
if not cam in skies:
log.warning("Missing sky for %s" % cam)
frames.pop(cam)
continue
if not cam in flats:
log.warning("Missing flat for %s" % cam)
frames.pop(cam)
continue
flat = flats[cam]
for frame, sky in zip(frames[cam], skies[cam]):
frame.flux = frame.flux[starindices]
frame.ivar = frame.ivar[starindices]
frame.ivar *= (frame.mask[starindices] == 0)
frame.ivar *= (sky.ivar[starindices] != 0)
frame.ivar *= (sky.mask[starindices] == 0)
frame.ivar *= (flat.ivar[starindices] != 0)
frame.ivar *= (flat.mask[starindices] == 0)
frame.flux *= (frame.ivar > 0) # just for clean plots
for star in range(frame.flux.shape[0]):
ok = np.where((frame.ivar[star] > 0)
& (flat.fiberflat[star] != 0))[0]
if ok.size > 0:
frame.flux[star] = frame.flux[star] / flat.fiberflat[
star] - sky.flux[star]
frame.resolution_data = frame.resolution_data[starindices]
nstars = starindices.size
fibermap = Table(fibermap[starindices])
# READ MODELS
############################################
log.info("reading star models in %s" % args.starmodels)
stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
args.starmodels)
# COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
############################################
#- Support older fibermaps
if 'PHOTSYS' not in fibermap.colnames:
log.warning('Old fibermap format; using defaults for missing columns')
log.warning(" PHOTSYS = 'S'")
log.warning(" MW_TRANSMISSION_G/R/Z = 1.0")
log.warning(" EBV = 0.0")
fibermap['PHOTSYS'] = 'S'
fibermap['MW_TRANSMISSION_G'] = 1.0
fibermap['MW_TRANSMISSION_R'] = 1.0
fibermap['MW_TRANSMISSION_Z'] = 1.0
fibermap['EBV'] = 0.0
model_filters = dict()
if 'S' in fibermap['PHOTSYS']:
for filter_name in ['DECAM_G', 'DECAM_R', 'DECAM_Z']:
model_filters[filter_name] = load_filter(filter_name)
if 'N' in fibermap['PHOTSYS']:
for filter_name in ['BASS_G', 'BASS_R', 'MZLS_Z']:
model_filters[filter_name] = load_filter(filter_name)
if len(model_filters) == 0:
raise ValueError("No filters loaded; neither 'N' nor 'S' in PHOTSYS?")
log.info("computing model mags for %s" % sorted(model_filters.keys()))
model_mags = dict()
fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
for filter_name, filter_response in model_filters.items():
model_mags[filter_name] = filter_response.get_ab_magnitude(
stdflux * fluxunits, stdwave)
log.info("done computing model mags")
# LOOP ON STARS TO FIND BEST MODEL
############################################
linear_coefficients = np.zeros((nstars, stdflux.shape[0]))
chi2dof = np.zeros((nstars))
redshift = np.zeros((nstars))
normflux = []
star_mags = dict()
star_unextincted_mags = dict()
for band in ['G', 'R', 'Z']:
star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_' + band])
star_unextincted_mags[band] = 22.5 - 2.5 * np.log10(
fibermap['FLUX_' + band] / fibermap['MW_TRANSMISSION_' + band])
star_colors = dict()
star_colors['G-R'] = star_mags['G'] - star_mags['R']
star_colors['R-Z'] = star_mags['R'] - star_mags['Z']
star_unextincted_colors = dict()
star_unextincted_colors[
'G-R'] = star_unextincted_mags['G'] - star_unextincted_mags['R']
star_unextincted_colors[
'R-Z'] = star_unextincted_mags['R'] - star_unextincted_mags['Z']
fitted_model_colors = np.zeros(nstars)
for star in range(nstars):
log.info("finding best model for observed star #%d" % star)
# np.array of wave,flux,ivar,resol
wave = {}
flux = {}
ivar = {}
resolution_data = {}
for camera in frames:
for i, frame in enumerate(frames[camera]):
identifier = "%s-%d" % (camera, i)
wave[identifier] = frame.wave
flux[identifier] = frame.flux[star]
ivar[identifier] = frame.ivar[star]
resolution_data[identifier] = frame.resolution_data[star]
# preselect models based on magnitudes
if fibermap['PHOTSYS'][star] == 'N':
if args.color == 'G-R':
model_colors = model_mags['BASS_G'] - model_mags['BASS_R']
elif args.color == 'R-Z':
model_colors = model_mags['BASS_R'] - model_mags['MZLS_Z']
else:
raise ValueError('Unknown color {}'.format(args.color))
else:
if args.color == 'G-R':
model_colors = model_mags['DECAM_G'] - model_mags['DECAM_R']
elif args.color == 'R-Z':
model_colors = model_mags['DECAM_R'] - model_mags['DECAM_Z']
else:
raise ValueError('Unknown color {}'.format(args.color))
color_diff = model_colors - star_unextincted_colors[args.color][star]
selection = np.abs(color_diff) < args.delta_color
# smallest cube in parameter space including this selection (needed for interpolation)
new_selection = (teff >= np.min(teff[selection])) & (teff <= np.max(
teff[selection]))
new_selection &= (logg >= np.min(logg[selection])) & (logg <= np.max(
logg[selection]))
new_selection &= (feh >= np.min(feh[selection])) & (feh <= np.max(
feh[selection]))
selection = np.where(new_selection)[0]
log.info(
"star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d"
% (star, starfibers[star], args.color,
star_unextincted_colors[args.color][star], selection.size,
stdflux.shape[0]))
# Match unextincted standard stars to data
coefficients, redshift[star], chi2dof[star] = match_templates(
wave,
flux,
ivar,
resolution_data,
stdwave,
stdflux[selection],
teff[selection],
logg[selection],
feh[selection],
ncpu=args.ncpu,
z_max=args.z_max,
z_res=args.z_res,
template_error=args.template_error)
linear_coefficients[star, selection] = coefficients
log.info(
'Star Fiber: {0}; TEFF: {1}; LOGG: {2}; FEH: {3}; Redshift: {4}; Chisq/dof: {5}'
.format(starfibers[star], np.inner(teff,
linear_coefficients[star]),
np.inner(logg, linear_coefficients[star]),
np.inner(feh, linear_coefficients[star]), redshift[star],
chi2dof[star]))
# Apply redshift to original spectrum at full resolution
model = np.zeros(stdwave.size)
redshifted_stdwave = stdwave * (1 + redshift[star])
for i, c in enumerate(linear_coefficients[star]):
if c != 0:
model += c * np.interp(stdwave, redshifted_stdwave, stdflux[i])
# Apply dust extinction to the model
model *= dust_transmission(stdwave, fibermap['EBV'][star])
# Compute final color of dust-extincted model
if fibermap['PHOTSYS'][star] == 'N':
if args.color == 'G-R':
model_mag1 = model_filters['BASS_G'].get_ab_magnitude(
model * fluxunits, stdwave)
model_mag2 = model_filters['BASS_R'].get_ab_magnitude(
model * fluxunits, stdwave)
model_magr = model_mag2
elif args.color == 'R-Z':
model_mag1 = model_filters['BASS_R'].get_ab_magnitude(
model * fluxunits, stdwave)
model_mag2 = model_filters['MZLS_Z'].get_ab_magnitude(
model * fluxunits, stdwave)
model_magr = model_mag1
else:
raise ValueError('Unknown color {}'.format(args.color))
else:
if args.color == 'G-R':
model_mag1 = model_filters['DECAM_G'].get_ab_magnitude(
model * fluxunits, stdwave)
model_mag2 = model_filters['DECAM_R'].get_ab_magnitude(
model * fluxunits, stdwave)
model_magr = model_mag2
elif args.color == 'R-Z':
model_mag1 = model_filters['DECAM_R'].get_ab_magnitude(
model * fluxunits, stdwave)
model_mag2 = model_filters['DECAM_Z'].get_ab_magnitude(
model * fluxunits, stdwave)
model_magr = model_mag1
else:
raise ValueError('Unknown color {}'.format(args.color))
fitted_model_colors[star] = model_mag1 - model_mag2
#- TODO: move this back into normalize_templates, at the cost of
#- recalculating a model magnitude?
# Normalize the best model using reported magnitude
scalefac = 10**((model_magr - star_mags['R'][star]) / 2.5)
log.info('scaling R mag {} to {} using scale {}'.format(
model_magr, star_mags['R'][star], scalefac))
normflux.append(model * scalefac)
# Now write the normalized flux for all best models to a file
normflux = np.array(normflux)
data = {}
data['LOGG'] = linear_coefficients.dot(logg)
data['TEFF'] = linear_coefficients.dot(teff)
data['FEH'] = linear_coefficients.dot(feh)
data['CHI2DOF'] = chi2dof
data['REDSHIFT'] = redshift
data['COEFF'] = linear_coefficients
data['DATA_%s' % args.color] = star_colors[args.color]
data['MODEL_%s' % args.color] = fitted_model_colors
io.write_stdstar_models(args.outfile, normflux, stdwave, starfibers, data)
def main(args):
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
log = get_logger()
log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
(args.color, args.delta_color))
frames = {}
flats = {}
skies = {}
spectrograph = None
starfibers = None
starindices = None
fibermap = None
# READ DATA
############################################
for filename in args.frames:
log.info("reading %s" % filename)
frame = io.read_frame(filename)
header = fits.getheader(filename, 0)
frame_fibermap = frame.fibermap
frame_starindices = np.where(frame_fibermap["OBJTYPE"] == "STD")[0]
# check magnitude are well defined or discard stars
tmp = []
for i in frame_starindices:
mags = frame_fibermap["MAG"][i]
ok = np.sum((mags > 0) & (mags < 30))
if np.sum((mags > 0) & (mags < 30)) == mags.size:
tmp.append(i)
frame_starindices = np.array(tmp).astype(int)
camera = safe_read_key(header, "CAMERA").strip().lower()
if spectrograph is None:
spectrograph = frame.spectrograph
fibermap = frame_fibermap
starindices = frame_starindices
starfibers = fibermap["FIBER"][starindices]
elif spectrograph != frame.spectrograph:
log.error("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
raise ValueError("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
elif starindices.size != frame_starindices.size or np.sum(
starindices != frame_starindices) > 0:
log.error("incompatible fibermap")
raise ValueError("incompatible fibermap")
if not camera in frames:
frames[camera] = []
frames[camera].append(frame)
for filename in args.skymodels:
log.info("reading %s" % filename)
sky = io.read_sky(filename)
header = fits.getheader(filename, 0)
camera = safe_read_key(header, "CAMERA").strip().lower()
if not camera in skies:
skies[camera] = []
skies[camera].append(sky)
for filename in args.fiberflats:
log.info("reading %s" % filename)
header = fits.getheader(filename, 0)
flat = io.read_fiberflat(filename)
camera = safe_read_key(header, "CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if camera in flats:
log.warning(
"cannot handle several flats of same camera (%s), will use only the first one"
% camera)
#raise ValueError("cannot handle several flats of same camera (%s)"%camera)
else:
flats[camera] = flat
if starindices.size == 0:
log.error("no STD star found in fibermap")
raise ValueError("no STD star found in fibermap")
log.info("found %d STD stars" % starindices.size)
imaging_filters = fibermap["FILTER"][starindices]
imaging_mags = fibermap["MAG"][starindices]
log.warning(
"NO MAG ERRORS IN FIBERMAP, I AM IGNORING MEASUREMENT ERRORS !!")
ebv = np.zeros(starindices.size)
if "SFD_EBV" in fibermap.columns.names:
log.info("Using 'SFD_EBV' from fibermap")
ebv = fibermap["SFD_EBV"][starindices]
else:
log.warning("NO EXTINCTION VALUES IN FIBERMAP!!")
# DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
############################################
for cam in frames:
if not cam in skies:
log.warning("Missing sky for %s" % cam)
frames.pop(cam)
continue
if not cam in flats:
log.warning("Missing flat for %s" % cam)
frames.pop(cam)
continue
flat = flats[cam]
for frame, sky in zip(frames[cam], skies[cam]):
frame.flux = frame.flux[starindices]
frame.ivar = frame.ivar[starindices]
frame.ivar *= (frame.mask[starindices] == 0)
frame.ivar *= (sky.ivar[starindices] != 0)
frame.ivar *= (sky.mask[starindices] == 0)
frame.ivar *= (flat.ivar[starindices] != 0)
frame.ivar *= (flat.mask[starindices] == 0)
frame.flux *= (frame.ivar > 0) # just for clean plots
for star in range(frame.flux.shape[0]):
ok = np.where((frame.ivar[star] > 0)
& (flat.fiberflat[star] != 0))[0]
if ok.size > 0:
frame.flux[star] = frame.flux[star] / flat.fiberflat[
star] - sky.flux[star]
frame.resolution_data = frame.resolution_data[starindices]
nstars = starindices.size
starindices = None # we don't need this anymore
# READ MODELS
############################################
log.info("reading star models in %s" % args.starmodels)
stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
args.starmodels)
# COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
############################################
model_filters = []
for tmp in np.unique(imaging_filters):
if len(tmp) > 0: # can be one empty entry
model_filters.append(tmp)
log.info("computing model mags %s" % model_filters)
model_mags = np.zeros((stdflux.shape[0], len(model_filters)))
fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
for index in range(len(model_filters)):
if model_filters[index].startswith('WISE'):
log.warning('not computing stdstar {} mags'.format(
model_filters[index]))
continue
filter_response = load_filter(model_filters[index])
for m in range(stdflux.shape[0]):
model_mags[m, index] = filter_response.get_ab_magnitude(
stdflux[m] * fluxunits, stdwave)
log.info("done computing model mags")
mean_extinction_delta_mags = None
mean_ebv = np.mean(ebv)
if mean_ebv > 0:
log.info(
"Compute a mean delta_color from average E(B-V) = %3.2f based on canonial model star"
% mean_ebv)
# compute a mean delta_color from mean_ebv based on canonial model star
#######################################################################
# will then use this color offset in the model pre-selection
# find canonical f-type model: Teff=6000, logg=4, Fe/H=-1.5
canonical_model = np.argmin((teff - 6000.0)**2 + (logg - 4.0)**2 +
(feh + 1.5)**2)
canonical_model_mags_without_extinction = model_mags[canonical_model]
canonical_model_mags_with_extinction = np.zeros(
canonical_model_mags_without_extinction.shape)
canonical_model_reddened_flux = stdflux[
canonical_model] * dust_transmission(stdwave, mean_ebv)
for index in range(len(model_filters)):
if model_filters[index].startswith('WISE'):
log.warning('not computing stdstar {} mags'.format(
model_filters[index]))
continue
filter_response = load_filter(model_filters[index])
canonical_model_mags_with_extinction[
index] = filter_response.get_ab_magnitude(
canonical_model_reddened_flux * fluxunits, stdwave)
mean_extinction_delta_mags = canonical_model_mags_with_extinction - canonical_model_mags_without_extinction
# LOOP ON STARS TO FIND BEST MODEL
############################################
linear_coefficients = np.zeros((nstars, stdflux.shape[0]))
chi2dof = np.zeros((nstars))
redshift = np.zeros((nstars))
normflux = []
star_colors_array = np.zeros((nstars))
model_colors_array = np.zeros((nstars))
for star in range(nstars):
log.info("finding best model for observed star #%d" % star)
# np.array of wave,flux,ivar,resol
wave = {}
flux = {}
ivar = {}
resolution_data = {}
for camera in frames:
for i, frame in enumerate(frames[camera]):
identifier = "%s-%d" % (camera, i)
wave[identifier] = frame.wave
flux[identifier] = frame.flux[star]
ivar[identifier] = frame.ivar[star]
resolution_data[identifier] = frame.resolution_data[star]
# preselec models based on magnitudes
# compute star color
index1, index2 = get_color_filter_indices(imaging_filters[star],
args.color)
if index1 < 0 or index2 < 0:
log.error("cannot compute '%s' color from %s" %
(color_name, filters))
filter1 = imaging_filters[star][index1]
filter2 = imaging_filters[star][index2]
star_color = imaging_mags[star][index1] - imaging_mags[star][index2]
star_colors_array[star] = star_color
# compute models color
model_index1 = -1
model_index2 = -1
for i, fname in enumerate(model_filters):
if fname == filter1:
model_index1 = i
elif fname == filter2:
model_index2 = i
if model_index1 < 0 or model_index2 < 0:
log.error("cannot compute '%s' model color from %s" %
(color_name, filters))
model_colors = model_mags[:, model_index1] - model_mags[:,
model_index2]
# apply extinction here
# use the colors derived from the cannonical model with the mean ebv of the stars
# and simply apply a scaling factor based on the ebv of this star
# this is sufficiently precise for the broad model pre-selection we are doing here
# the exact reddening of the star to each pre-selected model is
# apply afterwards
if mean_extinction_delta_mags is not None and mean_ebv != 0:
delta_color = (mean_extinction_delta_mags[model_index1] -
mean_extinction_delta_mags[model_index2]
) * ebv[star] / mean_ebv
model_colors += delta_color
log.info(
"Apply a %s-%s color offset = %4.3f to the models for star with E(B-V)=%4.3f"
% (model_filters[model_index1], model_filters[model_index2],
delta_color, ebv[star]))
# selection
selection = np.abs(model_colors - star_color) < args.delta_color
# smallest cube in parameter space including this selection (needed for interpolation)
new_selection = (teff >= np.min(teff[selection])) & (teff <= np.max(
teff[selection]))
new_selection &= (logg >= np.min(logg[selection])) & (logg <= np.max(
logg[selection]))
new_selection &= (feh >= np.min(feh[selection])) & (feh <= np.max(
feh[selection]))
selection = np.where(new_selection)[0]
log.info(
"star#%d fiber #%d, %s = %s-%s = %f, number of pre-selected models = %d/%d"
% (star, starfibers[star], args.color, filter1, filter2,
star_color, selection.size, stdflux.shape[0]))
# apply extinction to selected_models
dust_transmission_of_this_star = dust_transmission(stdwave, ebv[star])
selected_reddened_stdflux = stdflux[
selection] * dust_transmission_of_this_star
coefficients, redshift[star], chi2dof[star] = match_templates(
wave,
flux,
ivar,
resolution_data,
stdwave,
selected_reddened_stdflux,
teff[selection],
logg[selection],
feh[selection],
ncpu=args.ncpu,
z_max=args.z_max,
z_res=args.z_res,
template_error=args.template_error)
linear_coefficients[star, selection] = coefficients
log.info(
'Star Fiber: {0}; TEFF: {1}; LOGG: {2}; FEH: {3}; Redshift: {4}; Chisq/dof: {5}'
.format(starfibers[star], np.inner(teff,
linear_coefficients[star]),
np.inner(logg, linear_coefficients[star]),
np.inner(feh, linear_coefficients[star]), redshift[star],
chi2dof[star]))
# Apply redshift to original spectrum at full resolution
model = np.zeros(stdwave.size)
for i, c in enumerate(linear_coefficients[star]):
if c != 0:
model += c * np.interp(stdwave, stdwave *
(1 + redshift[star]), stdflux[i])
# Apply dust extinction
model *= dust_transmission_of_this_star
# Compute final model color
mag1 = load_filter(model_filters[model_index1]).get_ab_magnitude(
model * fluxunits, stdwave)
mag2 = load_filter(model_filters[model_index2]).get_ab_magnitude(
model * fluxunits, stdwave)
model_colors_array[star] = mag1 - mag2
# Normalize the best model using reported magnitude
normalizedflux = normalize_templates(stdwave, model,
imaging_mags[star],
imaging_filters[star])
normflux.append(normalizedflux)
# Now write the normalized flux for all best models to a file
normflux = np.array(normflux)
data = {}
data['LOGG'] = linear_coefficients.dot(logg)
data['TEFF'] = linear_coefficients.dot(teff)
data['FEH'] = linear_coefficients.dot(feh)
data['CHI2DOF'] = chi2dof
data['REDSHIFT'] = redshift
data['COEFF'] = linear_coefficients
data['DATA_%s' % args.color] = star_colors_array
data['MODEL_%s' % args.color] = model_colors_array
norm_model_file = args.outfile
io.write_stdstar_models(args.outfile, normflux, stdwave, starfibers, data)
def main(args):
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
log = get_logger()
log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
(args.color, args.delta_color))
log.info('multiprocess parallelizing with {} processes'.format(args.ncpu))
# READ DATA
############################################
# First loop through and group by exposure and spectrograph
frames_by_expid = {}
for filename in args.frames:
log.info("reading %s" % filename)
frame = io.read_frame(filename)
expid = safe_read_key(frame.meta, "EXPID")
camera = safe_read_key(frame.meta, "CAMERA").strip().lower()
spec = camera[1]
uniq_key = (expid, spec)
if uniq_key in frames_by_expid.keys():
frames_by_expid[uniq_key][camera] = frame
else:
frames_by_expid[uniq_key] = {camera: frame}
frames = {}
flats = {}
skies = {}
spectrograph = None
starfibers = None
starindices = None
fibermap = None
# For each unique expid,spec pair, get the logical OR of the FIBERSTATUS for all
# cameras and then proceed with extracting the frame information
# once we modify the fibermap FIBERSTATUS
for (expid, spec), camdict in frames_by_expid.items():
fiberstatus = None
for frame in camdict.values():
if fiberstatus is None:
fiberstatus = frame.fibermap['FIBERSTATUS'].data.copy()
else:
fiberstatus |= frame.fibermap['FIBERSTATUS']
for camera, frame in camdict.items():
frame.fibermap['FIBERSTATUS'] |= fiberstatus
# Set fibermask flagged spectra to have 0 flux and variance
frame = get_fiberbitmasked_frame(frame,
bitmask='stdstars',
ivar_framemask=True)
frame_fibermap = frame.fibermap
frame_starindices = np.where(isStdStar(frame_fibermap))[0]
#- Confirm that all fluxes have entries but trust targeting bits
#- to get basic magnitude range correct
keep = np.ones(len(frame_starindices), dtype=bool)
for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']: #- and W1 and W2?
keep &= frame_fibermap[colname][frame_starindices] > 10**(
(22.5 - 30) / 2.5)
keep &= frame_fibermap[colname][frame_starindices] < 10**(
(22.5 - 0) / 2.5)
frame_starindices = frame_starindices[keep]
if spectrograph is None:
spectrograph = frame.spectrograph
fibermap = frame_fibermap
starindices = frame_starindices
starfibers = fibermap["FIBER"][starindices]
elif spectrograph != frame.spectrograph:
log.error("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
raise ValueError("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
elif starindices.size != frame_starindices.size or np.sum(
starindices != frame_starindices) > 0:
log.error("incompatible fibermap")
raise ValueError("incompatible fibermap")
if not camera in frames:
frames[camera] = []
frames[camera].append(frame)
# possibly cleanup memory
del frames_by_expid
for filename in args.skymodels:
log.info("reading %s" % filename)
sky = io.read_sky(filename)
camera = safe_read_key(sky.header, "CAMERA").strip().lower()
if not camera in skies:
skies[camera] = []
skies[camera].append(sky)
for filename in args.fiberflats:
log.info("reading %s" % filename)
flat = io.read_fiberflat(filename)
camera = safe_read_key(flat.header, "CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if camera in flats:
log.warning(
"cannot handle several flats of same camera (%s), will use only the first one"
% camera)
#raise ValueError("cannot handle several flats of same camera (%s)"%camera)
else:
flats[camera] = flat
if starindices.size == 0:
log.error("no STD star found in fibermap")
raise ValueError("no STD star found in fibermap")
log.info("found %d STD stars" % starindices.size)
log.warning("Not using flux errors for Standard Star fits!")
# DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
############################################
# since poping dict, we need to copy keys to iterate over to avoid
# RuntimeError due to changing dict
frame_cams = list(frames.keys())
for cam in frame_cams:
if not cam in skies:
log.warning("Missing sky for %s" % cam)
frames.pop(cam)
continue
if not cam in flats:
log.warning("Missing flat for %s" % cam)
frames.pop(cam)
continue
flat = flats[cam]
for frame, sky in zip(frames[cam], skies[cam]):
frame.flux = frame.flux[starindices]
frame.ivar = frame.ivar[starindices]
frame.ivar *= (frame.mask[starindices] == 0)
frame.ivar *= (sky.ivar[starindices] != 0)
frame.ivar *= (sky.mask[starindices] == 0)
frame.ivar *= (flat.ivar[starindices] != 0)
frame.ivar *= (flat.mask[starindices] == 0)
frame.flux *= (frame.ivar > 0) # just for clean plots
for star in range(frame.flux.shape[0]):
ok = np.where((frame.ivar[star] > 0)
& (flat.fiberflat[star] != 0))[0]
if ok.size > 0:
frame.flux[star] = frame.flux[star] / flat.fiberflat[
star] - sky.flux[star]
frame.resolution_data = frame.resolution_data[starindices]
# CHECK S/N
############################################
# for each band in 'brz', record quadratic sum of median S/N across wavelength
snr = dict()
for band in ['b', 'r', 'z']:
snr[band] = np.zeros(starindices.size)
for cam in frames:
band = cam[0].lower()
for frame in frames[cam]:
msnr = np.median(frame.flux * np.sqrt(frame.ivar) /
np.sqrt(np.gradient(frame.wave)),
axis=1) # median SNR per sqrt(A.)
msnr *= (msnr > 0)
snr[band] = np.sqrt(snr[band]**2 + msnr**2)
log.info("SNR(B) = {}".format(snr['b']))
###############################
max_number_of_stars = 50
min_blue_snr = 4.
###############################
indices = np.argsort(snr['b'])[::-1][:max_number_of_stars]
validstars = np.where(snr['b'][indices] > min_blue_snr)[0]
#- TODO: later we filter on models based upon color, thus throwing
#- away very blue stars for which we don't have good models.
log.info("Number of stars with median stacked blue S/N > {} /sqrt(A) = {}".
format(min_blue_snr, validstars.size))
if validstars.size == 0:
log.error("No valid star")
sys.exit(12)
validstars = indices[validstars]
for band in ['b', 'r', 'z']:
snr[band] = snr[band][validstars]
log.info("BLUE SNR of selected stars={}".format(snr['b']))
for cam in frames:
for frame in frames[cam]:
frame.flux = frame.flux[validstars]
frame.ivar = frame.ivar[validstars]
frame.resolution_data = frame.resolution_data[validstars]
starindices = starindices[validstars]
starfibers = starfibers[validstars]
nstars = starindices.size
fibermap = Table(fibermap[starindices])
# MASK OUT THROUGHPUT DIP REGION
############################################
mask_throughput_dip_region = True
if mask_throughput_dip_region:
wmin = 4300.
wmax = 4500.
log.warning(
"Masking out the wavelength region [{},{}]A in the standard star fit"
.format(wmin, wmax))
for cam in frames:
for frame in frames[cam]:
ii = np.where((frame.wave >= wmin) & (frame.wave <= wmax))[0]
if ii.size > 0:
frame.ivar[:, ii] = 0
# READ MODELS
############################################
log.info("reading star models in %s" % args.starmodels)
stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
args.starmodels)
# COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
############################################
#- Support older fibermaps
if 'PHOTSYS' not in fibermap.colnames:
log.warning('Old fibermap format; using defaults for missing columns')
log.warning(" PHOTSYS = 'S'")
log.warning(" MW_TRANSMISSION_G/R/Z = 1.0")
log.warning(" EBV = 0.0")
fibermap['PHOTSYS'] = 'S'
fibermap['MW_TRANSMISSION_G'] = 1.0
fibermap['MW_TRANSMISSION_R'] = 1.0
fibermap['MW_TRANSMISSION_Z'] = 1.0
fibermap['EBV'] = 0.0
model_filters = dict()
for band in ["G", "R", "Z"]:
for photsys in np.unique(fibermap['PHOTSYS']):
model_filters[band + photsys] = load_legacy_survey_filter(
band=band, photsys=photsys)
log.info("computing model mags for %s" % sorted(model_filters.keys()))
model_mags = dict()
fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
for filter_name, filter_response in model_filters.items():
model_mags[filter_name] = filter_response.get_ab_magnitude(
stdflux * fluxunits, stdwave)
log.info("done computing model mags")
# LOOP ON STARS TO FIND BEST MODEL
############################################
linear_coefficients = np.zeros((nstars, stdflux.shape[0]))
chi2dof = np.zeros((nstars))
redshift = np.zeros((nstars))
normflux = []
star_mags = dict()
star_unextincted_mags = dict()
for band in ['G', 'R', 'Z']:
star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_' + band])
star_unextincted_mags[band] = 22.5 - 2.5 * np.log10(
fibermap['FLUX_' + band] / fibermap['MW_TRANSMISSION_' + band])
star_colors = dict()
star_colors['G-R'] = star_mags['G'] - star_mags['R']
star_colors['R-Z'] = star_mags['R'] - star_mags['Z']
star_unextincted_colors = dict()
star_unextincted_colors[
'G-R'] = star_unextincted_mags['G'] - star_unextincted_mags['R']
star_unextincted_colors[
'R-Z'] = star_unextincted_mags['R'] - star_unextincted_mags['Z']
fitted_model_colors = np.zeros(nstars)
for star in range(nstars):
log.info("finding best model for observed star #%d" % star)
# np.array of wave,flux,ivar,resol
wave = {}
flux = {}
ivar = {}
resolution_data = {}
for camera in frames:
for i, frame in enumerate(frames[camera]):
identifier = "%s-%d" % (camera, i)
wave[identifier] = frame.wave
flux[identifier] = frame.flux[star]
ivar[identifier] = frame.ivar[star]
resolution_data[identifier] = frame.resolution_data[star]
# preselect models based on magnitudes
photsys = fibermap['PHOTSYS'][star]
if not args.color in ['G-R', 'R-Z']:
raise ValueError('Unknown color {}'.format(args.color))
bands = args.color.split("-")
model_colors = model_mags[bands[0] + photsys] - model_mags[bands[1] +
photsys]
color_diff = model_colors - star_unextincted_colors[args.color][star]
selection = np.abs(color_diff) < args.delta_color
if np.sum(selection) == 0:
log.warning("no model in the selected color range for this star")
continue
# smallest cube in parameter space including this selection (needed for interpolation)
new_selection = (teff >= np.min(teff[selection])) & (teff <= np.max(
teff[selection]))
new_selection &= (logg >= np.min(logg[selection])) & (logg <= np.max(
logg[selection]))
new_selection &= (feh >= np.min(feh[selection])) & (feh <= np.max(
feh[selection]))
selection = np.where(new_selection)[0]
log.info(
"star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d"
% (star, starfibers[star], args.color,
star_unextincted_colors[args.color][star], selection.size,
stdflux.shape[0]))
# Match unextincted standard stars to data
coefficients, redshift[star], chi2dof[star] = match_templates(
wave,
flux,
ivar,
resolution_data,
stdwave,
stdflux[selection],
teff[selection],
logg[selection],
feh[selection],
ncpu=args.ncpu,
z_max=args.z_max,
z_res=args.z_res,
template_error=args.template_error)
linear_coefficients[star, selection] = coefficients
log.info(
'Star Fiber: {}; TEFF: {:.3f}; LOGG: {:.3f}; FEH: {:.3f}; Redshift: {:g}; Chisq/dof: {:.3f}'
.format(starfibers[star], np.inner(teff,
linear_coefficients[star]),
np.inner(logg, linear_coefficients[star]),
np.inner(feh, linear_coefficients[star]), redshift[star],
chi2dof[star]))
# Apply redshift to original spectrum at full resolution
model = np.zeros(stdwave.size)
redshifted_stdwave = stdwave * (1 + redshift[star])
for i, c in enumerate(linear_coefficients[star]):
if c != 0:
model += c * np.interp(stdwave, redshifted_stdwave, stdflux[i])
# Apply dust extinction to the model
log.info("Applying MW dust extinction to star {} with EBV = {}".format(
star, fibermap['EBV'][star]))
model *= dust_transmission(stdwave, fibermap['EBV'][star])
# Compute final color of dust-extincted model
photsys = fibermap['PHOTSYS'][star]
if not args.color in ['G-R', 'R-Z']:
raise ValueError('Unknown color {}'.format(args.color))
bands = args.color.split("-")
model_mag1 = model_filters[bands[0] + photsys].get_ab_magnitude(
model * fluxunits, stdwave)
model_mag2 = model_filters[bands[1] + photsys].get_ab_magnitude(
model * fluxunits, stdwave)
fitted_model_colors[star] = model_mag1 - model_mag2
if bands[0] == "R":
model_magr = model_mag1
elif bands[1] == "R":
model_magr = model_mag2
#- TODO: move this back into normalize_templates, at the cost of
#- recalculating a model magnitude?
# Normalize the best model using reported magnitude
scalefac = 10**((model_magr - star_mags['R'][star]) / 2.5)
log.info('scaling R mag {:.3f} to {:.3f} using scale {}'.format(
model_magr, star_mags['R'][star], scalefac))
normflux.append(model * scalefac)
# Now write the normalized flux for all best models to a file
normflux = np.array(normflux)
fitted_stars = np.where(chi2dof != 0)[0]
if fitted_stars.size == 0:
log.error("No star has been fit.")
sys.exit(12)
data = {}
data['LOGG'] = linear_coefficients[fitted_stars, :].dot(logg)
data['TEFF'] = linear_coefficients[fitted_stars, :].dot(teff)
data['FEH'] = linear_coefficients[fitted_stars, :].dot(feh)
data['CHI2DOF'] = chi2dof[fitted_stars]
data['REDSHIFT'] = redshift[fitted_stars]
data['COEFF'] = linear_coefficients[fitted_stars, :]
data['DATA_%s' % args.color] = star_colors[args.color][fitted_stars]
data['MODEL_%s' % args.color] = fitted_model_colors[fitted_stars]
data['BLUE_SNR'] = snr['b'][fitted_stars]
data['RED_SNR'] = snr['r'][fitted_stars]
data['NIR_SNR'] = snr['z'][fitted_stars]
io.write_stdstar_models(args.outfile, normflux, stdwave,
starfibers[fitted_stars], data)
def main(args, comm=None):
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
log = get_logger()
log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
(args.color, args.delta_color))
if args.mpi or comm is not None:
from mpi4py import MPI
if comm is None:
comm = MPI.COMM_WORLD
size = comm.Get_size()
rank = comm.Get_rank()
if rank == 0:
log.info('mpi parallelizing with {} ranks'.format(size))
else:
comm = None
rank = 0
size = 1
# disable multiprocess by forcing ncpu = 1 when using MPI
if comm is not None:
ncpu = 1
if rank == 0:
log.info('disabling multiprocess (forcing ncpu = 1)')
else:
ncpu = args.ncpu
if ncpu > 1:
if rank == 0:
log.info(
'multiprocess parallelizing with {} processes'.format(ncpu))
if args.ignore_gpu and desispec.fluxcalibration.use_gpu:
# Opt-out of GPU usage
desispec.fluxcalibration.use_gpu = False
if rank == 0:
log.info('ignoring GPU')
elif desispec.fluxcalibration.use_gpu:
# Nothing to do here, GPU is used by default if available
if rank == 0:
log.info('using GPU')
else:
if rank == 0:
log.info('GPU not available')
std_targetids = None
if args.std_targetids is not None:
std_targetids = args.std_targetids
# READ DATA
############################################
# First loop through and group by exposure and spectrograph
frames_by_expid = {}
rows = list()
for filename in args.frames:
log.info("reading %s" % filename)
frame = io.read_frame(filename)
night = safe_read_key(frame.meta, "NIGHT")
expid = safe_read_key(frame.meta, "EXPID")
camera = safe_read_key(frame.meta, "CAMERA").strip().lower()
rows.append((night, expid, camera))
spec = camera[1]
uniq_key = (expid, spec)
if uniq_key in frames_by_expid.keys():
frames_by_expid[uniq_key][camera] = frame
else:
frames_by_expid[uniq_key] = {camera: frame}
input_frames_table = Table(rows=rows, names=('NIGHT', 'EXPID', 'TILEID'))
frames = {}
flats = {}
skies = {}
spectrograph = None
starfibers = None
starindices = None
fibermap = None
# For each unique expid,spec pair, get the logical OR of the FIBERSTATUS for all
# cameras and then proceed with extracting the frame information
# once we modify the fibermap FIBERSTATUS
for (expid, spec), camdict in frames_by_expid.items():
fiberstatus = None
for frame in camdict.values():
if fiberstatus is None:
fiberstatus = frame.fibermap['FIBERSTATUS'].data.copy()
else:
fiberstatus |= frame.fibermap['FIBERSTATUS']
for camera, frame in camdict.items():
frame.fibermap['FIBERSTATUS'] |= fiberstatus
# Set fibermask flagged spectra to have 0 flux and variance
frame = get_fiberbitmasked_frame(frame,
bitmask='stdstars',
ivar_framemask=True)
frame_fibermap = frame.fibermap
if std_targetids is None:
frame_starindices = np.where(isStdStar(frame_fibermap))[0]
else:
frame_starindices = np.nonzero(
np.isin(frame_fibermap['TARGETID'], std_targetids))[0]
#- Confirm that all fluxes have entries but trust targeting bits
#- to get basic magnitude range correct
keep_legacy = np.ones(len(frame_starindices), dtype=bool)
for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']: #- and W1 and W2?
keep_legacy &= frame_fibermap[colname][
frame_starindices] > 10**((22.5 - 30) / 2.5)
keep_legacy &= frame_fibermap[colname][
frame_starindices] < 10**((22.5 - 0) / 2.5)
keep_gaia = np.ones(len(frame_starindices), dtype=bool)
for colname in ['G', 'BP', 'RP']: #- and W1 and W2?
keep_gaia &= frame_fibermap[
'GAIA_PHOT_' + colname +
'_MEAN_MAG'][frame_starindices] > 10
keep_gaia &= frame_fibermap[
'GAIA_PHOT_' + colname +
'_MEAN_MAG'][frame_starindices] < 20
n_legacy_std = keep_legacy.sum()
n_gaia_std = keep_gaia.sum()
keep = keep_legacy | keep_gaia
# accept both types of standards for the time being
# keep the indices for gaia/legacy subsets
gaia_indices = keep_gaia[keep]
legacy_indices = keep_legacy[keep]
frame_starindices = frame_starindices[keep]
if spectrograph is None:
spectrograph = frame.spectrograph
fibermap = frame_fibermap
starindices = frame_starindices
starfibers = fibermap["FIBER"][starindices]
elif spectrograph != frame.spectrograph:
log.error("incompatible spectrographs {} != {}".format(
spectrograph, frame.spectrograph))
raise ValueError("incompatible spectrographs {} != {}".format(
spectrograph, frame.spectrograph))
elif starindices.size != frame_starindices.size or np.sum(
starindices != frame_starindices) > 0:
log.error("incompatible fibermap")
raise ValueError("incompatible fibermap")
if not camera in frames:
frames[camera] = []
frames[camera].append(frame)
# possibly cleanup memory
del frames_by_expid
for filename in args.skymodels:
log.info("reading %s" % filename)
sky = io.read_sky(filename)
camera = safe_read_key(sky.header, "CAMERA").strip().lower()
if not camera in skies:
skies[camera] = []
skies[camera].append(sky)
for filename in args.fiberflats:
log.info("reading %s" % filename)
flat = io.read_fiberflat(filename)
camera = safe_read_key(flat.header, "CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if camera in flats:
log.warning(
"cannot handle several flats of same camera (%s), will use only the first one"
% camera)
#raise ValueError("cannot handle several flats of same camera (%s)"%camera)
else:
flats[camera] = flat
# if color is not specified we decide on the fly
color = args.color
if color is not None:
if color[:4] == 'GAIA':
legacy_color = False
gaia_color = True
else:
legacy_color = True
gaia_color = False
if n_legacy_std == 0 and legacy_color:
raise Exception(
'Specified Legacy survey color, but no legacy standards')
if n_gaia_std == 0 and gaia_color:
raise Exception('Specified gaia color, but no gaia stds')
if starindices.size == 0:
log.error("no STD star found in fibermap")
raise ValueError("no STD star found in fibermap")
log.info("found %d STD stars" % starindices.size)
if n_legacy_std == 0:
gaia_std = True
if color is None:
color = 'GAIA-BP-RP'
else:
gaia_std = False
if color is None:
color = 'G-R'
if n_gaia_std > 0:
log.info('Gaia standards found but not used')
if gaia_std:
# The name of the reference filter to which we normalize the flux
ref_mag_name = 'GAIA-G'
color_band1, color_band2 = ['GAIA-' + _ for _ in color[5:].split('-')]
log.info(
"Using Gaia standards with color {} and normalizing to {}".format(
color, ref_mag_name))
# select appropriate subset of standards
starindices = starindices[gaia_indices]
starfibers = starfibers[gaia_indices]
else:
ref_mag_name = 'R'
color_band1, color_band2 = color.split('-')
log.info("Using Legacy standards with color {} and normalizing to {}".
format(color, ref_mag_name))
# select appropriate subset of standards
starindices = starindices[legacy_indices]
starfibers = starfibers[legacy_indices]
# excessive check but just in case
if not color in ['G-R', 'R-Z', 'GAIA-BP-RP', 'GAIA-G-RP']:
raise ValueError('Unknown color {}'.format(color))
# log.warning("Not using flux errors for Standard Star fits!")
# DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
############################################
# since poping dict, we need to copy keys to iterate over to avoid
# RuntimeError due to changing dict
frame_cams = list(frames.keys())
for cam in frame_cams:
if not cam in skies:
log.warning("Missing sky for %s" % cam)
frames.pop(cam)
continue
if not cam in flats:
log.warning("Missing flat for %s" % cam)
frames.pop(cam)
continue
flat = flats[cam]
for frame, sky in zip(frames[cam], skies[cam]):
frame.flux = frame.flux[starindices]
frame.ivar = frame.ivar[starindices]
frame.ivar *= (frame.mask[starindices] == 0)
frame.ivar *= (sky.ivar[starindices] != 0)
frame.ivar *= (sky.mask[starindices] == 0)
frame.ivar *= (flat.ivar[starindices] != 0)
frame.ivar *= (flat.mask[starindices] == 0)
frame.flux *= (frame.ivar > 0) # just for clean plots
for star in range(frame.flux.shape[0]):
ok = np.where((frame.ivar[star] > 0)
& (flat.fiberflat[star] != 0))[0]
if ok.size > 0:
frame.flux[star] = frame.flux[star] / flat.fiberflat[
star] - sky.flux[star]
frame.resolution_data = frame.resolution_data[starindices]
nframes = len(frames[cam])
if nframes > 1:
# optimal weights for the coaddition = ivar*throughput, not directly ivar,
# we estimate the relative throughput with median fluxes at this stage
medflux = np.zeros(nframes)
for i, frame in enumerate(frames[cam]):
if np.sum(frame.ivar > 0) == 0:
log.error(
"ivar=0 for all std star spectra in frame {}-{:08d}".
format(cam, frame.meta["EXPID"]))
else:
medflux[i] = np.median(frame.flux[frame.ivar > 0])
log.debug("medflux = {}".format(medflux))
medflux *= (medflux > 0)
if np.sum(medflux > 0) == 0:
log.error(
"mean median flux = 0, for all stars in fibers {}".format(
list(frames[cam][0].fibermap["FIBER"][starindices])))
sys.exit(12)
mmedflux = np.mean(medflux[medflux > 0])
weights = medflux / mmedflux
log.info("coadding {} exposures in cam {}, w={}".format(
nframes, cam, weights))
sw = np.zeros(frames[cam][0].flux.shape)
swf = np.zeros(frames[cam][0].flux.shape)
swr = np.zeros(frames[cam][0].resolution_data.shape)
for i, frame in enumerate(frames[cam]):
sw += weights[i] * frame.ivar
swf += weights[i] * frame.ivar * frame.flux
swr += weights[i] * frame.ivar[:,
None, :] * frame.resolution_data
coadded_frame = frames[cam][0]
coadded_frame.ivar = sw
coadded_frame.flux = swf / (sw + (sw == 0))
coadded_frame.resolution_data = swr / ((sw +
(sw == 0))[:, None, :])
frames[cam] = [coadded_frame]
# CHECK S/N
############################################
# for each band in 'brz', record quadratic sum of median S/N across wavelength
snr = dict()
for band in ['b', 'r', 'z']:
snr[band] = np.zeros(starindices.size)
for cam in frames:
band = cam[0].lower()
for frame in frames[cam]:
msnr = np.median(frame.flux * np.sqrt(frame.ivar) /
np.sqrt(np.gradient(frame.wave)),
axis=1) # median SNR per sqrt(A.)
msnr *= (msnr > 0)
snr[band] = np.sqrt(snr[band]**2 + msnr**2)
log.info("SNR(B) = {}".format(snr['b']))
###############################
max_number_of_stars = 50
min_blue_snr = 4.
###############################
indices = np.argsort(snr['b'])[::-1][:max_number_of_stars]
validstars = np.where(snr['b'][indices] > min_blue_snr)[0]
#- TODO: later we filter on models based upon color, thus throwing
#- away very blue stars for which we don't have good models.
log.info("Number of stars with median stacked blue S/N > {} /sqrt(A) = {}".
format(min_blue_snr, validstars.size))
if validstars.size == 0:
log.error("No valid star")
sys.exit(12)
validstars = indices[validstars]
for band in ['b', 'r', 'z']:
snr[band] = snr[band][validstars]
log.info("BLUE SNR of selected stars={}".format(snr['b']))
for cam in frames:
for frame in frames[cam]:
frame.flux = frame.flux[validstars]
frame.ivar = frame.ivar[validstars]
frame.resolution_data = frame.resolution_data[validstars]
starindices = starindices[validstars]
starfibers = starfibers[validstars]
nstars = starindices.size
fibermap = Table(fibermap[starindices])
# MASK OUT THROUGHPUT DIP REGION
############################################
mask_throughput_dip_region = True
if mask_throughput_dip_region:
wmin = 4300.
wmax = 4500.
log.warning(
"Masking out the wavelength region [{},{}]A in the standard star fit"
.format(wmin, wmax))
for cam in frames:
for frame in frames[cam]:
ii = np.where((frame.wave >= wmin) & (frame.wave <= wmax))[0]
if ii.size > 0:
frame.ivar[:, ii] = 0
# READ MODELS
############################################
log.info("reading star models in %s" % args.starmodels)
stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
args.starmodels)
# COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
############################################
#- Support older fibermaps
if 'PHOTSYS' not in fibermap.colnames:
log.warning('Old fibermap format; using defaults for missing columns')
log.warning(" PHOTSYS = 'S'")
log.warning(" EBV = 0.0")
fibermap['PHOTSYS'] = 'S'
fibermap['EBV'] = 0.0
if not np.in1d(np.unique(fibermap['PHOTSYS']), ['', 'N', 'S', 'G']).all():
log.error('Unknown PHOTSYS found')
raise Exception('Unknown PHOTSYS found')
# Fetching Filter curves
model_filters = dict()
for band in ["G", "R", "Z"]:
for photsys in np.unique(fibermap['PHOTSYS']):
if photsys in ['N', 'S']:
model_filters[band + photsys] = load_legacy_survey_filter(
band=band, photsys=photsys)
if len(model_filters) == 0:
log.info('No Legacy survey photometry identified in fibermap')
# I will always load gaia data even if we are fitting LS standards only
for band in ["G", "BP", "RP"]:
model_filters["GAIA-" + band] = load_gaia_filter(band=band, dr=2)
# Compute model mags on rank 0 and bcast result to other ranks
# This sidesteps an OOM event on Cori Haswell with "-c 2"
model_mags = None
if rank == 0:
log.info("computing model mags for %s" % sorted(model_filters.keys()))
model_mags = dict()
for filter_name in model_filters.keys():
model_mags[filter_name] = get_magnitude(stdwave, stdflux,
model_filters, filter_name)
log.info("done computing model mags")
if comm is not None:
model_mags = comm.bcast(model_mags, root=0)
# LOOP ON STARS TO FIND BEST MODEL
############################################
star_mags = dict()
star_unextincted_mags = dict()
if gaia_std and (fibermap['EBV'] == 0).all():
log.info("Using E(B-V) from SFD rather than FIBERMAP")
# when doing gaia standards, on old tiles the
# EBV is not set so we fetch from SFD (in original SFD scaling)
ebv = SFDMap(scaling=1).ebv(
acoo.SkyCoord(ra=fibermap['TARGET_RA'] * units.deg,
dec=fibermap['TARGET_DEC'] * units.deg))
else:
ebv = fibermap['EBV']
photometric_systems = np.unique(fibermap['PHOTSYS'])
if not gaia_std:
for band in ['G', 'R', 'Z']:
star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_' + band])
star_unextincted_mags[band] = np.zeros(star_mags[band].shape)
for photsys in photometric_systems:
r_band = extinction_total_to_selective_ratio(
band, photsys) # dimensionless
# r_band = a_band / E(B-V)
# E(B-V) is a difference of magnitudes (dimensionless)
# a_band = -2.5*log10(effective dust transmission) , dimensionless
# effective dust transmission =
# integral( SED(lambda) * filter_transmission(lambda,band) * dust_transmission(lambda,E(B-V)) dlamdba)
# / integral( SED(lambda) * filter_transmission(lambda,band) dlamdba)
selection = (fibermap['PHOTSYS'] == photsys)
a_band = r_band * ebv[selection] # dimensionless
star_unextincted_mags[band][selection] = 22.5 - 2.5 * np.log10(
fibermap['FLUX_' + band][selection]) - a_band
for band in ['G', 'BP', 'RP']:
star_mags['GAIA-' + band] = fibermap['GAIA_PHOT_' + band + '_MEAN_MAG']
for band, extval in gaia_extinction(star_mags['GAIA-G'],
star_mags['GAIA-BP'],
star_mags['GAIA-RP'], ebv).items():
star_unextincted_mags['GAIA-' +
band] = star_mags['GAIA-' + band] - extval
star_colors = dict()
star_unextincted_colors = dict()
# compute the colors and define the unextincted colors
# the unextincted colors are filled later
if not gaia_std:
for c1, c2 in ['GR', 'RZ']:
star_colors[c1 + '-' + c2] = star_mags[c1] - star_mags[c2]
star_unextincted_colors[c1 + '-' +
c2] = (star_unextincted_mags[c1] -
star_unextincted_mags[c2])
for c1, c2 in [('BP', 'RP'), ('G', 'RP')]:
star_colors['GAIA-' + c1 + '-' + c2] = (star_mags['GAIA-' + c1] -
star_mags['GAIA-' + c2])
star_unextincted_colors['GAIA-' + c1 + '-' +
c2] = (star_unextincted_mags['GAIA-' + c1] -
star_unextincted_mags['GAIA-' + c2])
linear_coefficients = np.zeros((nstars, stdflux.shape[0]))
chi2dof = np.zeros((nstars))
redshift = np.zeros((nstars))
normflux = np.zeros((nstars, stdwave.size))
fitted_model_colors = np.zeros(nstars)
local_comm, head_comm = None, None
if comm is not None:
# All ranks in local_comm work on the same stars
local_comm = comm.Split(rank % nstars, rank)
# The color 1 in head_comm contains all ranks that are have rank 0 in local_comm
head_comm = comm.Split(rank < nstars, rank)
for star in range(rank % nstars, nstars, size):
log.info("rank %d: finding best model for observed star #%d" %
(rank, star))
# np.array of wave,flux,ivar,resol
wave = {}
flux = {}
ivar = {}
resolution_data = {}
for camera in frames:
for i, frame in enumerate(frames[camera]):
identifier = "%s-%d" % (camera, i)
wave[identifier] = frame.wave
flux[identifier] = frame.flux[star]
ivar[identifier] = frame.ivar[star]
resolution_data[identifier] = frame.resolution_data[star]
# preselect models based on magnitudes
photsys = fibermap['PHOTSYS'][star]
if gaia_std:
model_colors = model_mags[color_band1] - model_mags[color_band2]
else:
model_colors = model_mags[color_band1 +
photsys] - model_mags[color_band2 +
photsys]
color_diff = model_colors - star_unextincted_colors[color][star]
selection = np.abs(color_diff) < args.delta_color
if np.sum(selection) == 0:
log.warning("no model in the selected color range for this star")
continue
# smallest cube in parameter space including this selection (needed for interpolation)
new_selection = (teff >= np.min(teff[selection])) & (teff <= np.max(
teff[selection]))
new_selection &= (logg >= np.min(logg[selection])) & (logg <= np.max(
logg[selection]))
new_selection &= (feh >= np.min(feh[selection])) & (feh <= np.max(
feh[selection]))
selection = np.where(new_selection)[0]
log.info(
"star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d"
% (star, starfibers[star], color,
star_unextincted_colors[color][star], selection.size,
stdflux.shape[0]))
# Match unextincted standard stars to data
match_templates_result = match_templates(
wave,
flux,
ivar,
resolution_data,
stdwave,
stdflux[selection],
teff[selection],
logg[selection],
feh[selection],
ncpu=ncpu,
z_max=args.z_max,
z_res=args.z_res,
template_error=args.template_error,
comm=local_comm)
# Only local rank 0 can perform the remaining work
if local_comm is not None and local_comm.Get_rank() != 0:
continue
coefficients, redshift[star], chi2dof[star] = match_templates_result
linear_coefficients[star, selection] = coefficients
log.info(
'Star Fiber: {}; TEFF: {:.3f}; LOGG: {:.3f}; FEH: {:.3f}; Redshift: {:g}; Chisq/dof: {:.3f}'
.format(starfibers[star], np.inner(teff,
linear_coefficients[star]),
np.inner(logg, linear_coefficients[star]),
np.inner(feh, linear_coefficients[star]), redshift[star],
chi2dof[star]))
# Apply redshift to original spectrum at full resolution
model = np.zeros(stdwave.size)
redshifted_stdwave = stdwave * (1 + redshift[star])
for i, c in enumerate(linear_coefficients[star]):
if c != 0:
model += c * np.interp(stdwave, redshifted_stdwave, stdflux[i])
# Apply dust extinction to the model
log.info("Applying MW dust extinction to star {} with EBV = {}".format(
star, ebv[star]))
model *= dust_transmission(stdwave, ebv[star])
# Compute final color of dust-extincted model
photsys = fibermap['PHOTSYS'][star]
if not gaia_std:
model_mag1, model_mag2 = [
get_magnitude(stdwave, model, model_filters, _ + photsys)
for _ in [color_band1, color_band2]
]
else:
model_mag1, model_mag2 = [
get_magnitude(stdwave, model, model_filters, _)
for _ in [color_band1, color_band2]
]
if color_band1 == ref_mag_name:
model_magr = model_mag1
elif color_band2 == ref_mag_name:
model_magr = model_mag2
else:
# if the reference magnitude is not among colours
# I'm fetching it separately. This will happen when
# colour is BP-RP and ref magnitude is G
if gaia_std:
model_magr = get_magnitude(stdwave, model, model_filters,
ref_mag_name)
else:
model_magr = get_magnitude(stdwave, model, model_filters,
ref_mag_name + photsys)
fitted_model_colors[star] = model_mag1 - model_mag2
#- TODO: move this back into normalize_templates, at the cost of
#- recalculating a model magnitude?
cur_refmag = star_mags[ref_mag_name][star]
# Normalize the best model using reported magnitude
scalefac = 10**((model_magr - cur_refmag) / 2.5)
log.info('scaling {} mag {:.3f} to {:.3f} using scale {}'.format(
ref_mag_name, model_magr, cur_refmag, scalefac))
normflux[star] = model * scalefac
if head_comm is not None and rank < nstars: # head_comm color is 1
linear_coefficients = head_comm.reduce(linear_coefficients,
op=MPI.SUM,
root=0)
redshift = head_comm.reduce(redshift, op=MPI.SUM, root=0)
chi2dof = head_comm.reduce(chi2dof, op=MPI.SUM, root=0)
fitted_model_colors = head_comm.reduce(fitted_model_colors,
op=MPI.SUM,
root=0)
normflux = head_comm.reduce(normflux, op=MPI.SUM, root=0)
# Check at least one star was fit. The check is peformed on rank 0 and
# the result is bcast to other ranks so that all ranks exit together if
# the check fails.
atleastonestarfit = False
if rank == 0:
fitted_stars = np.where(chi2dof != 0)[0]
atleastonestarfit = fitted_stars.size > 0
if comm is not None:
atleastonestarfit = comm.bcast(atleastonestarfit, root=0)
if not atleastonestarfit:
log.error("No star has been fit.")
sys.exit(12)
# Now write the normalized flux for all best models to a file
if rank == 0:
# get the fibermap from any input frame for the standard stars
fibermap = Table(frame.fibermap)
keep = np.isin(fibermap['FIBER'], starfibers[fitted_stars])
fibermap = fibermap[keep]
# drop fibermap columns specific to exposures instead of targets
for col in [
'DELTA_X', 'DELTA_Y', 'EXPTIME', 'NUM_ITER', 'FIBER_RA',
'FIBER_DEC', 'FIBER_X', 'FIBER_Y'
]:
if col in fibermap.colnames:
fibermap.remove_column(col)
data = {}
data['LOGG'] = linear_coefficients[fitted_stars, :].dot(logg)
data['TEFF'] = linear_coefficients[fitted_stars, :].dot(teff)
data['FEH'] = linear_coefficients[fitted_stars, :].dot(feh)
data['CHI2DOF'] = chi2dof[fitted_stars]
data['REDSHIFT'] = redshift[fitted_stars]
data['COEFF'] = linear_coefficients[fitted_stars, :]
data['DATA_%s' % color] = star_colors[color][fitted_stars]
data['MODEL_%s' % color] = fitted_model_colors[fitted_stars]
data['BLUE_SNR'] = snr['b'][fitted_stars]
data['RED_SNR'] = snr['r'][fitted_stars]
data['NIR_SNR'] = snr['z'][fitted_stars]
io.write_stdstar_models(args.outfile, normflux, stdwave,
starfibers[fitted_stars], data, fibermap,
input_frames_table)