def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--infile',
type=str,
default=None,
required=True,
help=
'path of DESI frame fits file corresponding to a continuum lamp exposure'
)
parser.add_argument('--outfile',
type=str,
default=None,
required=True,
help='path of DESI fiberflat fits file')
args = parser.parse_args()
log = get_logger()
log.info("starting")
frame = read_frame(args.infile)
fiberflat = compute_fiberflat(frame)
write_fiberflat(args.outfile, fiberflat, frame.header)
log.info("successfully wrote %s" % args.outfile)
def main(args):
log = get_logger()
log.info("starting at {}".format(time.asctime()))
# Process
frame = read_frame(args.infile)
fiberflat = compute_fiberflat(frame,
nsig_clipping=args.nsig,
accuracy=args.acc,
smoothing_res=args.smoothing_resolution)
# QA
if (args.qafile is not None):
log.info("performing fiberflat QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame,
flavor=frame.meta['FLAVOR'])
# Run
qaframe.run_qa('FIBERFLAT', (frame, fiberflat))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fiberflat(args.qafig, qaframe, frame, fiberflat)
# Write
write_fiberflat(args.outfile, fiberflat, frame.meta)
log.info("successfully wrote %s" % args.outfile)
log.info("done at {}".format(time.asctime()))
def main(args) :
log=get_logger()
log.info("starting")
# Process
frame = read_frame(args.infile)
fiberflat = compute_fiberflat(frame)
# QA
if (args.qafile is not None):
log.info("performing fiberflat QA")
# Load
qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR'])
# Run
qaframe.run_qa('FIBERFLAT', (frame, fiberflat))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fiberflat(args.qafig, qaframe, frame, fiberflat)
# Write
write_fiberflat(args.outfile, fiberflat, frame.meta)
log.info("successfully wrote %s"%args.outfile)
def get_skyres(cframes, sub_sky=False):
from desispec.io import read_frame
from desispec.io.sky import read_sky
from desispec.sky import subtract_sky
if isinstance(cframes, list):
all_wave, all_flux, all_res, all_ivar = [], [], [], []
for cframe_file in cframes:
wave, flux, res, ivar = get_skyres(cframe_file)
# Concatenate and return
all_wave.append(wave)
all_flux.append(flux)
all_res.append(res)
all_ivar.append(ivar)
return np.concatenate(all_wave), np.concatenate(all_flux), \
np.concatenate(all_res), np.concatenate(all_ivar)
cframe = read_frame(cframes)
if cframe.meta['FLAVOR'] in ['flat', 'arc']:
raise ValueError("Bad flavor for exposure: {:s}".format(cframes))
# Sky
sky_file = cframes.replace('cframe', 'sky')
skymodel = read_sky(sky_file)
if sub_sky:
subtract_sky(cframe, skymodel)
# Resid
skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0]
res = cframe.flux[skyfibers]
ivar = cframe.ivar[skyfibers]
flux = skymodel.flux[skyfibers] # Residuals
wave = np.outer(np.ones(flux.shape[0]), cframe.wave)
# Return
return wave.flatten(), flux.flatten(), res.flatten(), ivar.flatten()
def main(args):
log = get_logger()
log.info("starting")
# Process
frame = read_frame(args.infile)
fiberflat = compute_fiberflat(frame)
# QA
if (args.qafile is not None):
log.info("performing fiberflat QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame,
flavor=frame.meta['FLAVOR'])
# Run
qaframe.run_qa('FIBERFLAT', (frame, fiberflat))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fiberflat(args.qafig, qaframe, frame, fiberflat)
# Write
write_fiberflat(args.outfile, fiberflat, frame.meta)
log.info("successfully wrote %s" % args.outfile)
def main(args):
log = get_logger()
for filename in args.infile:
log.info("reading %s" % filename)
frame = io.read_frame(filename)
flux_per_angstrom = None
if args.flux_per_angstrom:
flux_per_angstrom = True
elif args.flux_per_pixel:
flux_per_angstrom = False
else:
flux_per_angstrom = None
scores, comments = compute_and_append_frame_scores(
frame,
suffix=args.suffix,
flux_per_angstrom=flux_per_angstrom,
overwrite=args.overwrite)
log.info("Adding or replacing SCORES extention with {} in {}".format(
scores.keys(), filename))
write_bintable(filename,
data=scores,
comments=comments,
extname="SCORES",
clobber=True)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--infile',
type=str,
default=None,
required=True,
help='path of DESI exposure frame fits file')
parser.add_argument('--fibermap',
type=str,
default=None,
required=True,
help='path of DESI exposure frame fits file')
parser.add_argument('--fiberflat',
type=str,
default=None,
required=True,
help='path of DESI fiberflat fits file')
parser.add_argument('--outfile',
type=str,
default=None,
required=True,
help='path of DESI sky fits file')
args = parser.parse_args()
log = get_logger()
log.info("starting")
# read exposure to load data and get range of spectra
frame = read_frame(args.infile)
specmin = frame.header["SPECMIN"]
specmax = frame.header["SPECMAX"]
# read fibermap to locate sky fibers
fibermap = read_fibermap(args.fibermap)
selection = np.where((fibermap["OBJTYPE"] == "SKY")
& (fibermap["FIBER"] >= specmin)
& (fibermap["FIBER"] <= specmax))[0]
if selection.size == 0:
log.error("no sky fiber in fibermap %s" % args.fibermap)
sys.exit(12)
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
# compute sky model
skymodel = compute_sky(frame, fibermap)
# write result
write_sky(args.outfile, skymodel, frame.header)
log.info("successfully wrote %s" % args.outfile)
def load_s2n_values(objtype, nights, channel, sub_exposures=None):
fdict = dict(waves=[], s2n=[], fluxes=[], exptime=[], OII=[])
for night in nights:
if sub_exposures is not None:
exposures = sub_exposures
else:
exposures = get_exposures(night) #, raw=True)
for exposure in exposures:
fibermap_path = findfile(filetype='fibermap',
night=night,
expid=exposure)
fibermap_data = read_fibermap(fibermap_path)
flavor = fibermap_data.meta['FLAVOR']
if flavor.lower() in ('arc', 'flat', 'bias'):
log.debug('Skipping calibration {} exposure {:08d}'.format(
flavor, exposure))
continue
# Load simspec
simspec_file = fibermap_path.replace('fibermap', 'simspec')
sps_hdu = fits.open(simspec_file)
sps_tab = Table(sps_hdu['TRUTH'].data, masked=True)
sps_hdu.close()
objs = sps_tab['TEMPLATETYPE'] == objtype
if np.sum(objs) == 0:
continue
# Load spectra (flux or not fluxed; should not matter)
for ii in range(10):
camera = channel + str(ii)
cframe_path = findfile(filetype='cframe',
night=night,
expid=exposure,
camera=camera)
try:
cframe = read_frame(cframe_path)
except:
log.warn("Cannot find file: {:s}".format(cframe_path))
continue
# Calculate S/N per Ang
dwave = cframe.wave - np.roll(cframe.wave, 1)
dwave[0] = dwave[1]
#
iobjs = objs[cframe.fibers]
if np.sum(iobjs) == 0:
continue
s2n = cframe.flux[iobjs, :] * np.sqrt(
cframe.ivar[iobjs, :]) / np.sqrt(dwave)
# Save
fdict['waves'].append(cframe.wave)
fdict['s2n'].append(s2n)
fdict['fluxes'].append(sps_tab['MAG'][cframe.fibers[iobjs]])
if objtype == 'ELG':
fdict['OII'].append(
sps_tab['OIIFLUX'][cframe.fibers[iobjs]])
fdict['exptime'].append(cframe.meta['EXPTIME'])
# Return
return fdict
def main(args) :
log=get_logger()
log.info("read frame")
# read frame
frame = read_frame(args.infile)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel=read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux,model_wave,model_fibers=read_stdstar_models(args.models)
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
model_fibers = model_fibers%500
if np.any(fibermap['OBJTYPE'][model_fibers] != 'STD'):
for i in model_fibers:
log.error("inconsistency with spectrum %d, OBJTYPE='%s' in fibermap"%(i,fibermap["OBJTYPE"][i]))
sys.exit(12)
fluxcalib = compute_flux_calibration(frame, model_wave, model_flux)
# QA
if (args.qafile is not None):
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR'])
# Run
#import pdb; pdb.set_trace()
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s"%args.outfile)
def main() :
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--infile', type = str, default = None, required=True,
help = 'path of DESI exposure frame fits file')
parser.add_argument('--fiberflat', type = str, default = None,
help = 'path of DESI fiberflat fits file')
parser.add_argument('--sky', type = str, default = None,
help = 'path of DESI sky fits file')
parser.add_argument('--calib', type = str, default = None,
help = 'path of DESI calibration fits file')
parser.add_argument('--outfile', type = str, default = None, required=True,
help = 'path of DESI sky fits file')
# add calibration here when exists
args = parser.parse_args()
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
frame = read_frame(args.infile)
if args.fiberflat!=None :
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
if args.sky!=None :
log.info("subtract sky")
# read sky
skymodel=read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
if args.calib!=None :
log.info("calibrate")
# read calibration
fluxcalib=read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
# save output
write_frame(args.outfile, frame)
log.info("successfully wrote %s"%args.outfile)
def main(args):
log = get_logger()
frame=read_frame(args.infile, skip_resolution=True)
fibermap=read_fibermap(args.infile)
fiberflat=read_fiberflat(args.fiberflat)
skymodel=read_sky(args.sky)
fluxcalib=read_flux_calibration(args.calib)
cam=args.infile.split('/')[-1].split('-')[1]
band=cam[0]
bands=[band]
# Indices of sky fibers.
sky_indx = np.where(fibermap['OBJTYPE'] == 'SKY')[0]
rd_var, sky_var = calc_var(bands, args.nea, args.psf, frame, fluxcalib, fiberflat, skymodel, components=True)
var = calc_var(bands, args.nea, args.psf, frame, fluxcalib, fiberflat, skymodel, components=False)
nsky = 4
fig, axes = plt.subplots(1, nsky, figsize=(5 * nsky, 5))
for i in range(nsky):
def calc_alphavar(alpha):
return alpha * rd_var[sky_indx,:] + sky_var[sky_indx,:]
def alpha_fit(alpha):
_var = calc_alphavar(alpha)
ivar = 1. / _var
X2 = (frame.ivar[sky_indx,:] - ivar)**2.
return np.sum(X2)
res = minimize(alpha_fit, x0=[1.])
alpha = res.x[0]
indx = sky_indx[i]
axes[i].plot(skymodel.wave, median_filter(frame.ivar[indx,:], 10), lw=0.4, label='Sky frame IVAR', alpha=0.4)
axes[i].plot(skymodel.wave, 1./rd_var[indx,:], lw=0.4, label='Model rd. IVAR', alpha=0.4)
# axes[i].plot(skymodel.wave, 1./sky_var[indx,:], lw=0.4, label='Model Sky IVAR', alpha=0.4)
# axes[i].plot(skymodel.wave, 1./var[indx,:], lw=0.4, label=r'Model IVAR', alpha=0.4)
axes[i].plot(skymodel.wave, median_filter(1./calc_alphavar(alpha)[i,:], 10), lw=0.4, label=r'$\alpha$ Model IVAR', alpha=0.4)
axes[i].set_title(r'Fiber {:d} ($\alpha$ = {:.6f})'.format(indx, alpha))
axes[i].set_xlabel(r'Wavelength [$AA$]')
axes[i].set_yscale('log')
axes[i].set_ylim(bottom=5.e-4, top=3.e-2)
axes[i].legend(frameon=False, loc=2)
axes[0].set_ylabel('e/A')
pl.show()
def slurp(self, make_frameqa=False, remove=True, **kwargs):
""" Slurp all the individual QA files into one master QA file
Args:
make_frameqa: bool, optional
Regenerate the individual QA files (at the frame level first)
remove: bool, optional
Remove
Returns:
"""
from desispec.io import meta
from desispec.qa import QA_Exposure
from desispec.io import write_qa_prod
import pdb
log = get_logger()
# Remake?
if make_frameqa:
self.make_frameqa(**kwargs)
# Loop on nights
path_nights = glob.glob(self.specprod_dir + '/exposures/*')
nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights]
# Reset
log.info("Resetting qa_exps in qa_prod")
self.qa_exps = []
# Loop
for night in nights:
# Loop on exposures
for exposure in get_exposures(night,
specprod_dir=self.specprod_dir):
frames_dict = get_files(filetype=str('frame'),
night=night,
expid=exposure,
specprod_dir=self.specprod_dir)
if len(frames_dict) == 0:
continue
# Load any frame (for the type)
key = list(frames_dict.keys())[0]
frame_fil = frames_dict[key]
frame = read_frame(frame_fil)
qa_exp = QA_Exposure(exposure,
night,
frame.meta['FLAVOR'],
specprod_dir=self.specprod_dir,
remove=remove)
# Append
self.qa_exps.append(qa_exp)
# Write
outroot = self.specprod_dir + '/' + self.prod_name + '_qa'
write_qa_prod(outroot, self)
def main(args) :
log=get_logger()
log.info("starting")
# read exposure to load data and get range of spectra
frame = read_frame(args.infile)
specmin, specmax = np.min(frame.fibers), np.max(frame.fibers)
if args.cosmics_nsig>0 : # Reject cosmics
reject_cosmic_rays_1d(frame,args.cosmics_nsig)
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
# compute sky model
skymodel = compute_sky(frame,add_variance=(not args.no_extra_variance),\
angular_variation_deg=args.angular_variation_deg,\
chromatic_variation_deg=args.chromatic_variation_deg,\
adjust_wavelength=args.adjust_wavelength,\
adjust_lsf=args.adjust_lsf)
# QA
if (args.qafile is not None) or (args.qafig is not None):
log.info("performing skysub QA")
# Load
qaframe = load_qa_frame(args.qafile, frame_meta=frame.meta, flavor=frame.meta['FLAVOR'])
# Run
qaframe.run_qa('SKYSUB', (frame, skymodel))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_skyres(args.qafig, frame, skymodel, qaframe)
# record inputs
frame.meta['IN_FRAME'] = shorten_filename(args.infile)
frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat)
# write result
write_sky(args.outfile, skymodel, frame.meta)
log.info("successfully wrote %s"%args.outfile)
def main() :
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--infile', type = str, default = None, required=True,
help = 'path of DESI exposure frame fits file')
parser.add_argument('--fibermap', type = str, default = None, required=True,
help = 'path of DESI exposure frame fits file')
parser.add_argument('--fiberflat', type = str, default = None, required=True,
help = 'path of DESI fiberflat fits file')
parser.add_argument('--outfile', type = str, default = None, required=True,
help = 'path of DESI sky fits file')
args = parser.parse_args()
log=get_logger()
log.info("starting")
# read exposure to load data and get range of spectra
frame = read_frame(args.infile)
specmin=frame.header["SPECMIN"]
specmax=frame.header["SPECMAX"]
# read fibermap to locate sky fibers
fibermap = read_fibermap(args.fibermap)
selection=np.where((fibermap["OBJTYPE"]=="SKY")&(fibermap["FIBER"]>=specmin)&(fibermap["FIBER"]<=specmax))[0]
if selection.size == 0 :
log.error("no sky fiber in fibermap %s"%args.fibermap)
sys.exit(12)
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
# compute sky model
skymodel = compute_sky(frame, fibermap)
# write result
write_sky(args.outfile, skymodel, frame.header)
log.info("successfully wrote %s"%args.outfile)
def slurp(self, make_frameqa=False, remove=True, **kwargs):
""" Slurp all the individual QA files into one master QA file
Args:
make_frameqa: bool, optional
Regenerate the individual QA files (at the frame level first)
remove: bool, optional
Remove
Returns:
"""
from desispec.io import meta
from desispec.qa import QA_Exposure
from desispec.io import write_qa_prod
import pdb
# Remake?
if make_frameqa:
self.make_frameqa(**kwargs)
# Loop on nights
path_nights = glob.glob(self.specprod_dir+'/exposures/*')
nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights]
# Reset
log.info("Resetting qa_exps in qa_prod")
self.qa_exps = []
# Loop
for night in nights:
# Loop on exposures
for exposure in get_exposures(night, specprod_dir = self.specprod_dir):
frames_dict = get_files(filetype = str('frame'), night = night,
expid = exposure, specprod_dir = self.specprod_dir)
if len(frames_dict.keys()) == 0:
continue
# Load any frame (for the type)
key = frames_dict.keys()[0]
frame_fil = frames_dict[key]
frame = read_frame(frame_fil)
qa_exp = QA_Exposure(exposure, night, frame.meta['FLAVOR'],
specprod_dir=self.specprod_dir, remove=remove)
# Append
self.qa_exps.append(qa_exp)
# Write
outroot = self.specprod_dir+'/'+self.prod_name+'_qa'
write_qa_prod(outroot, self)
def main(args) :
log = get_logger()
for filename in args.infile :
log.info("reading %s"%filename)
frame=io.read_frame(filename)
flux_per_angstrom=None
if args.flux_per_angstrom :
flux_per_angstrom=True
elif args.flux_per_pixel :
flux_per_angstrom=False
else :
flux_per_angstrom=None
scores,comments=compute_and_append_frame_scores(frame,suffix=args.suffix,flux_per_angstrom=flux_per_angstrom,overwrite=args.overwrite)
log.info("Adding or replacing SCORES extention with {} in {}".format(scores.keys(),filename))
write_bintable(filename,data=scores,comments=comments,extname="SCORES",clobber=True)
def main() :
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--infile', type = str, default = None, required=True,
help = 'path of DESI frame fits file corresponding to a continuum lamp exposure')
parser.add_argument('--outfile', type = str, default = None, required=True,
help = 'path of DESI fiberflat fits file')
args = parser.parse_args()
log=get_logger()
log.info("starting")
frame = read_frame(args.infile)
fiberflat = compute_fiberflat(frame)
write_fiberflat(args.outfile, fiberflat, frame.header)
log.info("successfully wrote %s"%args.outfile)
def main(args):
log = get_logger()
log.info("starting")
# read exposure to load data and get range of spectra
frame = read_frame(args.infile)
specmin, specmax = np.min(frame.fibers), np.max(frame.fibers)
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
# compute sky model
skymodel = compute_sky(frame)
# QA
if (args.qafile is not None) or (args.qafig is not None):
log.info("performing skysub QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame,
flavor=frame.meta['FLAVOR'])
# Run
qaframe.run_qa('SKYSUB', (frame, skymodel))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_skyres(args.qafig, frame, skymodel, qaframe)
# write result
write_sky(args.outfile, skymodel, frame.meta)
log.info("successfully wrote %s" % args.outfile)
def main(args) :
log=get_logger()
log.info("starting")
# read exposure to load data and get range of spectra
frame = read_frame(args.infile)
specmin, specmax = np.min(frame.fibers), np.max(frame.fibers)
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
# compute sky model
skymodel = compute_sky(frame)
# QA
if (args.qafile is not None) or (args.qafig is not None):
log.info("performing skysub QA")
# Load
qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR'])
# Run
qaframe.run_qa('SKYSUB', (frame, skymodel))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_skyres(args.qafig, frame, skymodel, qaframe)
# write result
write_sky(args.outfile, skymodel, frame.meta)
log.info("successfully wrote %s"%args.outfile)
def main(args):
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
frame = read_frame(args.infile)
if args.fiberflat!=None :
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
if args.sky!=None :
log.info("subtract sky")
# read sky
skymodel=read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
if args.calib!=None :
log.info("calibrate")
# read calibration
fluxcalib=read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
# save output
write_frame(args.outfile, frame, units='1e-17 erg/(s cm2 A)')
log.info("successfully wrote %s"%args.outfile)
def get_sky(night,
expid,
exptime,
ftype="model",
redux="daily",
smoothing=100.0,
specsim_darksky=False,
nightly_darksky=False):
# AR ftype = "data" or "model"
# AR redux = "daily" or "blanc"
# AR if ftype = "data" : read the sky fibers from frame*fits + apply flat-field
# AR if ftype = "model": read the sky model from sky*.fits for the first fiber of each petal (see DJS email from 29Dec2020)
# AR those are in electron / angstrom
# AR to integrate over the decam-r-band, we need cameras b and r
if ftype not in ["data", "model"]:
sys.exit("ftype should be 'data' or 'model'")
sky = np.zeros(len(fullwave))
reduxdir = dailydir.replace("daily", redux)
# AR see AK email [desi-data 5218]
if redux == "blanc":
specs = ["0", "1", "3", "4", "5", "7", "8", "9"]
else:
specs = np.arange(10, dtype=int).astype(str)
for camera in ["b", "r", "z"]:
norm_cam = np.zeros(len(fullwave[cslice[camera]]))
sky_cam = np.zeros(len(fullwave[cslice[camera]]))
for spec in specs:
# AR data
if ftype == "data":
frfn = os.path.join(
reduxdir,
"exposures",
"{}".format(night),
expid,
"frame-{}{}-{}.fits".format(camera, spec, expid),
)
if not os.path.isfile(frfn):
print("Skipping non-existent {}".format(frfn))
continue
fr = read_frame(frfn, skip_resolution=True)
flfn = os.environ['DESI_SPECTRO_CALIB'] + '/' + CalibFinder(
[fr.meta]).data['FIBERFLAT']
# calib_flux [1e-17 erg/s/cm2/Angstrom] = uncalib_flux [electron/Angstrom] / (calibration_model * exptime [s])
# fl = read_fiberflat(flfn)
# No exptime correction.
# apply_fiberflat(fr, fl)
# AR cutting on sky fibers with at least one valid pixel.
ii = (fr.fibermap["OBJTYPE"]
== "SKY") & (fr.ivar.sum(axis=1) > 0)
# AR frame*fits are in e- / angstrom ; adding the N sky fibers
# sky_cam += fr.flux[ii, :].sum(axis=0)
# nspec += ii.sum()
# Ignores fiberflat corrected (fl), as includes e.g. fiberloss.
sky_cam += (fr.flux[ii, :] * fr.ivar[ii, :]).sum(axis=0)
norm_cam += fr.ivar[ii, :].sum(axis=0)
# AR model
if ftype == "model":
fn = os.path.join(
reduxdir,
"exposures",
"{}".format(night),
expid,
"sky-{}{}-{}.fits".format(camera, spec, expid),
)
if not os.path.isfile(fn):
print("Skipping non-existent {}".format(fn))
else:
print("Solving for {}".format(fn))
fd = fitsio.FITS(fn)
assert np.allclose(fullwave[cslice[camera]],
fd["WAVELENGTH"].read())
fd = fitsio.FITS(fn)
with fd as hdus:
exptime = hdus[0].read_header()['EXPTIME']
flux = hdus['SKY'].read()
ivar = hdus['IVAR'].read()
mask = hdus['MASK'].read()
# Verify that we have the expected wavelengths.
# assert np.allclose(detected[camera].wave, hdus['WAVELENGTH'].read())
# Verify that ivar is purely statistical.
# assert np.array_equal(ivar, hdus['STATIVAR'].read())
# Verify that the model has no masked pixels.
# assert np.all((mask == 0) & (ivar > 0))
# Verify that the sky model is constant.
# assert np.array_equal(np.max(ivar, axis=0), np.min(ivar, axis=0))
# assert np.allclose(np.max(flux, axis=0), np.min(flux, axis=0))
# There are actually small variations in flux!
# TODO: figure out where these variations come from.
# For now, take the median over fibers.
if fd["IVAR"][0, :][0].max() > 0:
sky_cam += fd["SKY"][0, :][
0] # AR reading the first fiber only
norm_cam += np.ones(len(fullwave[cslice[camera]]))
# sky_cam += np.median(flux, axis=0)
# norm_cam += np.ones(len(fullwave[cslice[camera]]))
fd.close()
# AR sky model flux in incident photon / angstrom / s
# if nspec > 0:
keep = norm_cam > 0
if keep.sum() > 0:
# [e/A/s] / throughput.
sky[cslice[camera]][keep] = (sky_cam[keep] / norm_cam[keep] /
exptime / spec_thru[camera][keep])
else:
print("{}-{}-{}: no spectra for {}".format(night, expid, camera,
ftype))
# AR sky model flux in erg / angstrom / s (using the photon energy in erg).
e_phot_erg = (constants.h.to(units.erg * units.s) *
constants.c.to(units.angstrom / units.s) /
(fullwave * units.angstrom))
sky *= e_phot_erg.value
# AR sky model flux in [erg / angstrom / s / cm**2 / arcsec**2].
sky /= (telap_cm2 * fiber_area_arcsec2)
if smoothing > 0.0:
sky = scipy.ndimage.gaussian_filter1d(sky, 100.)
# AR integrate over the DECam r-band
vfilter = filters.load_filters('bessell-V')
rfilter = filters.load_filters('decam2014-r')
# AR zero-padding spectrum so that it covers the DECam r-band range
sky_pad, fullwave_pad = vfilter.pad_spectrum(sky, fullwave, method="zero")
vmag = vfilter.get_ab_magnitudes(
sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom),
fullwave_pad * units.angstrom).as_array()[0][0]
# AR zero-padding spectrum so that it covers the DECam r-band range
sky_pad, fullwave_pad = rfilter.pad_spectrum(sky, fullwave, method="zero")
rmag = rfilter.get_ab_magnitudes(
sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom),
fullwave_pad * units.angstrom).as_array()[0][0]
if specsim_darksky:
fd = fitsio.FITS(fn)
# Dark Sky at given airmass (cnst. across spectrograph / camera).
simulator.atmosphere.airmass = fd['SKY'].read_header()['AIRMASS']
# Force below the horizon: dark sky.
simulator.atmosphere.moon.moon_zenith = 120. * u.deg
simulator.simulate()
# [1e-17 erg / (Angstrom arcsec2 cm2 s)].
sim_darksky = simulator.atmosphere.surface_brightness
sim_darksky *= 1.e-17
dsky_pad, dskywave_pad = rfilter.pad_spectrum(sim_darksky.value,
config.wavelength.value,
method="zero")
dsky_rmag = rfilter.get_ab_magnitudes(
dsky_pad * units.erg / (units.cm**2 * units.s * units.angstrom),
dskywave_pad * units.angstrom).as_array()[0][0]
sim_darksky = resample_flux(fullwave, config.wavelength.value,
sim_darksky.value)
sim_darksky *= u.erg / (u.cm**2 * u.s * u.angstrom * u.arcsec**2)
sky = np.clip(sky - sim_darksky.value, a_min=0.0, a_max=None)
# AR zero-padding spectrum so that it covers the DECam r-band range
sky_pad, fullwave_pad = vfilter.pad_spectrum(sky,
fullwave,
method="zero")
vmag_nodark = vfilter.get_ab_magnitudes(
sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom),
fullwave_pad * units.angstrom).as_array()[0][0]
# AR zero-padding spectrum so that it covers the DECam r-band range
sky_pad, fullwave_pad = rfilter.pad_spectrum(sky,
fullwave,
method="zero")
rmag_nodark = rfilter.get_ab_magnitudes(
sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom),
fullwave_pad * units.angstrom).as_array()[0][0]
return fullwave, sky, vmag, rmag, vmag_nodark, rmag_nodark
elif nightly_darksky:
from pkg_resources import resource_filename
if night in nightly_dsky_cache.keys():
return nightly_dsky_cache[night]
gfa_info = resource_filename('bgs-cmxsv', 'dat/sv1-exposures.fits')
gfa_info = Table.read(gfa_info)
bright_cut = 20.50
good_conds = (gfa_info['GFA_TRANSPARENCY_MED'] >
0.95) & (gfa_info['GFA_SKY_MAG_AB_MED'] >= bright_cut)
good_conds = gfa_info[good_conds]
expids = np.array([
np.int(x.split('/')[-1]) for x in glob.glob(
os.path.join(reduxdir, "exposures", "{}/00*".format(night)))
])
isgood = np.isin(good_conds['EXPID'], expids)
if np.any(isgood):
good_conds = good_conds[isgood]
best = good_conds['GFA_SKY_MAG_AB_MED'] == good_conds[
'GFA_SKY_MAG_AB_MED'].max()
print('Nightly Dark GFA r-mag for {}: {}'.format(
night, good_conds['GFA_SKY_MAG_AB_MED'].max()))
best_expid = good_conds[best]['EXPID'][0]
best_expid = '{:08d}'.format(best_expid)
darkwave, darksky, dark_vmag, dark_rmag = get_sky(
night,
best_expid,
exptime,
ftype="model",
redux="daily",
smoothing=0.0,
specsim_darksky=False,
nightly_darksky=False)
sky = np.clip(sky - darksky, a_min=0.0, a_max=None)
# AR zero-padding spectrum so that it covers the DECam r-band range
sky_pad, fullwave_pad = vfilter.pad_spectrum(sky,
fullwave,
method="zero")
vmag_nodark = vfilter.get_ab_magnitudes(
sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom),
fullwave_pad * units.angstrom).as_array()[0][0]
# AR zero-padding spectrum so that it covers the DECam r-band range
sky_pad, fullwave_pad = rfilter.pad_spectrum(sky,
fullwave,
method="zero")
rmag_nodark = rfilter.get_ab_magnitudes(
sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom),
fullwave_pad * units.angstrom).as_array()[0][0]
nightly_dsky_cache[
night] = fullwave, sky, vmag, rmag, vmag_nodark, rmag_nodark
else:
print(
'No nightly darksky available for night: {}. Defaulting to specsim.'
.format(night))
nightly_dsky_cache[night] = get_sky(night,
expid,
exptime,
ftype="model",
redux="daily",
smoothing=0.0,
specsim_darksky=True,
nightly_darksky=False)
return nightly_dsky_cache[night]
else:
pass
return fullwave, sky, vmag, rmag
def main() :
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--infile', type = str, default = None, required=True,
help = 'path of DESI exposure frame fits file')
parser.add_argument('--fibermap', type = str, default = None, required=True,
help = 'path of DESI exposure frame fits file')
parser.add_argument('--fiberflat', type = str, default = None, required=True,
help = 'path of DESI fiberflat fits file')
parser.add_argument('--sky', type = str, default = None, required=True,
help = 'path of DESI sky fits file')
parser.add_argument('--models', type = str, default = None, required=True,
help = 'path of spetro-photometric stellar spectra fits file')
parser.add_argument('--outfile', type = str, default = None, required=True,
help = 'path of DESI flux calbration fits file')
args = parser.parse_args()
log=get_logger()
log.info("read frame")
# read frame
frame = read_frame(args.infile)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel=read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux,model_wave,model_fibers=read_stdstar_models(args.models)
# select fibers
SPECMIN=frame.header["SPECMIN"]
SPECMAX=frame.header["SPECMAX"]
selec=np.where((model_fibers>=SPECMIN)&(model_fibers<=SPECMAX))[0]
if selec.size == 0 :
log.error("no stellar models for this spectro")
sys.exit(12)
fibers=model_fibers[selec]-frame.header["SPECMIN"]
log.info("star fibers= %s"%str(fibers))
table = read_fibermap(args.fibermap)
bad=np.where(table["OBJTYPE"][fibers]!="STD")[0]
if bad.size > 0 :
for fiber in fibers[bad] :
log.error("inconsistency with fiber %d, OBJTYPE='%s' in fibermap"%(fiber,table["OBJTYPE"][fiber]))
sys.exit(12)
fluxcalib = compute_flux_calibration(frame, fibers, model_wave, model_flux)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.header)
log.info("successfully wrote %s"%args.outfile)
def main(args):
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
log = get_logger()
log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
(args.color, args.delta_color))
frames = {}
flats = {}
skies = {}
spectrograph = None
starfibers = None
starindices = None
fibermap = None
# READ DATA
############################################
for filename in args.frames:
log.info("reading %s" % filename)
frame = io.read_frame(filename)
header = fits.getheader(filename, 0)
frame_fibermap = frame.fibermap
frame_starindices = np.where(isStdStar(frame_fibermap))[0]
#- Confirm that all fluxes have entries but trust targeting bits
#- to get basic magnitude range correct
keep = np.ones(len(frame_starindices), dtype=bool)
for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']: #- and W1 and W2?
keep &= frame_fibermap[colname][frame_starindices] > 10**(
(22.5 - 30) / 2.5)
keep &= frame_fibermap[colname][frame_starindices] < 10**(
(22.5 - 0) / 2.5)
frame_starindices = frame_starindices[keep]
camera = safe_read_key(header, "CAMERA").strip().lower()
if spectrograph is None:
spectrograph = frame.spectrograph
fibermap = frame_fibermap
starindices = frame_starindices
starfibers = fibermap["FIBER"][starindices]
elif spectrograph != frame.spectrograph:
log.error("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
raise ValueError("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
elif starindices.size != frame_starindices.size or np.sum(
starindices != frame_starindices) > 0:
log.error("incompatible fibermap")
raise ValueError("incompatible fibermap")
if not camera in frames:
frames[camera] = []
frames[camera].append(frame)
for filename in args.skymodels:
log.info("reading %s" % filename)
sky = io.read_sky(filename)
header = fits.getheader(filename, 0)
camera = safe_read_key(header, "CAMERA").strip().lower()
if not camera in skies:
skies[camera] = []
skies[camera].append(sky)
for filename in args.fiberflats:
log.info("reading %s" % filename)
header = fits.getheader(filename, 0)
flat = io.read_fiberflat(filename)
camera = safe_read_key(header, "CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if camera in flats:
log.warning(
"cannot handle several flats of same camera (%s), will use only the first one"
% camera)
#raise ValueError("cannot handle several flats of same camera (%s)"%camera)
else:
flats[camera] = flat
if starindices.size == 0:
log.error("no STD star found in fibermap")
raise ValueError("no STD star found in fibermap")
log.info("found %d STD stars" % starindices.size)
log.warning("Not using flux errors for Standard Star fits!")
# DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
############################################
for cam in frames:
if not cam in skies:
log.warning("Missing sky for %s" % cam)
frames.pop(cam)
continue
if not cam in flats:
log.warning("Missing flat for %s" % cam)
frames.pop(cam)
continue
flat = flats[cam]
for frame, sky in zip(frames[cam], skies[cam]):
frame.flux = frame.flux[starindices]
frame.ivar = frame.ivar[starindices]
frame.ivar *= (frame.mask[starindices] == 0)
frame.ivar *= (sky.ivar[starindices] != 0)
frame.ivar *= (sky.mask[starindices] == 0)
frame.ivar *= (flat.ivar[starindices] != 0)
frame.ivar *= (flat.mask[starindices] == 0)
frame.flux *= (frame.ivar > 0) # just for clean plots
for star in range(frame.flux.shape[0]):
ok = np.where((frame.ivar[star] > 0)
& (flat.fiberflat[star] != 0))[0]
if ok.size > 0:
frame.flux[star] = frame.flux[star] / flat.fiberflat[
star] - sky.flux[star]
frame.resolution_data = frame.resolution_data[starindices]
nstars = starindices.size
fibermap = Table(fibermap[starindices])
# READ MODELS
############################################
log.info("reading star models in %s" % args.starmodels)
stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
args.starmodels)
# COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
############################################
#- Support older fibermaps
if 'PHOTSYS' not in fibermap.colnames:
log.warning('Old fibermap format; using defaults for missing columns')
log.warning(" PHOTSYS = 'S'")
log.warning(" MW_TRANSMISSION_G/R/Z = 1.0")
log.warning(" EBV = 0.0")
fibermap['PHOTSYS'] = 'S'
fibermap['MW_TRANSMISSION_G'] = 1.0
fibermap['MW_TRANSMISSION_R'] = 1.0
fibermap['MW_TRANSMISSION_Z'] = 1.0
fibermap['EBV'] = 0.0
model_filters = dict()
if 'S' in fibermap['PHOTSYS']:
for filter_name in ['DECAM_G', 'DECAM_R', 'DECAM_Z']:
model_filters[filter_name] = load_filter(filter_name)
if 'N' in fibermap['PHOTSYS']:
for filter_name in ['BASS_G', 'BASS_R', 'MZLS_Z']:
model_filters[filter_name] = load_filter(filter_name)
if len(model_filters) == 0:
raise ValueError("No filters loaded; neither 'N' nor 'S' in PHOTSYS?")
log.info("computing model mags for %s" % sorted(model_filters.keys()))
model_mags = dict()
fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
for filter_name, filter_response in model_filters.items():
model_mags[filter_name] = filter_response.get_ab_magnitude(
stdflux * fluxunits, stdwave)
log.info("done computing model mags")
# LOOP ON STARS TO FIND BEST MODEL
############################################
linear_coefficients = np.zeros((nstars, stdflux.shape[0]))
chi2dof = np.zeros((nstars))
redshift = np.zeros((nstars))
normflux = []
star_mags = dict()
star_unextincted_mags = dict()
for band in ['G', 'R', 'Z']:
star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_' + band])
star_unextincted_mags[band] = 22.5 - 2.5 * np.log10(
fibermap['FLUX_' + band] / fibermap['MW_TRANSMISSION_' + band])
star_colors = dict()
star_colors['G-R'] = star_mags['G'] - star_mags['R']
star_colors['R-Z'] = star_mags['R'] - star_mags['Z']
star_unextincted_colors = dict()
star_unextincted_colors[
'G-R'] = star_unextincted_mags['G'] - star_unextincted_mags['R']
star_unextincted_colors[
'R-Z'] = star_unextincted_mags['R'] - star_unextincted_mags['Z']
fitted_model_colors = np.zeros(nstars)
for star in range(nstars):
log.info("finding best model for observed star #%d" % star)
# np.array of wave,flux,ivar,resol
wave = {}
flux = {}
ivar = {}
resolution_data = {}
for camera in frames:
for i, frame in enumerate(frames[camera]):
identifier = "%s-%d" % (camera, i)
wave[identifier] = frame.wave
flux[identifier] = frame.flux[star]
ivar[identifier] = frame.ivar[star]
resolution_data[identifier] = frame.resolution_data[star]
# preselect models based on magnitudes
if fibermap['PHOTSYS'][star] == 'N':
if args.color == 'G-R':
model_colors = model_mags['BASS_G'] - model_mags['BASS_R']
elif args.color == 'R-Z':
model_colors = model_mags['BASS_R'] - model_mags['MZLS_Z']
else:
raise ValueError('Unknown color {}'.format(args.color))
else:
if args.color == 'G-R':
model_colors = model_mags['DECAM_G'] - model_mags['DECAM_R']
elif args.color == 'R-Z':
model_colors = model_mags['DECAM_R'] - model_mags['DECAM_Z']
else:
raise ValueError('Unknown color {}'.format(args.color))
color_diff = model_colors - star_unextincted_colors[args.color][star]
selection = np.abs(color_diff) < args.delta_color
# smallest cube in parameter space including this selection (needed for interpolation)
new_selection = (teff >= np.min(teff[selection])) & (teff <= np.max(
teff[selection]))
new_selection &= (logg >= np.min(logg[selection])) & (logg <= np.max(
logg[selection]))
new_selection &= (feh >= np.min(feh[selection])) & (feh <= np.max(
feh[selection]))
selection = np.where(new_selection)[0]
log.info(
"star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d"
% (star, starfibers[star], args.color,
star_unextincted_colors[args.color][star], selection.size,
stdflux.shape[0]))
# Match unextincted standard stars to data
coefficients, redshift[star], chi2dof[star] = match_templates(
wave,
flux,
ivar,
resolution_data,
stdwave,
stdflux[selection],
teff[selection],
logg[selection],
feh[selection],
ncpu=args.ncpu,
z_max=args.z_max,
z_res=args.z_res,
template_error=args.template_error)
linear_coefficients[star, selection] = coefficients
log.info(
'Star Fiber: {0}; TEFF: {1}; LOGG: {2}; FEH: {3}; Redshift: {4}; Chisq/dof: {5}'
.format(starfibers[star], np.inner(teff,
linear_coefficients[star]),
np.inner(logg, linear_coefficients[star]),
np.inner(feh, linear_coefficients[star]), redshift[star],
chi2dof[star]))
# Apply redshift to original spectrum at full resolution
model = np.zeros(stdwave.size)
redshifted_stdwave = stdwave * (1 + redshift[star])
for i, c in enumerate(linear_coefficients[star]):
if c != 0:
model += c * np.interp(stdwave, redshifted_stdwave, stdflux[i])
# Apply dust extinction to the model
model *= dust_transmission(stdwave, fibermap['EBV'][star])
# Compute final color of dust-extincted model
if fibermap['PHOTSYS'][star] == 'N':
if args.color == 'G-R':
model_mag1 = model_filters['BASS_G'].get_ab_magnitude(
model * fluxunits, stdwave)
model_mag2 = model_filters['BASS_R'].get_ab_magnitude(
model * fluxunits, stdwave)
model_magr = model_mag2
elif args.color == 'R-Z':
model_mag1 = model_filters['BASS_R'].get_ab_magnitude(
model * fluxunits, stdwave)
model_mag2 = model_filters['MZLS_Z'].get_ab_magnitude(
model * fluxunits, stdwave)
model_magr = model_mag1
else:
raise ValueError('Unknown color {}'.format(args.color))
else:
if args.color == 'G-R':
model_mag1 = model_filters['DECAM_G'].get_ab_magnitude(
model * fluxunits, stdwave)
model_mag2 = model_filters['DECAM_R'].get_ab_magnitude(
model * fluxunits, stdwave)
model_magr = model_mag2
elif args.color == 'R-Z':
model_mag1 = model_filters['DECAM_R'].get_ab_magnitude(
model * fluxunits, stdwave)
model_mag2 = model_filters['DECAM_Z'].get_ab_magnitude(
model * fluxunits, stdwave)
model_magr = model_mag1
else:
raise ValueError('Unknown color {}'.format(args.color))
fitted_model_colors[star] = model_mag1 - model_mag2
#- TODO: move this back into normalize_templates, at the cost of
#- recalculating a model magnitude?
# Normalize the best model using reported magnitude
scalefac = 10**((model_magr - star_mags['R'][star]) / 2.5)
log.info('scaling R mag {} to {} using scale {}'.format(
model_magr, star_mags['R'][star], scalefac))
normflux.append(model * scalefac)
# Now write the normalized flux for all best models to a file
normflux = np.array(normflux)
data = {}
data['LOGG'] = linear_coefficients.dot(logg)
data['TEFF'] = linear_coefficients.dot(teff)
data['FEH'] = linear_coefficients.dot(feh)
data['CHI2DOF'] = chi2dof
data['REDSHIFT'] = redshift
data['COEFF'] = linear_coefficients
data['DATA_%s' % args.color] = star_colors[args.color]
data['MODEL_%s' % args.color] = fitted_model_colors
io.write_stdstar_models(args.outfile, normflux, stdwave, starfibers, data)
def main(args):
log = get_logger()
cmd = [
'desi_compute_fluxcalibration',
]
for key, value in args.__dict__.items():
if value is not None:
cmd += ['--' + key, str(value)]
cmd = ' '.join(cmd)
log.info(cmd)
log.info("read frame")
# read frame
frame = read_frame(args.infile)
# Set fibermask flagged spectra to have 0 flux and variance
frame = get_fiberbitmasked_frame(frame,
bitmask='flux',
ivar_framemask=True)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel = read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models(
args.models)
ok = np.ones(len(model_metadata), dtype=bool)
if args.chi2cut > 0:
log.info("Apply cut CHI2DOF<{}".format(args.chi2cut))
ok &= (model_metadata["CHI2DOF"] < args.chi2cut)
if args.delta_color_cut > 0:
log.info("Apply cut |delta color|<{}".format(args.delta_color_cut))
ok &= (np.abs(model_metadata["MODEL_G-R"] - model_metadata["DATA_G-R"])
< args.delta_color_cut)
if args.min_color is not None:
log.info("Apply cut DATA_G-R>{}".format(args.min_color))
ok &= (model_metadata["DATA_G-R"] > args.min_color)
if args.chi2cut_nsig > 0:
# automatically reject stars that ar chi2 outliers
mchi2 = np.median(model_metadata["CHI2DOF"])
rmschi2 = np.std(model_metadata["CHI2DOF"])
maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2
log.info("Apply cut CHI2DOF<{} based on chi2cut_nsig={}".format(
maxchi2, args.chi2cut_nsig))
ok &= (model_metadata["CHI2DOF"] <= maxchi2)
ok = np.where(ok)[0]
if ok.size == 0:
log.error("cuts discarded all stars")
sys.exit(12)
nstars = model_flux.shape[0]
nbad = nstars - ok.size
if nbad > 0:
log.warning("discarding %d star(s) out of %d because of cuts" %
(nbad, nstars))
model_flux = model_flux[ok]
model_fibers = model_fibers[ok]
model_metadata = model_metadata[:][ok]
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
## check whether star fibers from args.models are consistent with fibers from fibermap
## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
fibermap_std_indices = np.where(isStdStar(fibermap))[0]
if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)):
target_colnames, target_masks, survey = main_cmx_or_sv(fibermap)
colname = target_colnames[0]
for i in model_fibers % 500:
log.error(
"inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap"
.format(i, fibermap["OBJTYPE"][i], colname,
fibermap[colname][i]))
sys.exit(12)
# Make sure the fibers of interest aren't entirely masked.
if np.sum(
np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) ==
frame.nwave) == len(model_fibers):
log.warning('All standard-star spectra are masked!')
return
fluxcalib = compute_flux_calibration(
frame,
model_wave,
model_flux,
model_fibers % 500,
highest_throughput_nstars=args.highest_throughput)
# QA
if (args.qafile is not None):
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame_meta=frame.meta,
flavor=frame.meta['FLAVOR'])
# Run
#import pdb; pdb.set_trace()
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s" % args.outfile)
def main(args):
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
log = get_logger()
log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
(args.color, args.delta_color))
frames = {}
flats = {}
skies = {}
spectrograph = None
starfibers = None
starindices = None
fibermap = None
# READ DATA
############################################
for filename in args.frames:
log.info("reading %s" % filename)
frame = io.read_frame(filename)
header = fits.getheader(filename, 0)
frame_fibermap = frame.fibermap
frame_starindices = np.where(frame_fibermap["OBJTYPE"] == "STD")[0]
# check magnitude are well defined or discard stars
tmp = []
for i in frame_starindices:
mags = frame_fibermap["MAG"][i]
ok = np.sum((mags > 0) & (mags < 30))
if np.sum((mags > 0) & (mags < 30)) == mags.size:
tmp.append(i)
frame_starindices = np.array(tmp).astype(int)
camera = safe_read_key(header, "CAMERA").strip().lower()
if spectrograph is None:
spectrograph = frame.spectrograph
fibermap = frame_fibermap
starindices = frame_starindices
starfibers = fibermap["FIBER"][starindices]
elif spectrograph != frame.spectrograph:
log.error("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
raise ValueError("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
elif starindices.size != frame_starindices.size or np.sum(
starindices != frame_starindices) > 0:
log.error("incompatible fibermap")
raise ValueError("incompatible fibermap")
if not camera in frames:
frames[camera] = []
frames[camera].append(frame)
for filename in args.skymodels:
log.info("reading %s" % filename)
sky = io.read_sky(filename)
header = fits.getheader(filename, 0)
camera = safe_read_key(header, "CAMERA").strip().lower()
if not camera in skies:
skies[camera] = []
skies[camera].append(sky)
for filename in args.fiberflats:
log.info("reading %s" % filename)
header = fits.getheader(filename, 0)
flat = io.read_fiberflat(filename)
camera = safe_read_key(header, "CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if camera in flats:
log.warning(
"cannot handle several flats of same camera (%s), will use only the first one"
% camera)
#raise ValueError("cannot handle several flats of same camera (%s)"%camera)
else:
flats[camera] = flat
if starindices.size == 0:
log.error("no STD star found in fibermap")
raise ValueError("no STD star found in fibermap")
log.info("found %d STD stars" % starindices.size)
imaging_filters = fibermap["FILTER"][starindices]
imaging_mags = fibermap["MAG"][starindices]
log.warning(
"NO MAG ERRORS IN FIBERMAP, I AM IGNORING MEASUREMENT ERRORS !!")
ebv = np.zeros(starindices.size)
if "SFD_EBV" in fibermap.columns.names:
log.info("Using 'SFD_EBV' from fibermap")
ebv = fibermap["SFD_EBV"][starindices]
else:
log.warning("NO EXTINCTION VALUES IN FIBERMAP!!")
# DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
############################################
for cam in frames:
if not cam in skies:
log.warning("Missing sky for %s" % cam)
frames.pop(cam)
continue
if not cam in flats:
log.warning("Missing flat for %s" % cam)
frames.pop(cam)
continue
flat = flats[cam]
for frame, sky in zip(frames[cam], skies[cam]):
frame.flux = frame.flux[starindices]
frame.ivar = frame.ivar[starindices]
frame.ivar *= (frame.mask[starindices] == 0)
frame.ivar *= (sky.ivar[starindices] != 0)
frame.ivar *= (sky.mask[starindices] == 0)
frame.ivar *= (flat.ivar[starindices] != 0)
frame.ivar *= (flat.mask[starindices] == 0)
frame.flux *= (frame.ivar > 0) # just for clean plots
for star in range(frame.flux.shape[0]):
ok = np.where((frame.ivar[star] > 0)
& (flat.fiberflat[star] != 0))[0]
if ok.size > 0:
frame.flux[star] = frame.flux[star] / flat.fiberflat[
star] - sky.flux[star]
frame.resolution_data = frame.resolution_data[starindices]
nstars = starindices.size
starindices = None # we don't need this anymore
# READ MODELS
############################################
log.info("reading star models in %s" % args.starmodels)
stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
args.starmodels)
# COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
############################################
model_filters = []
for tmp in np.unique(imaging_filters):
if len(tmp) > 0: # can be one empty entry
model_filters.append(tmp)
log.info("computing model mags %s" % model_filters)
model_mags = np.zeros((stdflux.shape[0], len(model_filters)))
fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
for index in range(len(model_filters)):
if model_filters[index].startswith('WISE'):
log.warning('not computing stdstar {} mags'.format(
model_filters[index]))
continue
filter_response = load_filter(model_filters[index])
for m in range(stdflux.shape[0]):
model_mags[m, index] = filter_response.get_ab_magnitude(
stdflux[m] * fluxunits, stdwave)
log.info("done computing model mags")
mean_extinction_delta_mags = None
mean_ebv = np.mean(ebv)
if mean_ebv > 0:
log.info(
"Compute a mean delta_color from average E(B-V) = %3.2f based on canonial model star"
% mean_ebv)
# compute a mean delta_color from mean_ebv based on canonial model star
#######################################################################
# will then use this color offset in the model pre-selection
# find canonical f-type model: Teff=6000, logg=4, Fe/H=-1.5
canonical_model = np.argmin((teff - 6000.0)**2 + (logg - 4.0)**2 +
(feh + 1.5)**2)
canonical_model_mags_without_extinction = model_mags[canonical_model]
canonical_model_mags_with_extinction = np.zeros(
canonical_model_mags_without_extinction.shape)
canonical_model_reddened_flux = stdflux[
canonical_model] * dust_transmission(stdwave, mean_ebv)
for index in range(len(model_filters)):
if model_filters[index].startswith('WISE'):
log.warning('not computing stdstar {} mags'.format(
model_filters[index]))
continue
filter_response = load_filter(model_filters[index])
canonical_model_mags_with_extinction[
index] = filter_response.get_ab_magnitude(
canonical_model_reddened_flux * fluxunits, stdwave)
mean_extinction_delta_mags = canonical_model_mags_with_extinction - canonical_model_mags_without_extinction
# LOOP ON STARS TO FIND BEST MODEL
############################################
linear_coefficients = np.zeros((nstars, stdflux.shape[0]))
chi2dof = np.zeros((nstars))
redshift = np.zeros((nstars))
normflux = []
star_colors_array = np.zeros((nstars))
model_colors_array = np.zeros((nstars))
for star in range(nstars):
log.info("finding best model for observed star #%d" % star)
# np.array of wave,flux,ivar,resol
wave = {}
flux = {}
ivar = {}
resolution_data = {}
for camera in frames:
for i, frame in enumerate(frames[camera]):
identifier = "%s-%d" % (camera, i)
wave[identifier] = frame.wave
flux[identifier] = frame.flux[star]
ivar[identifier] = frame.ivar[star]
resolution_data[identifier] = frame.resolution_data[star]
# preselec models based on magnitudes
# compute star color
index1, index2 = get_color_filter_indices(imaging_filters[star],
args.color)
if index1 < 0 or index2 < 0:
log.error("cannot compute '%s' color from %s" %
(color_name, filters))
filter1 = imaging_filters[star][index1]
filter2 = imaging_filters[star][index2]
star_color = imaging_mags[star][index1] - imaging_mags[star][index2]
star_colors_array[star] = star_color
# compute models color
model_index1 = -1
model_index2 = -1
for i, fname in enumerate(model_filters):
if fname == filter1:
model_index1 = i
elif fname == filter2:
model_index2 = i
if model_index1 < 0 or model_index2 < 0:
log.error("cannot compute '%s' model color from %s" %
(color_name, filters))
model_colors = model_mags[:, model_index1] - model_mags[:,
model_index2]
# apply extinction here
# use the colors derived from the cannonical model with the mean ebv of the stars
# and simply apply a scaling factor based on the ebv of this star
# this is sufficiently precise for the broad model pre-selection we are doing here
# the exact reddening of the star to each pre-selected model is
# apply afterwards
if mean_extinction_delta_mags is not None and mean_ebv != 0:
delta_color = (mean_extinction_delta_mags[model_index1] -
mean_extinction_delta_mags[model_index2]
) * ebv[star] / mean_ebv
model_colors += delta_color
log.info(
"Apply a %s-%s color offset = %4.3f to the models for star with E(B-V)=%4.3f"
% (model_filters[model_index1], model_filters[model_index2],
delta_color, ebv[star]))
# selection
selection = np.abs(model_colors - star_color) < args.delta_color
# smallest cube in parameter space including this selection (needed for interpolation)
new_selection = (teff >= np.min(teff[selection])) & (teff <= np.max(
teff[selection]))
new_selection &= (logg >= np.min(logg[selection])) & (logg <= np.max(
logg[selection]))
new_selection &= (feh >= np.min(feh[selection])) & (feh <= np.max(
feh[selection]))
selection = np.where(new_selection)[0]
log.info(
"star#%d fiber #%d, %s = %s-%s = %f, number of pre-selected models = %d/%d"
% (star, starfibers[star], args.color, filter1, filter2,
star_color, selection.size, stdflux.shape[0]))
# apply extinction to selected_models
dust_transmission_of_this_star = dust_transmission(stdwave, ebv[star])
selected_reddened_stdflux = stdflux[
selection] * dust_transmission_of_this_star
coefficients, redshift[star], chi2dof[star] = match_templates(
wave,
flux,
ivar,
resolution_data,
stdwave,
selected_reddened_stdflux,
teff[selection],
logg[selection],
feh[selection],
ncpu=args.ncpu,
z_max=args.z_max,
z_res=args.z_res,
template_error=args.template_error)
linear_coefficients[star, selection] = coefficients
log.info(
'Star Fiber: {0}; TEFF: {1}; LOGG: {2}; FEH: {3}; Redshift: {4}; Chisq/dof: {5}'
.format(starfibers[star], np.inner(teff,
linear_coefficients[star]),
np.inner(logg, linear_coefficients[star]),
np.inner(feh, linear_coefficients[star]), redshift[star],
chi2dof[star]))
# Apply redshift to original spectrum at full resolution
model = np.zeros(stdwave.size)
for i, c in enumerate(linear_coefficients[star]):
if c != 0:
model += c * np.interp(stdwave, stdwave *
(1 + redshift[star]), stdflux[i])
# Apply dust extinction
model *= dust_transmission_of_this_star
# Compute final model color
mag1 = load_filter(model_filters[model_index1]).get_ab_magnitude(
model * fluxunits, stdwave)
mag2 = load_filter(model_filters[model_index2]).get_ab_magnitude(
model * fluxunits, stdwave)
model_colors_array[star] = mag1 - mag2
# Normalize the best model using reported magnitude
normalizedflux = normalize_templates(stdwave, model,
imaging_mags[star],
imaging_filters[star])
normflux.append(normalizedflux)
# Now write the normalized flux for all best models to a file
normflux = np.array(normflux)
data = {}
data['LOGG'] = linear_coefficients.dot(logg)
data['TEFF'] = linear_coefficients.dot(teff)
data['FEH'] = linear_coefficients.dot(feh)
data['CHI2DOF'] = chi2dof
data['REDSHIFT'] = redshift
data['COEFF'] = linear_coefficients
data['DATA_%s' % args.color] = star_colors_array
data['MODEL_%s' % args.color] = model_colors_array
norm_model_file = args.outfile
io.write_stdstar_models(args.outfile, normflux, stdwave, starfibers, data)
def make_frameqa(self, make_plots=False, clobber=True):
""" Work through the Production and make QA for all frames
Parameters:
make_plots: bool, optional
Remake the plots too?
clobber: bool, optional
Returns:
"""
# imports
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.fiberflat import read_fiberflat
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots
from desispec.io.fluxcalibration import read_stdstar_models
# Loop on nights
path_nights = glob.glob(self.specprod_dir+'/exposures/*')
nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights]
for night in nights:
for exposure in get_exposures(night, specprod_dir = self.specprod_dir):
# Object only??
frames_dict = get_files(filetype = str('frame'), night = night,
expid = exposure, specprod_dir = self.specprod_dir)
for camera,frame_fil in frames_dict.items():
# Load frame
frame = read_frame(frame_fil)
spectro = int(frame.meta['CAMERA'][-1])
if frame.meta['FLAVOR'] in ['flat','arc']:
qatype = 'qa_calib'
else:
qatype = 'qa_data'
qafile = meta.findfile(qatype, night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
if (not clobber) & os.path.isfile(qafile):
log.info("qafile={:s} exists. Not over-writing. Consider clobber=True".format(qafile))
continue
# Load
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame.meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
fiberflat = read_fiberflat(fiberflat_fil)
qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber)
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
skymodel = read_sky(sky_fil)
qaframe.run_qa('SKYSUB', (frame, skymodel))
if make_plots:
qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir,
spectrograph=spectro)
model_tuple=read_stdstar_models(stdstar_fil)
flux_fil = meta.findfile('calib', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
fluxcalib = read_flux_calibration(flux_fil)
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib, model_tuple))#, indiv_stars))
if make_plots:
qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib, model_tuple)
# Write
write_qa_frame(qafile, qaframe)
ii = np.where((table["PETAL"] == 0) & (table["FIBER"] >= 0))[0]
xk = "OFFSET_X"
yk = "OFFSET_Y"
fiberid = "FIBER"
dico = {}
for i in ii:
dico[table[fiberid][i]] = i
expid_arr = ["00033801"] #["00020206","00020207"]
flux_b3_all = []
flux_r3_all = []
flux_z3_all = []
for i in range(len(expid_arr)):
expid = expid_arr[i]
frame_b3 = read_frame("flatfielded-frame-b3-" + expid +
"-twilightcorr.fits")
flux_b3 = np.median(frame_b3.flux, axis=1)
flux_b3 /= np.median(flux_b3)
frame_r3 = read_frame("flatfielded-frame-r3-" + expid +
"-twilightcorr.fits")
flux_r3 = np.median(frame_r3.flux, axis=1)
flux_r3 /= np.median(flux_r3)
frame_z3 = read_frame("flatfielded-frame-z3-" + expid +
"-twilightcorr.fits")
flux_z3 = np.median(frame_z3.flux, axis=1)
flux_z3 /= np.median(flux_z3)
flux_b3_all.append(flux_b3)
flux_r3_all.append(flux_r3)
def get_skyres(cframes, sub_sky=False, flatten=True):
"""
Args:
cframes: str or list
Single cframe or a list of them
sub_sky: bool, optional
Subtract the sky? This should probably not be done
flatten: bool, optional
Return a flat, 1D array for each variable
combine: bool, optional
combine the individual sky fibers? Median 'smash'
Returns:
wave : ndarray
flux : ndarray
res : ndarray
ivar : ndarray
"""
from desispec.io import read_frame
from desispec.io.sky import read_sky
from desispec.sky import subtract_sky
if isinstance(cframes,list):
all_wave, all_flux, all_res, all_ivar = [], [], [], []
for cframe_file in cframes:
wave, flux, res, ivar = get_skyres(cframe_file, flatten=flatten)
# Save
all_wave.append(wave)
all_flux.append(flux)
all_res.append(res)
all_ivar.append(ivar)
# Concatenate -- Shape is preserved (nfibers, npix)
twave = np.concatenate(all_wave)
tflux = np.concatenate(all_flux)
tres = np.concatenate(all_res)
tivar = np.concatenate(all_ivar)
# Return
return twave, tflux, tres, tivar
cframe = read_frame(cframes, skip_resolution=True)
if cframe.meta['FLAVOR'] in ['flat','arc']:
raise ValueError("Bad flavor for exposure: {:s}".format(cframes))
# Sky
sky_file = cframes.replace('cframe', 'sky')
skymodel = read_sky(sky_file)
if sub_sky:
subtract_sky(cframe, skymodel)
# Resid
skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0]
res = cframe.flux[skyfibers] # Flux calibrated
ivar = cframe.ivar[skyfibers] # Flux calibrated
flux = skymodel.flux[skyfibers] # Residuals; not flux calibrated!
wave = np.outer(np.ones(flux.shape[0]), cframe.wave)
# Combine?
'''
if combine:
res = np.median(res, axis=0)
ivar = np.median(ivar, axis=0)
flux = np.median(flux, axis=0)
wave = np.median(wave, axis=0)
'''
# Return
if flatten:
return wave.flatten(), flux.flatten(), res.flatten(), ivar.flatten()
else:
return wave, flux, res, ivar
def main(args) :
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
log = get_logger()
log.info("mag delta %s = %f (for the pre-selection of stellar models)"%(args.color,args.delta_color))
frames={}
flats={}
skies={}
spectrograph=None
starfibers=None
starindices=None
fibermap=None
# READ DATA
############################################
for filename in args.frames :
log.info("reading %s"%filename)
frame=io.read_frame(filename)
header=fits.getheader(filename, 0)
frame_fibermap = frame.fibermap
frame_starindices = np.where(isStdStar(frame_fibermap['DESI_TARGET']))[0]
#- Confirm that all fluxes have entries but trust targeting bits
#- to get basic magnitude range correct
keep = np.ones(len(frame_starindices), dtype=bool)
for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']: #- and W1 and W2?
keep &= frame_fibermap[colname][frame_starindices] > 10**((22.5-30)/2.5)
keep &= frame_fibermap[colname][frame_starindices] < 10**((22.5-0)/2.5)
frame_starindices = frame_starindices[keep]
camera=safe_read_key(header,"CAMERA").strip().lower()
if spectrograph is None :
spectrograph = frame.spectrograph
fibermap = frame_fibermap
starindices=frame_starindices
starfibers=fibermap["FIBER"][starindices]
elif spectrograph != frame.spectrograph :
log.error("incompatible spectrographs %d != %d"%(spectrograph,frame.spectrograph))
raise ValueError("incompatible spectrographs %d != %d"%(spectrograph,frame.spectrograph))
elif starindices.size != frame_starindices.size or np.sum(starindices!=frame_starindices)>0 :
log.error("incompatible fibermap")
raise ValueError("incompatible fibermap")
if not camera in frames :
frames[camera]=[]
frames[camera].append(frame)
for filename in args.skymodels :
log.info("reading %s"%filename)
sky=io.read_sky(filename)
header=fits.getheader(filename, 0)
camera=safe_read_key(header,"CAMERA").strip().lower()
if not camera in skies :
skies[camera]=[]
skies[camera].append(sky)
for filename in args.fiberflats :
log.info("reading %s"%filename)
header=fits.getheader(filename, 0)
flat=io.read_fiberflat(filename)
camera=safe_read_key(header,"CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if camera in flats:
log.warning("cannot handle several flats of same camera (%s), will use only the first one"%camera)
#raise ValueError("cannot handle several flats of same camera (%s)"%camera)
else :
flats[camera]=flat
if starindices.size == 0 :
log.error("no STD star found in fibermap")
raise ValueError("no STD star found in fibermap")
log.info("found %d STD stars"%starindices.size)
log.warning("Not using flux errors for Standard Star fits!")
# DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
############################################
for cam in frames :
if not cam in skies:
log.warning("Missing sky for %s"%cam)
frames.pop(cam)
continue
if not cam in flats:
log.warning("Missing flat for %s"%cam)
frames.pop(cam)
continue
flat=flats[cam]
for frame,sky in zip(frames[cam],skies[cam]) :
frame.flux = frame.flux[starindices]
frame.ivar = frame.ivar[starindices]
frame.ivar *= (frame.mask[starindices] == 0)
frame.ivar *= (sky.ivar[starindices] != 0)
frame.ivar *= (sky.mask[starindices] == 0)
frame.ivar *= (flat.ivar[starindices] != 0)
frame.ivar *= (flat.mask[starindices] == 0)
frame.flux *= ( frame.ivar > 0) # just for clean plots
for star in range(frame.flux.shape[0]) :
ok=np.where((frame.ivar[star]>0)&(flat.fiberflat[star]!=0))[0]
if ok.size > 0 :
frame.flux[star] = frame.flux[star]/flat.fiberflat[star] - sky.flux[star]
frame.resolution_data = frame.resolution_data[starindices]
nstars = starindices.size
fibermap = Table(fibermap[starindices])
# READ MODELS
############################################
log.info("reading star models in %s"%args.starmodels)
stdwave,stdflux,templateid,teff,logg,feh=io.read_stdstar_templates(args.starmodels)
# COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
############################################
#- Support older fibermaps
if 'PHOTSYS' not in fibermap.colnames:
log.warning('Old fibermap format; using defaults for missing columns')
log.warning(" PHOTSYS = 'S'")
log.warning(" MW_TRANSMISSION_G/R/Z = 1.0")
log.warning(" EBV = 0.0")
fibermap['PHOTSYS'] = 'S'
fibermap['MW_TRANSMISSION_G'] = 1.0
fibermap['MW_TRANSMISSION_R'] = 1.0
fibermap['MW_TRANSMISSION_Z'] = 1.0
fibermap['EBV'] = 0.0
model_filters = dict()
if 'S' in fibermap['PHOTSYS']:
for filter_name in ['DECAM_G', 'DECAM_R', 'DECAM_Z']:
model_filters[filter_name] = load_filter(filter_name)
if 'N' in fibermap['PHOTSYS']:
for filter_name in ['BASS_G', 'BASS_R', 'MZLS_Z']:
model_filters[filter_name] = load_filter(filter_name)
if len(model_filters) == 0:
raise ValueError("No filters loaded; neither 'N' nor 'S' in PHOTSYS?")
log.info("computing model mags for %s"%sorted(model_filters.keys()))
model_mags = dict()
fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
for filter_name, filter_response in model_filters.items():
model_mags[filter_name] = filter_response.get_ab_magnitude(stdflux*fluxunits,stdwave)
log.info("done computing model mags")
# LOOP ON STARS TO FIND BEST MODEL
############################################
linear_coefficients=np.zeros((nstars,stdflux.shape[0]))
chi2dof=np.zeros((nstars))
redshift=np.zeros((nstars))
normflux=[]
star_mags = dict()
star_unextincted_mags = dict()
for band in ['G', 'R', 'Z']:
star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_'+band])
star_unextincted_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_'+band] / fibermap['MW_TRANSMISSION_'+band])
star_colors = dict()
star_colors['G-R'] = star_mags['G'] - star_mags['R']
star_colors['R-Z'] = star_mags['R'] - star_mags['Z']
star_unextincted_colors = dict()
star_unextincted_colors['G-R'] = star_unextincted_mags['G'] - star_unextincted_mags['R']
star_unextincted_colors['R-Z'] = star_unextincted_mags['R'] - star_unextincted_mags['Z']
fitted_model_colors = np.zeros(nstars)
for star in range(nstars) :
log.info("finding best model for observed star #%d"%star)
# np.array of wave,flux,ivar,resol
wave = {}
flux = {}
ivar = {}
resolution_data = {}
for camera in frames :
for i,frame in enumerate(frames[camera]) :
identifier="%s-%d"%(camera,i)
wave[identifier]=frame.wave
flux[identifier]=frame.flux[star]
ivar[identifier]=frame.ivar[star]
resolution_data[identifier]=frame.resolution_data[star]
# preselect models based on magnitudes
if fibermap['PHOTSYS'][star] == 'N':
if args.color == 'G-R':
model_colors = model_mags['BASS_G'] - model_mags['BASS_R']
elif args.color == 'R-Z':
model_colors = model_mags['BASS_R'] - model_mags['MZLS_Z']
else:
raise ValueError('Unknown color {}'.format(args.color))
else:
if args.color == 'G-R':
model_colors = model_mags['DECAM_G'] - model_mags['DECAM_R']
elif args.color == 'R-Z':
model_colors = model_mags['DECAM_R'] - model_mags['DECAM_Z']
else:
raise ValueError('Unknown color {}'.format(args.color))
color_diff = model_colors - star_unextincted_colors[args.color][star]
selection = np.abs(color_diff) < args.delta_color
# smallest cube in parameter space including this selection (needed for interpolation)
new_selection = (teff>=np.min(teff[selection]))&(teff<=np.max(teff[selection]))
new_selection &= (logg>=np.min(logg[selection]))&(logg<=np.max(logg[selection]))
new_selection &= (feh>=np.min(feh[selection]))&(feh<=np.max(feh[selection]))
selection = np.where(new_selection)[0]
log.info("star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d"%(
star, starfibers[star], args.color, star_unextincted_colors[args.color][star],
selection.size, stdflux.shape[0]))
# Match unextincted standard stars to data
coefficients, redshift[star], chi2dof[star] = match_templates(
wave, flux, ivar, resolution_data,
stdwave, stdflux[selection],
teff[selection], logg[selection], feh[selection],
ncpu=args.ncpu, z_max=args.z_max, z_res=args.z_res,
template_error=args.template_error
)
linear_coefficients[star,selection] = coefficients
log.info('Star Fiber: {0}; TEFF: {1}; LOGG: {2}; FEH: {3}; Redshift: {4}; Chisq/dof: {5}'.format(
starfibers[star],
np.inner(teff,linear_coefficients[star]),
np.inner(logg,linear_coefficients[star]),
np.inner(feh,linear_coefficients[star]),
redshift[star],
chi2dof[star])
)
# Apply redshift to original spectrum at full resolution
model=np.zeros(stdwave.size)
redshifted_stdwave = stdwave*(1+redshift[star])
for i,c in enumerate(linear_coefficients[star]) :
if c != 0 :
model += c*np.interp(stdwave,redshifted_stdwave,stdflux[i])
# Apply dust extinction to the model
model *= dust_transmission(stdwave, fibermap['EBV'][star])
# Compute final color of dust-extincted model
if fibermap['PHOTSYS'][star] == 'N':
if args.color == 'G-R':
model_mag1 = model_filters['BASS_G'].get_ab_magnitude(model*fluxunits, stdwave)
model_mag2 = model_filters['BASS_R'].get_ab_magnitude(model*fluxunits, stdwave)
model_magr = model_mag2
elif args.color == 'R-Z':
model_mag1 = model_filters['BASS_R'].get_ab_magnitude(model*fluxunits, stdwave)
model_mag2 = model_filters['MZLS_Z'].get_ab_magnitude(model*fluxunits, stdwave)
model_magr = model_mag1
else:
raise ValueError('Unknown color {}'.format(args.color))
else:
if args.color == 'G-R':
model_mag1 = model_filters['DECAM_G'].get_ab_magnitude(model*fluxunits, stdwave)
model_mag2 = model_filters['DECAM_R'].get_ab_magnitude(model*fluxunits, stdwave)
model_magr = model_mag2
elif args.color == 'R-Z':
model_mag1 = model_filters['DECAM_R'].get_ab_magnitude(model*fluxunits, stdwave)
model_mag2 = model_filters['DECAM_Z'].get_ab_magnitude(model*fluxunits, stdwave)
model_magr = model_mag1
else:
raise ValueError('Unknown color {}'.format(args.color))
fitted_model_colors[star] = model_mag1 - model_mag2
#- TODO: move this back into normalize_templates, at the cost of
#- recalculating a model magnitude?
# Normalize the best model using reported magnitude
scalefac=10**((model_magr - star_mags['R'][star])/2.5)
log.info('scaling R mag {} to {} using scale {}'.format(model_magr, star_mags['R'][star], scalefac))
normflux.append(model*scalefac)
# Now write the normalized flux for all best models to a file
normflux=np.array(normflux)
data={}
data['LOGG']=linear_coefficients.dot(logg)
data['TEFF']= linear_coefficients.dot(teff)
data['FEH']= linear_coefficients.dot(feh)
data['CHI2DOF']=chi2dof
data['REDSHIFT']=redshift
data['COEFF']=linear_coefficients
data['DATA_%s'%args.color]=star_colors[args.color]
data['MODEL_%s'%args.color]=fitted_model_colors
io.write_stdstar_models(args.outfile,normflux,stdwave,starfibers,data)
def main(args) :
log=get_logger()
cmd = ['desi_compute_fluxcalibration',]
for key, value in args.__dict__.items():
if value is not None:
cmd += ['--'+key, str(value)]
cmd = ' '.join(cmd)
log.info(cmd)
log.info("read frame")
# read frame
frame = read_frame(args.infile)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel=read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux,model_wave,model_fibers,model_metadata=read_stdstar_models(args.models)
if args.chi2cut > 0 :
ok = np.where(model_metadata["CHI2DOF"]<args.chi2cut)[0]
if ok.size == 0 :
log.error("chi2cut has discarded all stars")
sys.exit(12)
nstars=model_flux.shape[0]
nbad=nstars-ok.size
if nbad>0 :
log.warning("discarding %d star(s) out of %d because of chi2cut"%(nbad,nstars))
model_flux=model_flux[ok]
model_fibers=model_fibers[ok]
model_metadata=model_metadata[:][ok]
if args.delta_color_cut > 0 :
ok = np.where(np.abs(model_metadata["MODEL_G-R"]-model_metadata["DATA_G-R"])<args.delta_color_cut)[0]
nstars=model_flux.shape[0]
nbad=nstars-ok.size
if nbad>0 :
log.warning("discarding %d star(s) out of %d because |delta_color|>%f"%(nbad,nstars,args.delta_color_cut))
model_flux=model_flux[ok]
model_fibers=model_fibers[ok]
model_metadata=model_metadata[:][ok]
# automatically reject stars that ar chi2 outliers
if args.chi2cut_nsig > 0 :
mchi2=np.median(model_metadata["CHI2DOF"])
rmschi2=np.std(model_metadata["CHI2DOF"])
maxchi2=mchi2+args.chi2cut_nsig*rmschi2
ok=np.where(model_metadata["CHI2DOF"]<=maxchi2)[0]
nstars=model_flux.shape[0]
nbad=nstars-ok.size
if nbad>0 :
log.warning("discarding %d star(s) out of %d because reduced chi2 outliers (at %d sigma, giving rchi2<%f )"%(nbad,nstars,args.chi2cut_nsig,maxchi2))
model_flux=model_flux[ok]
model_fibers=model_fibers[ok]
model_metadata=model_metadata[:][ok]
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
## check whether star fibers from args.models are consistent with fibers from fibermap
## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
fibermap_std_indices = np.where(isStdStar(fibermap['DESI_TARGET']))[0]
if np.any(~np.in1d(model_fibers%500, fibermap_std_indices)):
for i in model_fibers%500:
log.error("inconsistency with spectrum {}, OBJTYPE='{}', DESI_TARGET={} in fibermap".format(
(i, fibermap["OBJTYPE"][i], fibermap["DESI_TARGET"][i])))
sys.exit(12)
fluxcalib = compute_flux_calibration(frame, model_wave, model_flux, model_fibers%500)
# QA
if (args.qafile is not None):
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR'])
# Run
#import pdb; pdb.set_trace()
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s"%args.outfile)
xk = "OFFSET_X"
yk = "OFFSET_Y"
fiberid = "FIBER"
dico = {}
for i in ii:
dico[table[fiberid][i]] = i
frame_file_b3 = '/project/projectdirs/desi/spectro/redux/v0/exposures/20191028/00022301/frame-b3-00022301.fits'
frame_file_r3 = '/project/projectdirs/desi/spectro/redux/v0/exposures/20191028/00022301/frame-r3-00022301.fits'
frame_file_z3 = '/project/projectdirs/desi/spectro/redux/v0/exposures/20191028/00022301/frame-z3-00022301.fits'
sframe_file_b3 = '/project/projectdirs/desi/spectro/redux/v0/exposures/20191028/00022301/sframe-b3-00022301.fits'
sframe_file_r3 = '/project/projectdirs/desi/spectro/redux/v0/exposures/20191028/00022301/sframe-r3-00022301.fits'
sframe_file_z3 = '/project/projectdirs/desi/spectro/redux/v0/exposures/20191028/00022301/sframe-z3-00022301.fits'
frame_b3 = read_frame(frame_file_b3)
frame_r3 = read_frame(frame_file_r3)
frame_z3 = read_frame(frame_file_z3)
sframe_b3 = read_frame(sframe_file_b3)
sframe_r3 = read_frame(sframe_file_r3)
sframe_z3 = read_frame(sframe_file_z3)
flux_b3 = frame_b3.flux
flux_r3 = frame_r3.flux
flux_z3 = frame_z3.flux
sflux_b3 = sframe_b3.flux
sflux_r3 = sframe_r3.flux
sflux_z3 = sframe_z3.flux
fiber = frame_b3.fibermap["FIBER"] - 1500
x = np.zeros(fiber.size)
def qaframe_from_frame(frame_file,
specprod_dir=None,
make_plots=False,
qaprod_dir=None,
output_dir=None,
clobber=True):
""" Generate a qaframe object from an input frame_file name (and night)
Write QA to disk
Will also make plots if directed
Args:
frame_file: str
specprod_dir: str, optional
qa_dir: str, optional -- Location of QA
make_plots: bool, optional
output_dir: str, optional
Returns:
"""
import glob
import os
from desispec.io import read_frame
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.qa import qafile_from_framefile
from desispec.io.frame import search_for_framefile
from desispec.io.fiberflat import read_fiberflat
from desispec.fiberflat import apply_fiberflat
from desispec.qa import qa_plots
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
if '/' in frame_file: # If present, assume full path is used here
pass
else: # Find the frame file in the desispec hierarchy?
frame_file = search_for_framefile(frame_file)
# Load frame
frame = read_frame(frame_file)
frame_meta = frame.meta
night = frame_meta['NIGHT'].strip()
camera = frame_meta['CAMERA'].strip()
expid = frame_meta['EXPID']
spectro = int(frame_meta['CAMERA'][-1])
# Filename
qafile, qatype = qafile_from_framefile(frame_file,
qaprod_dir=qaprod_dir,
output_dir=output_dir)
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame_meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try: # Backwards compatibility
fiberflat = read_fiberflat(fiberflat_fil)
except FileNotFoundError:
fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
path, basen = os.path.split(fiberflat_fil)
path, _ = os.path.split(path)
fiberflat_fil = os.path.join(path, basen)
fiberflat = read_fiberflat(fiberflat_fil)
qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber)
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
fiberflat_fil = meta.findfile('fiberflatnight',
night=night,
camera=camera)
if not os.path.exists(fiberflat_fil):
# Backwards compatibility (for now)
dummy_fiberflat_fil = meta.findfile(
'fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir) # This is dummy
path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
fiberflat_files = glob.glob(
os.path.join(path, '*', 'fiberflat-' + camera + '*.fits'))
if len(fiberflat_files) == 0:
path = path.replace('exposures', 'calib2d')
path, _ = os.path.split(path) # Remove night
fiberflat_files = glob.glob(
os.path.join(path, 'fiberflat-' + camera + '*.fits'))
# Sort and take the first (same as old pipeline)
fiberflat_files.sort()
fiberflat_fil = fiberflat_files[0]
fiberflat = read_fiberflat(fiberflat_fil)
apply_fiberflat(frame, fiberflat)
# Load sky model and run
try:
skymodel = read_sky(sky_fil)
except FileNotFoundError:
warnings.warn(
"Sky file {:s} not found. Skipping..".format(sky_fil))
else:
qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber)
if make_plots:
qafig = meta.findfile('qa_sky_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qafig2 = meta.findfile('qa_skychi_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
#qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
spectrograph=spectro)
# try:
# model_tuple=read_stdstar_models(stdstar_fil)
# except FileNotFoundError:
# warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil))
# else:
flux_fil = meta.findfile('calib',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try:
fluxcalib = read_flux_calibration(flux_fil)
except FileNotFoundError:
warnings.warn(
"Flux file {:s} not found. Skipping..".format(flux_fil))
else:
qaframe.run_qa(
'FLUXCALIB',
(frame, fluxcalib)) # , model_tuple))#, indiv_stars))
if make_plots:
qafig = meta.findfile('qa_flux_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qa_plots.frame_fluxcalib(qafig, qaframe, frame,
fluxcalib) # , model_tuple)
# Write
write_qa_frame(qafile, qaframe, verbose=True)
return qaframe
def main(args):
# imports
import glob
from desispec.io import findfile
from desispec.io import get_exposures
from desispec.io import get_files
from desispec.io import read_frame
from desispec.io.sky import read_sky
from desispec.qa.qa_plots import skysub_resid
import copy
import pdb
# Log
log = get_logger()
log.info("starting")
# Exposures?
if args.expids is not None:
expids = [int(iarg) for iarg in args.expids.split(',')]
else:
expids = 'all'
# Nights?
if args.nights is not None:
gdnights = [iarg for iarg in args.nights.split(',')]
else:
gdnights = 'all'
# Channels?
if args.channels is not None:
gdchannels = [iarg for iarg in args.channels.split(',')]
else:
gdchannels = 'all'
# Sky dict
sky_dict = dict(wave=[], skyflux=[], res=[], count=0)
channel_dict = dict(
b=copy.deepcopy(sky_dict),
r=copy.deepcopy(sky_dict),
z=copy.deepcopy(sky_dict),
)
# Loop on nights
path_nights = glob.glob(args.specprod_dir + '/exposures/*')
nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights]
for night in nights:
if gdnights == 'all':
pass
else:
if night not in gdnights:
continue
# Get em
for exposure in get_exposures(night, specprod_dir=args.specprod_dir):
# Check against input expids
if expids == 'all':
pass
else:
if exposure not in expids:
continue
# Get em
frames_dict = get_files(filetype=str('cframe'),
night=night,
expid=exposure,
specprod_dir=args.specprod_dir)
for camera, cframe_fil in frames_dict.items():
channel = camera[0]
# Check against input
if gdchannels == 'all':
pass
else:
if channel not in gdchannels:
continue
# Load frame
log.info('Loading {:s}'.format(cframe_fil))
cframe = read_frame(cframe_fil)
if cframe.meta['FLAVOR'] in ['flat',
'arc']: # Probably can't happen
continue
# Sky
sky_file = findfile(str('sky'),
night=night,
camera=camera,
expid=exposure,
specprod_dir=args.specprod_dir)
skymodel = read_sky(sky_file)
# Resid
skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0]
res = cframe.flux[skyfibers]
flux = skymodel.flux[skyfibers] # Residuals
tmp = np.outer(np.ones(flux.shape[0]), cframe.wave)
# Append
#from xastropy.xutils import xdebug as xdb
#xdb.set_trace()
channel_dict[channel]['wave'].append(tmp.flatten())
channel_dict[channel]['skyflux'].append(
np.log10(np.maximum(flux.flatten(), 1e-1)))
channel_dict[channel]['res'].append(res.flatten())
channel_dict[channel]['count'] += 1
# Figure
for channel in ['b', 'r', 'z']:
if channel_dict[channel]['count'] > 0:
sky_wave = np.concatenate(channel_dict[channel]['wave'])
sky_flux = np.concatenate(channel_dict[channel]['skyflux'])
sky_res = np.concatenate(channel_dict[channel]['res'])
# Plot
skysub_resid(sky_wave,
sky_flux,
sky_res,
outfile='tmp{:s}.png'.format(channel))
def main(args):
log = get_logger()
log.info("read frame")
# read frame
frame = read_frame(args.infile)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel = read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux, model_wave, model_fibers = read_stdstar_models(args.models)
model_tuple = model_flux, model_wave, model_fibers
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
model_fibers = model_fibers % 500
if np.any(fibermap['OBJTYPE'][model_fibers] != 'STD'):
for i in model_fibers:
log.error(
"inconsistency with spectrum %d, OBJTYPE='%s' in fibermap" %
(i, fibermap["OBJTYPE"][i]))
sys.exit(12)
#fluxcalib, indiv_stars = compute_flux_calibration(frame, model_wave, model_flux)
fluxcalib = compute_flux_calibration(frame, model_wave, model_flux)
# QA
if (args.qafile is not None):
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame,
flavor=frame.meta['FLAVOR'])
# Run
qaframe.run_qa('FLUXCALIB',
(frame, fluxcalib, model_tuple)) #, indiv_stars))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib,
model_tuple)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s" % args.outfile)
def main(args=None):
log = get_logger()
if args is None:
args = parse()
if args.lin_step is not None and args.log10_step is not None:
log.critical(
"cannot have both linear and logarthmic bins :-), choose either --lin-step or --log10-step"
)
return 12
if args.coadd_cameras and (args.lin_step is not None
or args.log10_step is not None):
log.critical(
"cannot specify a new wavelength binning along with --coadd-cameras option"
)
return 12
if len(args.infile) == 0:
log.critical("You must specify input files")
return 12
log.info("reading input ...")
# inspect headers
input_is_frames = False
input_is_spectra = False
for filename in args.infile:
ifile = fitsio.FITS(filename)
head = ifile[0].read_header()
identified = False
if "EXTNAME" in head and head["EXTNAME"] == "FLUX":
print(filename, "is a frame")
input_is_frames = True
identified = True
ifile.close()
continue
for hdu in ifile:
head = hdu.read_header()
if "EXTNAME" in head and head["EXTNAME"].find("_FLUX") >= 0:
print(filename, "is a spectra")
input_is_spectra = True
identified = True
break
ifile.close()
if not identified:
log.error(
"{} not identified as frame of spectra file".format(filename))
sys.exit(1)
if input_is_frames and input_is_spectra:
log.error("cannot combine input spectra and frames")
sys.exit(1)
if input_is_spectra:
spectra = read_spectra(args.infile[0])
for filename in args.infile[1:]:
log.info("append {}".format(filename))
spectra.update(read_spectra(filename))
else: # frames
frames = dict()
cameras = {}
for filename in args.infile:
frame = read_frame(filename)
night = frame.meta['NIGHT']
expid = frame.meta['EXPID']
camera = frame.meta['CAMERA']
frames[(night, expid, camera)] = frame
if args.coadd_cameras:
cam, spec = camera[0], camera[1]
# Keep a list of cameras (b,r,z) for each exposure + spec
if (night, expid) not in cameras.keys():
cameras[(night, expid)] = {spec: [cam]}
elif spec not in cameras[(night, expid)].keys():
cameras[(night, expid)][spec] = [cam]
else:
cameras[(night, expid)][spec].append(cam)
if args.coadd_cameras:
# If not all 3 cameras are available, remove the incomplete sets
for (night, expid), camdict in cameras.items():
for spec, camlist in camdict.items():
log.info("Found {} for SP{} on NIGHT {} EXP {}".format(
camlist, spec, night, expid))
if len(camlist) != 3 or np.any(
np.sort(camlist) != np.array(['b', 'r', 'z'])):
for cam in camlist:
frames.pop((night, expid, cam + spec))
log.warning(
"Removing {}{} from Night {} EXP {}".format(
cam, spec, night, expid))
#import pdb
#pdb.set_trace()
spectra = frames2spectra(frames)
#- hacks to make SpectraLite like a Spectra
spectra.fibermap = Table(spectra.fibermap)
del frames #- maybe free some memory
if args.coadd_cameras:
log.info("coadding cameras ...")
spectra = coadd_cameras(spectra, cosmics_nsig=args.nsig)
else:
log.info("coadding ...")
coadd(spectra, cosmics_nsig=args.nsig)
if args.lin_step is not None:
log.info("resampling ...")
spectra = resample_spectra_lin_or_log(spectra,
linear_step=args.lin_step,
wave_min=args.wave_min,
wave_max=args.wave_max,
fast=args.fast,
nproc=args.nproc)
if args.log10_step is not None:
log.info("resampling ...")
spectra = resample_spectra_lin_or_log(spectra,
log10_step=args.log10_step,
wave_min=args.wave_min,
wave_max=args.wave_max,
fast=args.fast,
nproc=args.nproc)
#- Add input files to header
if spectra.meta is None:
spectra.meta = dict()
for i, filename in enumerate(args.infile):
spectra.meta['INFIL{:03d}'.format(i)] = os.path.basename(filename)
log.info("writing {} ...".format(args.outfile))
write_spectra(args.outfile, spectra)
log.info("done")
def main(args):
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
log = get_logger()
log.info("mag delta %s = %f (for the pre-selection of stellar models)" %
(args.color, args.delta_color))
frames = {}
flats = {}
skies = {}
spectrograph = None
starfibers = None
starindices = None
fibermap = None
# READ DATA
############################################
for filename in args.frames:
log.info("reading %s" % filename)
frame = io.read_frame(filename)
header = fits.getheader(filename, 0)
frame_fibermap = frame.fibermap
frame_starindices = np.where(frame_fibermap["OBJTYPE"] == "STD")[0]
camera = safe_read_key(header, "CAMERA").strip().lower()
if spectrograph is None:
spectrograph = frame.spectrograph
fibermap = frame_fibermap
starindices = frame_starindices
starfibers = fibermap["FIBER"][starindices]
elif spectrograph != frame.spectrograph:
log.error("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
raise ValueError("incompatible spectrographs %d != %d" %
(spectrograph, frame.spectrograph))
elif starindices.size != frame_starindices.size or np.sum(
starindices != frame_starindices) > 0:
log.error("incompatible fibermap")
raise ValueError("incompatible fibermap")
if frames.has_key(camera):
log.error(
"cannot handle for now several frame of same camera (%s)" %
camera)
raise ValueError(
"cannot handle for now several frame of same camera (%s)" %
camera)
frames[camera] = frame
for filename in args.skymodels:
log.info("reading %s" % filename)
sky = io.read_sky(filename)
header = fits.getheader(filename, 0)
camera = safe_read_key(header, "CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if skies.has_key(camera):
log.error("cannot handle several skymodels of same camera (%s)" %
camera)
raise ValueError(
"cannot handle several skymodels of same camera (%s)" % camera)
skies[camera] = sky
for filename in args.fiberflats:
log.info("reading %s" % filename)
header = fits.getheader(filename, 0)
flat = io.read_fiberflat(filename)
camera = safe_read_key(header, "CAMERA").strip().lower()
# NEED TO ADD MORE CHECKS
if flats.has_key(camera):
log.error("cannot handle several flats of same camera (%s)" %
camera)
raise ValueError(
"cannot handle several flats of same camera (%s)" % camera)
flats[camera] = flat
if starindices.size == 0:
log.error("no STD star found in fibermap")
raise ValueError("no STD star found in fibermap")
log.info("found %d STD stars" % starindices.size)
imaging_filters = fibermap["FILTER"][starindices]
imaging_mags = fibermap["MAG"][starindices]
log.warning(
"NO MAG ERRORS IN FIBERMAP, I AM IGNORING MEASUREMENT ERRORS !!")
log.warning(
"NO EXTINCTION VALUES IN FIBERMAP, I AM IGNORING THIS FOR NOW !!")
# DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA
############################################
for cam in frames:
if not skies.has_key(cam):
log.warning("Missing sky for %s" % cam)
frames.pop(cam)
continue
if not flats.has_key(cam):
log.warning("Missing flat for %s" % cam)
frames.pop(cam)
continue
frames[cam].flux = frames[cam].flux[starindices]
frames[cam].ivar = frames[cam].ivar[starindices]
frames[cam].ivar *= (frames[cam].mask[starindices] == 0)
frames[cam].ivar *= (skies[cam].ivar[starindices] != 0)
frames[cam].ivar *= (skies[cam].mask[starindices] == 0)
frames[cam].ivar *= (flats[cam].ivar[starindices] != 0)
frames[cam].ivar *= (flats[cam].mask[starindices] == 0)
frames[cam].flux *= (frames[cam].ivar > 0) # just for clean plots
for star in range(frames[cam].flux.shape[0]):
ok = np.where((frames[cam].ivar[star] > 0)
& (flats[cam].fiberflat[star] != 0))[0]
if ok.size > 0:
frames[cam].flux[star] = frames[cam].flux[star] / flats[
cam].fiberflat[star] - skies[cam].flux[star]
nstars = starindices.size
starindices = None # we don't need this anymore
# READ MODELS
############################################
log.info("reading star models in %s" % args.starmodels)
stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates(
args.starmodels)
# COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG
############################################
model_filters = []
for tmp in np.unique(imaging_filters):
if len(tmp) > 0: # can be one empty entry
model_filters.append(tmp)
log.info("computing model mags %s" % model_filters)
model_mags = np.zeros((stdflux.shape[0], len(model_filters)))
fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom
for index in range(len(model_filters)):
filter_response = load_filter(model_filters[index])
for m in range(stdflux.shape[0]):
model_mags[m, index] = filter_response.get_ab_magnitude(
stdflux[m] * fluxunits, stdwave)
log.info("done computing model mags")
# LOOP ON STARS TO FIND BEST MODEL
############################################
bestModelIndex = np.arange(nstars)
templateID = np.arange(nstars)
chi2dof = np.zeros((nstars))
redshift = np.zeros((nstars))
normflux = []
for star in range(nstars):
log.info("finding best model for observed star #%d" % star)
# np.array of wave,flux,ivar,resol
wave = {}
flux = {}
ivar = {}
resolution_data = {}
for camera in frames:
band = camera[0]
wave[band] = frames[camera].wave
flux[band] = frames[camera].flux[star]
ivar[band] = frames[camera].ivar[star]
resolution_data[band] = frames[camera].resolution_data[star]
# preselec models based on magnitudes
# compute star color
index1, index2 = get_color_filter_indices(imaging_filters[star],
args.color)
if index1 < 0 or index2 < 0:
log.error("cannot compute '%s' color from %s" %
(color_name, filters))
filter1 = imaging_filters[star][index1]
filter2 = imaging_filters[star][index2]
star_color = imaging_mags[star][index1] - imaging_mags[star][index2]
# compute models color
model_index1 = -1
model_index2 = -1
for i, fname in enumerate(model_filters):
if fname == filter1:
model_index1 = i
elif fname == filter2:
model_index2 = i
if model_index1 < 0 or model_index2 < 0:
log.error("cannot compute '%s' model color from %s" %
(color_name, filters))
model_colors = model_mags[:, model_index1] - model_mags[:,
model_index2]
# selection
selection = np.where(
np.abs(model_colors - star_color) < args.delta_color)[0]
log.info(
"star#%d fiber #%d, %s = %s-%s = %f, number of pre-selected models = %d/%d"
% (star, starfibers[star], args.color, filter1, filter2,
star_color, selection.size, stdflux.shape[0]))
index_in_selection, redshift[star], chi2dof[star] = match_templates(
wave,
flux,
ivar,
resolution_data,
stdwave,
stdflux[selection],
teff[selection],
logg[selection],
feh[selection],
ncpu=args.ncpu,
z_max=args.z_max,
z_res=args.z_res)
bestModelIndex[star] = selection[index_in_selection]
log.info(
'Star Fiber: {0}; TemplateID: {1}; Redshift: {2}; Chisq/dof: {3}'.
format(starfibers[star], bestModelIndex[star], redshift[star],
chi2dof[star]))
# Apply redshift to original spectrum at full resolution
tmp = np.interp(stdwave, stdwave / (1 + redshift[star]),
stdflux[bestModelIndex[star]])
# Normalize the best model using reported magnitude
normalizedflux = normalize_templates(stdwave, tmp, imaging_mags[star],
imaging_filters[star])
normflux.append(normalizedflux)
# Now write the normalized flux for all best models to a file
normflux = np.array(normflux)
data = {}
data['BESTMODEL'] = bestModelIndex
data['TEMPLATEID'] = bestModelIndex # IS THAT IT?
data['CHI2DOF'] = chi2dof
data['REDSHIFT'] = redshift
norm_model_file = args.outfile
io.write_stdstar_models(args.outfile, normflux, stdwave, starfibers, data)
def main(args) :
# imports
import glob
from desispec.io import findfile
from desispec.io import get_exposures
from desispec.io import get_files
from desispec.io import read_frame
from desispec.io.sky import read_sky
from desispec.qa.qa_plots import skysub_resid
import copy
import pdb
# Log
log=get_logger()
log.info("starting")
# Exposures?
if args.expids is not None:
expids = [int(iarg) for iarg in args.expids.split(',')]
else:
expids = 'all'
# Nights?
if args.nights is not None:
gdnights = [iarg for iarg in args.nights.split(',')]
else:
gdnights = 'all'
# Channels?
if args.channels is not None:
gdchannels = [iarg for iarg in args.channels.split(',')]
else:
gdchannels = 'all'
# Sky dict
sky_dict = dict(wave=[], skyflux=[], res=[], count=0)
channel_dict = dict(b=copy.deepcopy(sky_dict),
r=copy.deepcopy(sky_dict),
z=copy.deepcopy(sky_dict),
)
# Loop on nights
path_nights = glob.glob(args.specprod_dir+'/exposures/*')
nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights]
for night in nights:
if gdnights == 'all':
pass
else:
if night not in gdnights:
continue
# Get em
for exposure in get_exposures(night, specprod_dir = args.specprod_dir):
# Check against input expids
if expids == 'all':
pass
else:
if exposure not in expids:
continue
# Get em
frames_dict = get_files(filetype=str('cframe'), night=night,
expid=exposure, specprod_dir=args.specprod_dir)
for camera, cframe_fil in frames_dict.items():
channel = camera[0]
# Check against input
if gdchannels == 'all':
pass
else:
if channel not in gdchannels:
continue
# Load frame
log.info('Loading {:s}'.format(cframe_fil))
cframe = read_frame(cframe_fil)
if cframe.meta['FLAVOR'] in ['flat','arc']: # Probably can't happen
continue
# Sky
sky_file = findfile(str('sky'), night=night, camera=camera,
expid=exposure, specprod_dir=args.specprod_dir)
skymodel = read_sky(sky_file)
# Resid
skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0]
res = cframe.flux[skyfibers]
flux = skymodel.flux[skyfibers] # Residuals
tmp = np.outer(np.ones(flux.shape[0]), cframe.wave)
# Append
#from xastropy.xutils import xdebug as xdb
#xdb.set_trace()
channel_dict[channel]['wave'].append(tmp.flatten())
channel_dict[channel]['skyflux'].append(
np.log10(np.maximum(flux.flatten(),1e-1)))
channel_dict[channel]['res'].append(res.flatten())
channel_dict[channel]['count'] += 1
# Figure
for channel in ['b', 'r', 'z']:
if channel_dict[channel]['count'] > 0:
sky_wave = np.concatenate(channel_dict[channel]['wave'])
sky_flux = np.concatenate(channel_dict[channel]['skyflux'])
sky_res = np.concatenate(channel_dict[channel]['res'])
# Plot
skysub_resid(sky_wave, sky_flux, sky_res,
outfile='tmp{:s}.png'.format(channel))
def qaframe_from_frame(frame_file,
specprod_dir=None,
make_plots=False,
qaprod_dir=None,
output_dir=None,
clobber=True):
""" Generate a qaframe object from an input frame_file name (and night)
Write QA to disk
Will also make plots if directed
Args:
frame_file: str
specprod_dir: str, optional
qa_dir: str, optional -- Location of QA
make_plots: bool, optional
output_dir: str, optional
Returns:
"""
import glob
import os
from desispec.io import read_frame
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.qa import qafile_from_framefile
from desispec.io.frame import search_for_framefile
from desispec.io.fiberflat import read_fiberflat
from desispec.fiberflat import apply_fiberflat
from desispec.qa import qa_plots
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots_ql
if '/' in frame_file: # If present, assume full path is used here
pass
else: # Find the frame file in the desispec hierarchy?
frame_file = search_for_framefile(frame_file)
# Load frame
frame = read_frame(frame_file)
frame_meta = frame.meta
night = frame_meta['NIGHT'].strip()
camera = frame_meta['CAMERA'].strip()
expid = frame_meta['EXPID']
spectro = int(frame_meta['CAMERA'][-1])
# Filename
qafile, qatype = qafile_from_framefile(frame_file,
qaprod_dir=qaprod_dir,
output_dir=output_dir)
if os.path.isfile(qafile) and (not clobber):
write = False
else:
write = True
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame_meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try: # Backwards compatibility
fiberflat = read_fiberflat(fiberflat_fil)
except FileNotFoundError:
fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
path, basen = os.path.split(fiberflat_fil)
path, _ = os.path.split(path)
fiberflat_fil = os.path.join(path, basen)
fiberflat = read_fiberflat(fiberflat_fil)
if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber):
write = True
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig',
night=night,
camera=camera,
expid=expid,
qaprod_dir=qaprod_dir,
specprod_dir=specprod_dir,
outdir=output_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
fiberflat_fil = meta.findfile('fiberflatnight',
night=night,
camera=camera)
if not os.path.exists(fiberflat_fil):
# Backwards compatibility (for now)
dummy_fiberflat_fil = meta.findfile(
'fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir) # This is dummy
path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
fiberflat_files = glob.glob(
os.path.join(path, '*', 'fiberflat-' + camera + '*.fits'))
if len(fiberflat_files) == 0:
path = path.replace('exposures', 'calib2d')
path, _ = os.path.split(path) # Remove night
fiberflat_files = glob.glob(
os.path.join(path, 'fiberflat-' + camera + '*.fits'))
# Sort and take the first (same as old pipeline)
fiberflat_files.sort()
fiberflat_fil = fiberflat_files[0]
fiberflat = read_fiberflat(fiberflat_fil)
apply_fiberflat(frame, fiberflat)
# Load sky model and run
try:
skymodel = read_sky(sky_fil)
except FileNotFoundError:
warnings.warn(
"Sky file {:s} not found. Skipping..".format(sky_fil))
else:
if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber):
write = True
if make_plots:
qafig = meta.findfile('qa_sky_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
qafig2 = meta.findfile('qa_skychi_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
#qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)
# S/N QA on cframe
if qatype == 'qa_data':
# cframe
cframe_file = frame_file.replace('frame-', 'cframe-')
cframe = read_frame(cframe_file)
if qaframe.run_qa('S2N', (cframe, ), clobber=clobber):
write = True
# Figure?
if make_plots:
s2n_dict = copy.deepcopy(qaframe.qa_data['S2N'])
qafig = meta.findfile('qa_s2n_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
#badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist()
#sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE')
coeff = s2n_dict['METRICS']['FITCOEFF_TGT'] #[sci_idx]
# Add an item or two for the QL method
s2n_dict['CAMERA'] = camera
s2n_dict['EXPID'] = expid
s2n_dict['PANAME'] = 'SNRFit'
s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA']
s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC']
objlist = s2n_dict['METRICS']['OBJLIST']
# Deal with YAML list instead of ndarray
s2n_dict['METRICS']['MEDIAN_SNR'] = np.array(
s2n_dict['METRICS']['MEDIAN_SNR'])
# Generate
if (not os.path.isfile(qafig)) or clobber:
qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist,
[[]] * len(objlist), coeff)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
spectrograph=spectro)
# try:
# model_tuple=read_stdstar_models(stdstar_fil)
# except FileNotFoundError:
# warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil))
# else:
flux_fil = meta.findfile('calib',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try:
fluxcalib = read_flux_calibration(flux_fil)
except FileNotFoundError:
warnings.warn(
"Flux file {:s} not found. Skipping..".format(flux_fil))
else:
if qaframe.run_qa(
'FLUXCALIB', (frame, fluxcalib),
clobber=clobber): # , model_tuple))#, indiv_stars))
write = True
if make_plots:
qafig = meta.findfile('qa_flux_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fluxcalib(qafig, qaframe, frame,
fluxcalib) # , model_tuple)
# Write
if write:
write_qa_frame(qafile, qaframe, verbose=True)
return qaframe
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None,
output_dir=None, clobber=True):
""" Generate a qaframe object from an input frame_file name (and night)
Write QA to disk
Will also make plots if directed
Args:
frame_file: str
specprod_dir: str, optional
qa_dir: str, optional -- Location of QA
make_plots: bool, optional
output_dir: str, optional
Returns:
"""
import glob
import os
from desispec.io import read_frame
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.qa import qafile_from_framefile
from desispec.io.frame import search_for_framefile
from desispec.io.fiberflat import read_fiberflat
from desispec.fiberflat import apply_fiberflat
from desispec.qa import qa_plots
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots_ql
if '/' in frame_file: # If present, assume full path is used here
pass
else: # Find the frame file in the desispec hierarchy?
frame_file = search_for_framefile(frame_file)
# Load frame
frame = read_frame(frame_file)
frame_meta = frame.meta
night = frame_meta['NIGHT'].strip()
camera = frame_meta['CAMERA'].strip()
expid = frame_meta['EXPID']
spectro = int(frame_meta['CAMERA'][-1])
# Filename
qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir)
if os.path.isfile(qafile) and (not clobber):
write = False
else:
write = True
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame_meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir)
try: # Backwards compatibility
fiberflat = read_fiberflat(fiberflat_fil)
except FileNotFoundError:
fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
path, basen = os.path.split(fiberflat_fil)
path,_ = os.path.split(path)
fiberflat_fil = os.path.join(path, basen)
fiberflat = read_fiberflat(fiberflat_fil)
if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber):
write = True
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid,
qaprod_dir=qaprod_dir, specprod_dir=specprod_dir, outdir=output_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir)
fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera)
if not os.path.exists(fiberflat_fil):
# Backwards compatibility (for now)
dummy_fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir) # This is dummy
path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
fiberflat_files = glob.glob(os.path.join(path,'*','fiberflat-'+camera+'*.fits'))
if len(fiberflat_files) == 0:
path = path.replace('exposures', 'calib2d')
path,_ = os.path.split(path) # Remove night
fiberflat_files = glob.glob(os.path.join(path,'fiberflat-'+camera+'*.fits'))
# Sort and take the first (same as old pipeline)
fiberflat_files.sort()
fiberflat_fil = fiberflat_files[0]
fiberflat = read_fiberflat(fiberflat_fil)
apply_fiberflat(frame, fiberflat)
# Load sky model and run
try:
skymodel = read_sky(sky_fil)
except FileNotFoundError:
warnings.warn("Sky file {:s} not found. Skipping..".format(sky_fil))
else:
if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber):
write=True
if make_plots:
qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
#qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)
# S/N QA on cframe
if qatype == 'qa_data':
# cframe
cframe_file = frame_file.replace('frame-', 'cframe-')
cframe = read_frame(cframe_file)
if qaframe.run_qa('S2N', (cframe,), clobber=clobber):
write=True
# Figure?
if make_plots:
s2n_dict = copy.deepcopy(qaframe.qa_data['S2N'])
qafig = meta.findfile('qa_s2n_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
#badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist()
#sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE')
coeff = s2n_dict['METRICS']['FITCOEFF_TGT']#[sci_idx]
# Add an item or two for the QL method
s2n_dict['CAMERA'] = camera
s2n_dict['EXPID'] = expid
s2n_dict['PANAME'] = 'SNRFit'
s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA']
s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC']
objlist = s2n_dict['METRICS']['OBJLIST']
# Deal with YAML list instead of ndarray
s2n_dict['METRICS']['MEDIAN_SNR'] = np.array(s2n_dict['METRICS']['MEDIAN_SNR'])
# Generate
if (not os.path.isfile(qafig)) or clobber:
qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist, [[]]*len(objlist), coeff)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir,
spectrograph=spectro)
# try:
# model_tuple=read_stdstar_models(stdstar_fil)
# except FileNotFoundError:
# warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil))
# else:
flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir)
try:
fluxcalib = read_flux_calibration(flux_fil)
except FileNotFoundError:
warnings.warn("Flux file {:s} not found. Skipping..".format(flux_fil))
else:
if qaframe.run_qa('FLUXCALIB', (frame, fluxcalib), clobber=clobber): # , model_tuple))#, indiv_stars))
write = True
if make_plots:
qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple)
# Write
if write:
write_qa_frame(qafile, qaframe, verbose=True)
return qaframe
def main() :
""" finds the best models of all standard stars in the frame
and normlize the model flux. Output is written to a file and will be called for calibration.
"""
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--fiberflatexpid', type = int, help = 'fiberflat exposure ID')
parser.add_argument('--fibermap', type = str, help = 'path of fibermap file')
parser.add_argument('--models', type = str, help = 'path of spectro-photometric stellar spectra fits')
parser.add_argument('--spectrograph', type = int, default = 0, help = 'spectrograph number, can go 0-9')
parser.add_argument('--outfile', type = str, help = 'output file for normalized stdstar model flux')
args = parser.parse_args()
log = get_logger()
# Call necessary environment variables. No need if add argument to give full file path.
if 'DESI_SPECTRO_REDUX' not in os.environ:
raise RuntimeError('Set environment DESI_SPECTRO_REDUX. It is needed to read the needed datafiles')
DESI_SPECTRO_REDUX=os.environ['DESI_SPECTRO_REDUX']
PRODNAME=os.environ['PRODNAME']
if 'DESISIM' not in os.environ:
raise RuntimeError('Set environment DESISIM. It will be neede to read the filter transmission files for calibration')
DESISIM=os.environ['DESISIM'] # to read the filter transmission files
if args.fibermap is None or args.models is None or \
args.spectrograph is None or args.outfile is None or \
args.fiberflatexpid is None:
log.critical('Missing a required argument')
parser.print_help()
sys.exit(12)
# read Standard Stars from the fibermap file
# returns the Fiber id, filter names and mags for the standard stars
fiber_tbdata,fiber_header=io.read_fibermap(args.fibermap, header=True)
#- Trim to just fibers on this spectrograph
ii = (500*args.spectrograph <= fiber_tbdata["FIBER"])
ii &= (fiber_tbdata["FIBER"] < 500*(args.spectrograph+1))
fiber_tbdata = fiber_tbdata[ii]
#- Get info for the standard stars
refStarIdx=np.where(fiber_tbdata["OBJTYPE"]=="STD")
refFibers=fiber_tbdata["FIBER"][refStarIdx]
refFilters=fiber_tbdata["FILTER"][refStarIdx]
refMags=fiber_tbdata["MAG"]
fibers={"FIBER":refFibers,"FILTER":refFilters,"MAG":refMags}
NIGHT=fiber_header['NIGHT']
EXPID=fiber_header['EXPID']
filters=fibers["FILTER"]
if 'DESISIM' not in os.environ:
raise RuntimeError('Set environment DESISIM. Can not find filter response files')
basepath=DESISIM+"/data/"
#now load all the skyfiles, framefiles, fiberflatfiles etc
# all three channels files are simultaneously treated for model fitting
skyfile={}
framefile={}
fiberflatfile={}
for i in ["b","r","z"]:
camera = i+str(args.spectrograph)
skyfile[i] = io.findfile('sky', NIGHT, EXPID, camera)
framefile[i] = io.findfile('frame', NIGHT, EXPID, camera)
fiberflatfile[i] = io.findfile('fiberflat', NIGHT, args.fiberflatexpid, camera)
#Read Frames, Flats and Sky files
frameFlux={}
frameIvar={}
frameWave={}
frameResolution={}
framehdr={}
fiberFlat={}
ivarFlat={}
maskFlat={}
meanspecFlat={}
waveFlat={}
headerFlat={}
sky={}
skyivar={}
skymask={}
skywave={}
skyhdr={}
for i in ["b","r","z"]:
#arg=(night,expid,'%s%s'%(i,spectrograph))
#- minimal code change for refactored I/O, while not taking advantage of simplified structure
frame = io.read_frame(framefile[i])
frameFlux[i] = frame.flux
frameIvar[i] = frame.ivar
frameWave[i] = frame.wave
frameResolution[i] = frame.resolution_data
framehdr[i] = frame.header
ff = io.read_fiberflat(fiberflatfile[i])
fiberFlat[i] = ff.fiberflat
ivarFlat[i] = ff.ivar
maskFlat[i] = ff.mask
meanspecFlat[i] = ff.meanspec
waveFlat[i] = ff.wave
headerFlat[i] = ff.header
skymodel = io.read_sky(skyfile[i])
sky[i] = skymodel.flux
skyivar[i] = skymodel.ivar
skymask[i] = skymodel.mask
skywave[i] = skymodel.wave
skyhdr[i] = skymodel.header
# Convolve Sky with Detector Resolution, so as to subtract from data. Convolve for all 500 specs. Subtracting sky this way should be equivalent to sky_subtract
convolvedsky={"b":sky["b"], "r":sky["r"], "z":sky["z"]}
# Read the standard Star data and divide by flat and subtract sky
stars=[]
ivars=[]
for i in fibers["FIBER"]:
#flat and sky should have same wavelength binning as data, otherwise should be rebinned.
stars.append((i,{"b":[frameFlux["b"][i]/fiberFlat["b"][i]-convolvedsky["b"][i],frameWave["b"]],
"r":[frameFlux["r"][i]/fiberFlat["r"][i]-convolvedsky["r"][i],frameWave["r"]],
"z":[frameFlux["z"][i]/fiberFlat["z"][i]-convolvedsky["z"][i],frameWave]},fibers["MAG"][i]))
ivars.append((i,{"b":[frameIvar["b"][i]],"r":[frameIvar["r"][i,:]],"z":[frameIvar["z"][i,:]]}))
stdwave,stdflux,templateid=io.read_stdstar_templates(args.models)
#- Trim standard star wavelengths to just the range we need
minwave = min([min(w) for w in frameWave.values()])
maxwave = max([max(w) for w in frameWave.values()])
ii = (minwave-10 < stdwave) & (stdwave < maxwave+10)
stdwave = stdwave[ii]
stdflux = stdflux[:, ii]
log.info('Number of Standard Stars in this frame: {0:d}'.format(len(stars)))
if len(stars) == 0:
log.critical("No standard stars! Exiting")
sys.exit(1)
# Now for each star, find the best model and normalize.
normflux=[]
bestModelIndex=np.arange(len(stars))
templateID=np.arange(len(stars))
chi2dof=np.zeros(len(stars))
#- TODO: don't use 'l' as a variable name. Can look like a '1'
for k,l in enumerate(stars):
log.info("checking best model for star {0}".format(l[0]))
starindex=l[0]
mags=l[2]
filters=fibers["FILTER"][k]
rflux=stars[k][1]["r"][0]
bflux=stars[k][1]["b"][0]
zflux=stars[k][1]["z"][0]
flux={"b":bflux,"r":rflux,"z":zflux}
#print ivars
rivar=ivars[k][1]["r"][0]
bivar=ivars[k][1]["b"][0]
zivar=ivars[k][1]["z"][0]
ivar={"b":bivar,"r":rivar,"z":zivar}
resol_star={"r":frameResolution["r"][l[0]],"b":frameResolution["b"][l[0]],"z":frameResolution["z"][l[0]]}
# Now find the best Model
bestModelIndex[k],bestmodelWave,bestModelFlux,chi2dof[k]=match_templates(frameWave,flux,ivar,resol_star,stdwave,stdflux)
log.info('Star Fiber: {0}; Best Model Fiber: {1}; TemplateID: {2}; Chisq/dof: {3}'.format(l[0],bestModelIndex[k],templateid[bestModelIndex[k]],chi2dof[k]))
# Normalize the best model using reported magnitude
modelwave,normalizedflux=normalize_templates(stdwave,stdflux[bestModelIndex[k]],mags,filters,basepath)
normflux.append(normalizedflux)
# Now write the normalized flux for all best models to a file
normflux=np.array(normflux)
stdfibers=fibers["FIBER"]
data={}
data['BESTMODEL']=bestModelIndex
data['CHI2DOF']=chi2dof
data['TEMPLATEID']=templateid[bestModelIndex]
norm_model_file=args.outfile
io.write_stdstar_model(norm_model_file,normflux,stdwave,stdfibers,data)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--infile',
type=str,
default=None,
required=True,
help='path of DESI exposure frame fits file')
parser.add_argument('--fiberflat',
type=str,
default=None,
help='path of DESI fiberflat fits file')
parser.add_argument('--sky',
type=str,
default=None,
help='path of DESI sky fits file')
parser.add_argument('--calib',
type=str,
default=None,
help='path of DESI calibration fits file')
parser.add_argument('--outfile',
type=str,
default=None,
required=True,
help='path of DESI sky fits file')
# add calibration here when exists
args = parser.parse_args()
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is
None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
frame = read_frame(args.infile)
if args.fiberflat != None:
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
if args.sky != None:
log.info("subtract sky")
# read sky
skymodel = read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
if args.calib != None:
log.info("calibrate")
# read calibration
fluxcalib = read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
# save output
write_frame(args.outfile, frame)
log.info("successfully wrote %s" % args.outfile)
def load_all_s2n_values(nights, channel, sub_exposures=None):
"""
Calculate S/N values for a set of spectra from an input list of nights
Args:
nights: list
channel: str ('b','r','z')
sub_exposures:
Returns:
fdict: dict
Contains all the S/N info for all nights in the given channel
"""
fdict = dict(waves=[], s2n=[], fluxes=[], exptime=[], OII=[], objtype=[])
for night in nights:
if sub_exposures is not None:
exposures = sub_exposures
else:
exposures = get_exposures(night)#, raw=True)
for exposure in exposures:
fibermap_path = findfile(filetype='fibermap', night=night, expid=exposure)
fibermap_data = read_fibermap(fibermap_path)
flavor = fibermap_data.meta['FLAVOR']
if flavor.lower() in ('arc', 'flat', 'bias'):
log.debug('Skipping calibration {} exposure {:08d}'.format(flavor, exposure))
continue
# Load simspec
simspec_file = fibermap_path.replace('fibermap', 'simspec')
log.debug('Getting truth from {}'.format(simspec_file))
sps_hdu = fits.open(simspec_file)
sps_tab = Table(sps_hdu['TRUTH'].data,masked=True)
#- Get OIIFLUX from separate HDU and join
if ('OIIFLUX' not in sps_tab.colnames) and ('TRUTH_ELG' in sps_hdu):
elg_truth = Table(sps_hdu['TRUTH_ELG'].data)
sps_tab = join(sps_tab, elg_truth['TARGETID', 'OIIFLUX'],
keys='TARGETID', join_type='left')
else:
sps_tab['OIIFLUX'] = 0.0
sps_hdu.close()
#objs = sps_tab['TEMPLATETYPE'] == objtype
#if np.sum(objs) == 0:
# continue
# Load spectra (flux or not fluxed; should not matter)
for ii in range(10):
camera = channel+str(ii)
cframe_path = findfile(filetype='cframe', night=night, expid=exposure, camera=camera)
try:
log.debug('Reading from {}'.format(cframe_path))
cframe = read_frame(cframe_path)
except (IOError, OSError):
log.warn("Cannot find file: {:s}".format(cframe_path))
continue
# Calculate S/N per Ang
dwave = cframe.wave - np.roll(cframe.wave,1)
dwave[0] = dwave[1]
# Calculate
s2n = cframe.flux * np.sqrt(cframe.ivar) / np.sqrt(dwave)
#s2n = cframe.flux[iobjs,:] * np.sqrt(cframe.ivar[iobjs,:]) / np.sqrt(dwave)
# Save
fdict['objtype'].append(sps_tab['TEMPLATETYPE'].data[cframe.fibers])
fdict['waves'].append(cframe.wave)
fdict['s2n'].append(s2n)
fdict['fluxes'].append(sps_tab['MAG'].data[cframe.fibers])
fdict['OII'].append(sps_tab['OIIFLUX'].data[cframe.fibers])
fdict['exptime'].append(cframe.meta['EXPTIME'])
# Return
return fdict
ff_file_b3='/project/projectdirs/desi/spectro/desi_spectro_calib/trunk/spec/sp3/fiberflat-sm4-b-20191022.fits'
ff_file_r3='/project/projectdirs/desi/spectro/desi_spectro_calib/trunk/spec/sp3/fiberflat-sm4-r-20191022.fits'
ff_file_z3='/project/projectdirs/desi/spectro/desi_spectro_calib/trunk/spec/sp3/fiberflat-sm4-z-20191022.fits'
fff_b3=fitsio.FITS(ff_file_b3)
fff_r3=fitsio.FITS(ff_file_r3)
fff_z3=fitsio.FITS(ff_file_z3)
ff_b3=fff_b3['FIBERFLAT'][:,:]
ff_r3=fff_r3['FIBERFLAT'][:,:]
ff_z3=fff_z3['FIBERFLAT'][:,:]
frame_b3=read_frame('/project/projectdirs/desi/spectro/redux/v0/exposures/20191028/00022301/frame-b3-00022301.fits')
fiber=frame_b3.fibermap["FIBER"]-1500
x=np.zeros(fiber.size)
y=np.zeros(fiber.size)
for j,fib in enumerate(fiber) :
x[j]=table[xk][dico[fib]]
y[j]=table[yk][dico[fib]]
ff_b3_median=np.median(ff_b3,axis=1)
ff_r3_median=np.median(ff_r3,axis=1)
ff_z3_median=np.median(ff_z3,axis=1)
s1=30
contrast=2
plt.figure(0,figsize=(12,10))
def make_frameqa(self, make_plots=False, clobber=True):
""" Work through the Production and make QA for all frames
Parameters:
make_plots: bool, optional
Remake the plots too?
clobber: bool, optional
Returns:
"""
# imports
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.fiberflat import read_fiberflat
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots
from desispec.io.fluxcalibration import read_stdstar_models
# Loop on nights
path_nights = glob.glob(self.specprod_dir + '/exposures/*')
nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights]
for night in nights:
for exposure in get_exposures(night,
specprod_dir=self.specprod_dir):
# Object only??
frames_dict = get_files(filetype=str('frame'),
night=night,
expid=exposure,
specprod_dir=self.specprod_dir)
for camera, frame_fil in frames_dict.items():
# Load frame
frame = read_frame(frame_fil)
spectro = int(frame.meta['CAMERA'][-1])
if frame.meta['FLAVOR'] in ['flat', 'arc']:
qatype = 'qa_calib'
else:
qatype = 'qa_data'
qafile = meta.findfile(qatype,
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
if (not clobber) & os.path.isfile(qafile):
log.info(
"qafile={:s} exists. Not over-writing. Consider clobber=True"
.format(qafile))
continue
# Load
qaframe = load_qa_frame(qafile,
frame,
flavor=frame.meta['FLAVOR'])
# Flat QA
if frame.meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile(
'fiberflat',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
fiberflat = read_fiberflat(fiberflat_fil)
qaframe.run_qa('FIBERFLAT', (frame, fiberflat),
clobber=clobber)
if make_plots:
# Do it
qafig = meta.findfile(
'qa_flat_fig',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
qa_plots.frame_fiberflat(qafig, qaframe, frame,
fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
skymodel = read_sky(sky_fil)
qaframe.run_qa('SKYSUB', (frame, skymodel))
if make_plots:
qafig = meta.findfile(
'qa_sky_fig',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
qa_plots.frame_skyres(qafig, frame, skymodel,
qaframe)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile(
'stdstars',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir,
spectrograph=spectro)
model_tuple = read_stdstar_models(stdstar_fil)
flux_fil = meta.findfile(
'calib',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
fluxcalib = read_flux_calibration(flux_fil)
qaframe.run_qa(
'FLUXCALIB',
(frame, fluxcalib, model_tuple)) #, indiv_stars))
if make_plots:
qafig = meta.findfile(
'qa_flux_fig',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
qa_plots.frame_fluxcalib(qafig, qaframe, frame,
fluxcalib, model_tuple)
# Write
write_qa_frame(qafile, qaframe)
def get_skyres(cframes, sub_sky=False, flatten=True):
"""
Args:
cframes: str or list
Single cframe or a list of them
sub_sky: bool, optional
Subtract the sky? This should probably not be done
flatten: bool, optional
Return a flat, 1D array for each variable
combine: bool, optional
combine the individual sky fibers? Median 'smash'
Returns:
wave : ndarray
flux : ndarray
res : ndarray
ivar : ndarray
"""
from desispec.io import read_frame
from desispec.io.sky import read_sky
from desispec.sky import subtract_sky
if isinstance(cframes, list):
all_wave, all_flux, all_res, all_ivar = [], [], [], []
for cframe_file in cframes:
wave, flux, res, ivar = get_skyres(cframe_file, flatten=flatten)
# Save
all_wave.append(wave)
all_flux.append(flux)
all_res.append(res)
all_ivar.append(ivar)
# Concatenate -- Shape is preserved (nfibers, npix)
twave = np.concatenate(all_wave)
tflux = np.concatenate(all_flux)
tres = np.concatenate(all_res)
tivar = np.concatenate(all_ivar)
# Return
return twave, tflux, tres, tivar
cframe = read_frame(cframes, skip_resolution=True)
if cframe.meta['FLAVOR'] in ['flat', 'arc']:
raise ValueError("Bad flavor for exposure: {:s}".format(cframes))
# Sky
sky_file = cframes.replace('cframe', 'sky')
skymodel = read_sky(sky_file)
if sub_sky:
subtract_sky(cframe, skymodel)
# Resid
skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0]
res = cframe.flux[skyfibers] # Flux calibrated
ivar = cframe.ivar[skyfibers] # Flux calibrated
flux = skymodel.flux[skyfibers] # Residuals; not flux calibrated!
wave = np.outer(np.ones(flux.shape[0]), cframe.wave)
# Combine?
'''
if combine:
res = np.median(res, axis=0)
ivar = np.median(ivar, axis=0)
flux = np.median(flux, axis=0)
wave = np.median(wave, axis=0)
'''
# Return
if flatten:
return wave.flatten(), flux.flatten(), res.flatten(), ivar.flatten()
else:
return wave, flux, res, ivar
