def main() :
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--infile', type = str, default = None, required=True,
help = 'path of DESI exposure frame fits file')
parser.add_argument('--fiberflat', type = str, default = None,
help = 'path of DESI fiberflat fits file')
parser.add_argument('--sky', type = str, default = None,
help = 'path of DESI sky fits file')
parser.add_argument('--calib', type = str, default = None,
help = 'path of DESI calibration fits file')
parser.add_argument('--outfile', type = str, default = None, required=True,
help = 'path of DESI sky fits file')
# add calibration here when exists
args = parser.parse_args()
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
frame = read_frame(args.infile)
if args.fiberflat!=None :
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
if args.sky!=None :
log.info("subtract sky")
# read sky
skymodel=read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
if args.calib!=None :
log.info("calibrate")
# read calibration
fluxcalib=read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
# save output
write_frame(args.outfile, frame)
log.info("successfully wrote %s"%args.outfile)
def main(args):
log = get_logger()
frame=read_frame(args.infile, skip_resolution=True)
fibermap=read_fibermap(args.infile)
fiberflat=read_fiberflat(args.fiberflat)
skymodel=read_sky(args.sky)
fluxcalib=read_flux_calibration(args.calib)
cam=args.infile.split('/')[-1].split('-')[1]
band=cam[0]
bands=[band]
# Indices of sky fibers.
sky_indx = np.where(fibermap['OBJTYPE'] == 'SKY')[0]
rd_var, sky_var = calc_var(bands, args.nea, args.psf, frame, fluxcalib, fiberflat, skymodel, components=True)
var = calc_var(bands, args.nea, args.psf, frame, fluxcalib, fiberflat, skymodel, components=False)
nsky = 4
fig, axes = plt.subplots(1, nsky, figsize=(5 * nsky, 5))
for i in range(nsky):
def calc_alphavar(alpha):
return alpha * rd_var[sky_indx,:] + sky_var[sky_indx,:]
def alpha_fit(alpha):
_var = calc_alphavar(alpha)
ivar = 1. / _var
X2 = (frame.ivar[sky_indx,:] - ivar)**2.
return np.sum(X2)
res = minimize(alpha_fit, x0=[1.])
alpha = res.x[0]
indx = sky_indx[i]
axes[i].plot(skymodel.wave, median_filter(frame.ivar[indx,:], 10), lw=0.4, label='Sky frame IVAR', alpha=0.4)
axes[i].plot(skymodel.wave, 1./rd_var[indx,:], lw=0.4, label='Model rd. IVAR', alpha=0.4)
# axes[i].plot(skymodel.wave, 1./sky_var[indx,:], lw=0.4, label='Model Sky IVAR', alpha=0.4)
# axes[i].plot(skymodel.wave, 1./var[indx,:], lw=0.4, label=r'Model IVAR', alpha=0.4)
axes[i].plot(skymodel.wave, median_filter(1./calc_alphavar(alpha)[i,:], 10), lw=0.4, label=r'$\alpha$ Model IVAR', alpha=0.4)
axes[i].set_title(r'Fiber {:d} ($\alpha$ = {:.6f})'.format(indx, alpha))
axes[i].set_xlabel(r'Wavelength [$AA$]')
axes[i].set_yscale('log')
axes[i].set_ylim(bottom=5.e-4, top=3.e-2)
axes[i].legend(frameon=False, loc=2)
axes[0].set_ylabel('e/A')
pl.show()
def main(args):
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
frame = read_frame(args.infile)
if args.fiberflat!=None :
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
if args.sky!=None :
log.info("subtract sky")
# read sky
skymodel=read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
if args.calib!=None :
log.info("calibrate")
# read calibration
fluxcalib=read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
# save output
write_frame(args.outfile, frame, units='1e-17 erg/(s cm2 A)')
log.info("successfully wrote %s"%args.outfile)
def make_frameqa(self, make_plots=False, clobber=True):
""" Work through the Production and make QA for all frames
Parameters:
make_plots: bool, optional
Remake the plots too?
clobber: bool, optional
Returns:
"""
# imports
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.fiberflat import read_fiberflat
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots
from desispec.io.fluxcalibration import read_stdstar_models
# Loop on nights
path_nights = glob.glob(self.specprod_dir + '/exposures/*')
nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights]
for night in nights:
for exposure in get_exposures(night,
specprod_dir=self.specprod_dir):
# Object only??
frames_dict = get_files(filetype=str('frame'),
night=night,
expid=exposure,
specprod_dir=self.specprod_dir)
for camera, frame_fil in frames_dict.items():
# Load frame
frame = read_frame(frame_fil)
spectro = int(frame.meta['CAMERA'][-1])
if frame.meta['FLAVOR'] in ['flat', 'arc']:
qatype = 'qa_calib'
else:
qatype = 'qa_data'
qafile = meta.findfile(qatype,
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
if (not clobber) & os.path.isfile(qafile):
log.info(
"qafile={:s} exists. Not over-writing. Consider clobber=True"
.format(qafile))
continue
# Load
qaframe = load_qa_frame(qafile,
frame,
flavor=frame.meta['FLAVOR'])
# Flat QA
if frame.meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile(
'fiberflat',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
fiberflat = read_fiberflat(fiberflat_fil)
qaframe.run_qa('FIBERFLAT', (frame, fiberflat),
clobber=clobber)
if make_plots:
# Do it
qafig = meta.findfile(
'qa_flat_fig',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
qa_plots.frame_fiberflat(qafig, qaframe, frame,
fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
skymodel = read_sky(sky_fil)
qaframe.run_qa('SKYSUB', (frame, skymodel))
if make_plots:
qafig = meta.findfile(
'qa_sky_fig',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
qa_plots.frame_skyres(qafig, frame, skymodel,
qaframe)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile(
'stdstars',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir,
spectrograph=spectro)
model_tuple = read_stdstar_models(stdstar_fil)
flux_fil = meta.findfile(
'calib',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
fluxcalib = read_flux_calibration(flux_fil)
qaframe.run_qa(
'FLUXCALIB',
(frame, fluxcalib, model_tuple)) #, indiv_stars))
if make_plots:
qafig = meta.findfile(
'qa_flux_fig',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
qa_plots.frame_fluxcalib(qafig, qaframe, frame,
fluxcalib, model_tuple)
# Write
write_qa_frame(qafile, qaframe)
def make_frameqa(self, make_plots=False, clobber=True):
""" Work through the Production and make QA for all frames
Parameters:
make_plots: bool, optional
Remake the plots too?
clobber: bool, optional
Returns:
"""
# imports
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.fiberflat import read_fiberflat
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots
from desispec.io.fluxcalibration import read_stdstar_models
# Loop on nights
path_nights = glob.glob(self.specprod_dir+'/exposures/*')
nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights]
for night in nights:
for exposure in get_exposures(night, specprod_dir = self.specprod_dir):
# Object only??
frames_dict = get_files(filetype = str('frame'), night = night,
expid = exposure, specprod_dir = self.specprod_dir)
for camera,frame_fil in frames_dict.items():
# Load frame
frame = read_frame(frame_fil)
spectro = int(frame.meta['CAMERA'][-1])
if frame.meta['FLAVOR'] in ['flat','arc']:
qatype = 'qa_calib'
else:
qatype = 'qa_data'
qafile = meta.findfile(qatype, night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
if (not clobber) & os.path.isfile(qafile):
log.info("qafile={:s} exists. Not over-writing. Consider clobber=True".format(qafile))
continue
# Load
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame.meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
fiberflat = read_fiberflat(fiberflat_fil)
qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber)
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
skymodel = read_sky(sky_fil)
qaframe.run_qa('SKYSUB', (frame, skymodel))
if make_plots:
qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir,
spectrograph=spectro)
model_tuple=read_stdstar_models(stdstar_fil)
flux_fil = meta.findfile('calib', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
fluxcalib = read_flux_calibration(flux_fil)
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib, model_tuple))#, indiv_stars))
if make_plots:
qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib, model_tuple)
# Write
write_qa_frame(qafile, qaframe)
def qaframe_from_frame(frame_file,
specprod_dir=None,
make_plots=False,
qaprod_dir=None,
output_dir=None,
clobber=True):
""" Generate a qaframe object from an input frame_file name (and night)
Write QA to disk
Will also make plots if directed
Args:
frame_file: str
specprod_dir: str, optional
qa_dir: str, optional -- Location of QA
make_plots: bool, optional
output_dir: str, optional
Returns:
"""
import glob
import os
from desispec.io import read_frame
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.qa import qafile_from_framefile
from desispec.io.frame import search_for_framefile
from desispec.io.fiberflat import read_fiberflat
from desispec.fiberflat import apply_fiberflat
from desispec.qa import qa_plots
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots_ql
if '/' in frame_file: # If present, assume full path is used here
pass
else: # Find the frame file in the desispec hierarchy?
frame_file = search_for_framefile(frame_file)
# Load frame
frame = read_frame(frame_file)
frame_meta = frame.meta
night = frame_meta['NIGHT'].strip()
camera = frame_meta['CAMERA'].strip()
expid = frame_meta['EXPID']
spectro = int(frame_meta['CAMERA'][-1])
# Filename
qafile, qatype = qafile_from_framefile(frame_file,
qaprod_dir=qaprod_dir,
output_dir=output_dir)
if os.path.isfile(qafile) and (not clobber):
write = False
else:
write = True
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame_meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try: # Backwards compatibility
fiberflat = read_fiberflat(fiberflat_fil)
except FileNotFoundError:
fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
path, basen = os.path.split(fiberflat_fil)
path, _ = os.path.split(path)
fiberflat_fil = os.path.join(path, basen)
fiberflat = read_fiberflat(fiberflat_fil)
if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber):
write = True
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig',
night=night,
camera=camera,
expid=expid,
qaprod_dir=qaprod_dir,
specprod_dir=specprod_dir,
outdir=output_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
fiberflat_fil = meta.findfile('fiberflatnight',
night=night,
camera=camera)
if not os.path.exists(fiberflat_fil):
# Backwards compatibility (for now)
dummy_fiberflat_fil = meta.findfile(
'fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir) # This is dummy
path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
fiberflat_files = glob.glob(
os.path.join(path, '*', 'fiberflat-' + camera + '*.fits'))
if len(fiberflat_files) == 0:
path = path.replace('exposures', 'calib2d')
path, _ = os.path.split(path) # Remove night
fiberflat_files = glob.glob(
os.path.join(path, 'fiberflat-' + camera + '*.fits'))
# Sort and take the first (same as old pipeline)
fiberflat_files.sort()
fiberflat_fil = fiberflat_files[0]
fiberflat = read_fiberflat(fiberflat_fil)
apply_fiberflat(frame, fiberflat)
# Load sky model and run
try:
skymodel = read_sky(sky_fil)
except FileNotFoundError:
warnings.warn(
"Sky file {:s} not found. Skipping..".format(sky_fil))
else:
if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber):
write = True
if make_plots:
qafig = meta.findfile('qa_sky_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
qafig2 = meta.findfile('qa_skychi_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
#qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)
# S/N QA on cframe
if qatype == 'qa_data':
# cframe
cframe_file = frame_file.replace('frame-', 'cframe-')
cframe = read_frame(cframe_file)
if qaframe.run_qa('S2N', (cframe, ), clobber=clobber):
write = True
# Figure?
if make_plots:
s2n_dict = copy.deepcopy(qaframe.qa_data['S2N'])
qafig = meta.findfile('qa_s2n_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
#badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist()
#sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE')
coeff = s2n_dict['METRICS']['FITCOEFF_TGT'] #[sci_idx]
# Add an item or two for the QL method
s2n_dict['CAMERA'] = camera
s2n_dict['EXPID'] = expid
s2n_dict['PANAME'] = 'SNRFit'
s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA']
s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC']
objlist = s2n_dict['METRICS']['OBJLIST']
# Deal with YAML list instead of ndarray
s2n_dict['METRICS']['MEDIAN_SNR'] = np.array(
s2n_dict['METRICS']['MEDIAN_SNR'])
# Generate
if (not os.path.isfile(qafig)) or clobber:
qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist,
[[]] * len(objlist), coeff)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
spectrograph=spectro)
# try:
# model_tuple=read_stdstar_models(stdstar_fil)
# except FileNotFoundError:
# warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil))
# else:
flux_fil = meta.findfile('calib',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try:
fluxcalib = read_flux_calibration(flux_fil)
except FileNotFoundError:
warnings.warn(
"Flux file {:s} not found. Skipping..".format(flux_fil))
else:
if qaframe.run_qa(
'FLUXCALIB', (frame, fluxcalib),
clobber=clobber): # , model_tuple))#, indiv_stars))
write = True
if make_plots:
qafig = meta.findfile('qa_flux_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fluxcalib(qafig, qaframe, frame,
fluxcalib) # , model_tuple)
# Write
if write:
write_qa_frame(qafile, qaframe, verbose=True)
return qaframe
def qaframe_from_frame(frame_file,
specprod_dir=None,
make_plots=False,
qaprod_dir=None,
output_dir=None,
clobber=True):
""" Generate a qaframe object from an input frame_file name (and night)
Write QA to disk
Will also make plots if directed
Args:
frame_file: str
specprod_dir: str, optional
qa_dir: str, optional -- Location of QA
make_plots: bool, optional
output_dir: str, optional
Returns:
"""
import glob
import os
from desispec.io import read_frame
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.qa import qafile_from_framefile
from desispec.io.frame import search_for_framefile
from desispec.io.fiberflat import read_fiberflat
from desispec.fiberflat import apply_fiberflat
from desispec.qa import qa_plots
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
if '/' in frame_file: # If present, assume full path is used here
pass
else: # Find the frame file in the desispec hierarchy?
frame_file = search_for_framefile(frame_file)
# Load frame
frame = read_frame(frame_file)
frame_meta = frame.meta
night = frame_meta['NIGHT'].strip()
camera = frame_meta['CAMERA'].strip()
expid = frame_meta['EXPID']
spectro = int(frame_meta['CAMERA'][-1])
# Filename
qafile, qatype = qafile_from_framefile(frame_file,
qaprod_dir=qaprod_dir,
output_dir=output_dir)
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame_meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try: # Backwards compatibility
fiberflat = read_fiberflat(fiberflat_fil)
except FileNotFoundError:
fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
path, basen = os.path.split(fiberflat_fil)
path, _ = os.path.split(path)
fiberflat_fil = os.path.join(path, basen)
fiberflat = read_fiberflat(fiberflat_fil)
qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber)
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
fiberflat_fil = meta.findfile('fiberflatnight',
night=night,
camera=camera)
if not os.path.exists(fiberflat_fil):
# Backwards compatibility (for now)
dummy_fiberflat_fil = meta.findfile(
'fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir) # This is dummy
path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
fiberflat_files = glob.glob(
os.path.join(path, '*', 'fiberflat-' + camera + '*.fits'))
if len(fiberflat_files) == 0:
path = path.replace('exposures', 'calib2d')
path, _ = os.path.split(path) # Remove night
fiberflat_files = glob.glob(
os.path.join(path, 'fiberflat-' + camera + '*.fits'))
# Sort and take the first (same as old pipeline)
fiberflat_files.sort()
fiberflat_fil = fiberflat_files[0]
fiberflat = read_fiberflat(fiberflat_fil)
apply_fiberflat(frame, fiberflat)
# Load sky model and run
try:
skymodel = read_sky(sky_fil)
except FileNotFoundError:
warnings.warn(
"Sky file {:s} not found. Skipping..".format(sky_fil))
else:
qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber)
if make_plots:
qafig = meta.findfile('qa_sky_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qafig2 = meta.findfile('qa_skychi_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
#qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
spectrograph=spectro)
# try:
# model_tuple=read_stdstar_models(stdstar_fil)
# except FileNotFoundError:
# warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil))
# else:
flux_fil = meta.findfile('calib',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try:
fluxcalib = read_flux_calibration(flux_fil)
except FileNotFoundError:
warnings.warn(
"Flux file {:s} not found. Skipping..".format(flux_fil))
else:
qaframe.run_qa(
'FLUXCALIB',
(frame, fluxcalib)) # , model_tuple))#, indiv_stars))
if make_plots:
qafig = meta.findfile('qa_flux_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qa_plots.frame_fluxcalib(qafig, qaframe, frame,
fluxcalib) # , model_tuple)
# Write
write_qa_frame(qafile, qaframe, verbose=True)
return qaframe
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--infile',
type=str,
default=None,
required=True,
help='path of DESI exposure frame fits file')
parser.add_argument('--fiberflat',
type=str,
default=None,
help='path of DESI fiberflat fits file')
parser.add_argument('--sky',
type=str,
default=None,
help='path of DESI sky fits file')
parser.add_argument('--calib',
type=str,
default=None,
help='path of DESI calibration fits file')
parser.add_argument('--outfile',
type=str,
default=None,
required=True,
help='path of DESI sky fits file')
# add calibration here when exists
args = parser.parse_args()
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is
None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
frame = read_frame(args.infile)
if args.fiberflat != None:
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
if args.sky != None:
log.info("subtract sky")
# read sky
skymodel = read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
if args.calib != None:
log.info("calibrate")
# read calibration
fluxcalib = read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
# save output
write_frame(args.outfile, frame)
log.info("successfully wrote %s" % args.outfile)
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None,
output_dir=None, clobber=True):
""" Generate a qaframe object from an input frame_file name (and night)
Write QA to disk
Will also make plots if directed
Args:
frame_file: str
specprod_dir: str, optional
qa_dir: str, optional -- Location of QA
make_plots: bool, optional
output_dir: str, optional
Returns:
"""
import glob
import os
from desispec.io import read_frame
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.qa import qafile_from_framefile
from desispec.io.frame import search_for_framefile
from desispec.io.fiberflat import read_fiberflat
from desispec.fiberflat import apply_fiberflat
from desispec.qa import qa_plots
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots_ql
if '/' in frame_file: # If present, assume full path is used here
pass
else: # Find the frame file in the desispec hierarchy?
frame_file = search_for_framefile(frame_file)
# Load frame
frame = read_frame(frame_file)
frame_meta = frame.meta
night = frame_meta['NIGHT'].strip()
camera = frame_meta['CAMERA'].strip()
expid = frame_meta['EXPID']
spectro = int(frame_meta['CAMERA'][-1])
# Filename
qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir)
if os.path.isfile(qafile) and (not clobber):
write = False
else:
write = True
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame_meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir)
try: # Backwards compatibility
fiberflat = read_fiberflat(fiberflat_fil)
except FileNotFoundError:
fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
path, basen = os.path.split(fiberflat_fil)
path,_ = os.path.split(path)
fiberflat_fil = os.path.join(path, basen)
fiberflat = read_fiberflat(fiberflat_fil)
if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber):
write = True
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid,
qaprod_dir=qaprod_dir, specprod_dir=specprod_dir, outdir=output_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir)
fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera)
if not os.path.exists(fiberflat_fil):
# Backwards compatibility (for now)
dummy_fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir) # This is dummy
path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
fiberflat_files = glob.glob(os.path.join(path,'*','fiberflat-'+camera+'*.fits'))
if len(fiberflat_files) == 0:
path = path.replace('exposures', 'calib2d')
path,_ = os.path.split(path) # Remove night
fiberflat_files = glob.glob(os.path.join(path,'fiberflat-'+camera+'*.fits'))
# Sort and take the first (same as old pipeline)
fiberflat_files.sort()
fiberflat_fil = fiberflat_files[0]
fiberflat = read_fiberflat(fiberflat_fil)
apply_fiberflat(frame, fiberflat)
# Load sky model and run
try:
skymodel = read_sky(sky_fil)
except FileNotFoundError:
warnings.warn("Sky file {:s} not found. Skipping..".format(sky_fil))
else:
if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber):
write=True
if make_plots:
qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
#qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)
# S/N QA on cframe
if qatype == 'qa_data':
# cframe
cframe_file = frame_file.replace('frame-', 'cframe-')
cframe = read_frame(cframe_file)
if qaframe.run_qa('S2N', (cframe,), clobber=clobber):
write=True
# Figure?
if make_plots:
s2n_dict = copy.deepcopy(qaframe.qa_data['S2N'])
qafig = meta.findfile('qa_s2n_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
#badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist()
#sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE')
coeff = s2n_dict['METRICS']['FITCOEFF_TGT']#[sci_idx]
# Add an item or two for the QL method
s2n_dict['CAMERA'] = camera
s2n_dict['EXPID'] = expid
s2n_dict['PANAME'] = 'SNRFit'
s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA']
s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC']
objlist = s2n_dict['METRICS']['OBJLIST']
# Deal with YAML list instead of ndarray
s2n_dict['METRICS']['MEDIAN_SNR'] = np.array(s2n_dict['METRICS']['MEDIAN_SNR'])
# Generate
if (not os.path.isfile(qafig)) or clobber:
qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist, [[]]*len(objlist), coeff)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir,
spectrograph=spectro)
# try:
# model_tuple=read_stdstar_models(stdstar_fil)
# except FileNotFoundError:
# warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil))
# else:
flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir)
try:
fluxcalib = read_flux_calibration(flux_fil)
except FileNotFoundError:
warnings.warn("Flux file {:s} not found. Skipping..".format(flux_fil))
else:
if qaframe.run_qa('FLUXCALIB', (frame, fluxcalib), clobber=clobber): # , model_tuple))#, indiv_stars))
write = True
if make_plots:
qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple)
# Write
if write:
write_qa_frame(qafile, qaframe, verbose=True)
return qaframe
def main(args):
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is
None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
frame = read_frame(args.infile)
#- Raw scores already added in extraction, but just in case they weren't
#- it is harmless to rerun to make sure we have them.
compute_and_append_frame_scores(frame, suffix="RAW")
if args.cosmics_nsig > 0 and args.sky == None: # Reject cosmics (otherwise do it after sky subtraction)
log.info("cosmics ray 1D rejection")
reject_cosmic_rays_1d(frame, args.cosmics_nsig)
if args.fiberflat != None:
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to all fibers
apply_fiberflat(frame, fiberflat)
compute_and_append_frame_scores(frame, suffix="FFLAT")
if args.sky != None:
# read sky
skymodel = read_sky(args.sky)
if args.cosmics_nsig > 0:
# first subtract sky without throughput correction
subtract_sky(frame, skymodel, throughput_correction=False)
# then find cosmics
log.info("cosmics ray 1D rejection after sky subtraction")
reject_cosmic_rays_1d(frame, args.cosmics_nsig)
if args.sky_throughput_correction:
# and (re-)subtract sky, but just the correction term
subtract_sky(frame,
skymodel,
throughput_correction=True,
default_throughput_correction=0.)
else:
# subtract sky
subtract_sky(frame,
skymodel,
throughput_correction=args.sky_throughput_correction)
compute_and_append_frame_scores(frame, suffix="SKYSUB")
if args.calib != None:
log.info("calibrate")
# read calibration
fluxcalib = read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
compute_and_append_frame_scores(frame, suffix="CALIB")
# save output
write_frame(args.outfile, frame, units='1e-17 erg/(s cm2 Angstrom)')
log.info("successfully wrote %s" % args.outfile)
def main(args):
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
frame = read_frame(args.infile)
#- Raw scores already added in extraction, but just in case they weren't
#- it is harmless to rerun to make sure we have them.
compute_and_append_frame_scores(frame,suffix="RAW")
if args.cosmics_nsig>0 and args.sky==None : # Reject cosmics (otherwise do it after sky subtraction)
log.info("cosmics ray 1D rejection")
reject_cosmic_rays_1d(frame,args.cosmics_nsig)
if args.fiberflat!=None :
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to all fibers
apply_fiberflat(frame, fiberflat)
compute_and_append_frame_scores(frame,suffix="FFLAT")
if args.sky!=None :
# read sky
skymodel=read_sky(args.sky)
if args.cosmics_nsig>0 :
# use a copy the frame (not elegant but robust)
copied_frame = copy.deepcopy(frame)
# first subtract sky without throughput correction
subtract_sky(copied_frame, skymodel, apply_throughput_correction = False)
# then find cosmics
log.info("cosmics ray 1D rejection after sky subtraction")
reject_cosmic_rays_1d(copied_frame,args.cosmics_nsig)
# copy mask
frame.mask = copied_frame.mask
# and (re-)subtract sky, but just the correction term
subtract_sky(frame, skymodel, apply_throughput_correction = (not args.no_sky_throughput_correction) )
else :
# subtract sky
subtract_sky(frame, skymodel, apply_throughput_correction = (not args.no_sky_throughput_correction) )
compute_and_append_frame_scores(frame,suffix="SKYSUB")
if args.calib!=None :
log.info("calibrate")
# read calibration
fluxcalib=read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
# Ensure that ivars are set to 0 for all values if any designated
# fibermask bit is set. Also flips a bits for each frame.mask value using specmask.BADFIBER
frame = get_fiberbitmasked_frame(frame,bitmask="flux",ivar_framemask=True)
compute_and_append_frame_scores(frame,suffix="CALIB")
# save output
write_frame(args.outfile, frame, units='10**-17 erg/(s cm2 Angstrom)')
log.info("successfully wrote %s"%args.outfile)
def main(args):
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is
None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
if (not args.no_tsnr) and (args.calib is None):
log.critical(
'need --fiberflat --sky and --calib to compute template SNR')
sys.exit(12)
frame = read_frame(args.infile)
if not args.no_tsnr:
# tsnr alpha calc. requires uncalibrated + no substraction rame.
uncalibrated_frame = copy.deepcopy(frame)
#- Raw scores already added in extraction, but just in case they weren't
#- it is harmless to rerun to make sure we have them.
compute_and_append_frame_scores(frame, suffix="RAW")
if args.cosmics_nsig > 0 and args.sky == None: # Reject cosmics (otherwise do it after sky subtraction)
log.info("cosmics ray 1D rejection")
reject_cosmic_rays_1d(frame, args.cosmics_nsig)
if args.fiberflat != None:
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to all fibers
apply_fiberflat(frame, fiberflat)
compute_and_append_frame_scores(frame, suffix="FFLAT")
else:
fiberflat = None
if args.no_xtalk:
zero_ivar = (not args.no_zero_ivar)
else:
zero_ivar = False
if args.sky != None:
# read sky
skymodel = read_sky(args.sky)
if args.cosmics_nsig > 0:
# use a copy the frame (not elegant but robust)
copied_frame = copy.deepcopy(frame)
# first subtract sky without throughput correction
subtract_sky(copied_frame,
skymodel,
apply_throughput_correction=False,
zero_ivar=zero_ivar)
# then find cosmics
log.info("cosmics ray 1D rejection after sky subtraction")
reject_cosmic_rays_1d(copied_frame, args.cosmics_nsig)
# copy mask
frame.mask = copied_frame.mask
# and (re-)subtract sky, but just the correction term
subtract_sky(frame,
skymodel,
apply_throughput_correction=(
not args.no_sky_throughput_correction),
zero_ivar=zero_ivar)
else:
# subtract sky
subtract_sky(frame,
skymodel,
apply_throughput_correction=(
not args.no_sky_throughput_correction),
zero_ivar=zero_ivar)
compute_and_append_frame_scores(frame, suffix="SKYSUB")
if not args.no_xtalk:
log.info("fiber crosstalk correction")
correct_fiber_crosstalk(frame, fiberflat)
if not args.no_zero_ivar:
frame.ivar *= (frame.mask == 0)
if args.calib != None:
log.info("calibrate")
# read calibration
fluxcalib = read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
# Ensure that ivars are set to 0 for all values if any designated
# fibermask bit is set. Also flips a bits for each frame.mask value using specmask.BADFIBER
frame = get_fiberbitmasked_frame(
frame, bitmask="flux", ivar_framemask=(not args.no_zero_ivar))
compute_and_append_frame_scores(frame, suffix="CALIB")
if not args.no_tsnr:
log.info("calculating tsnr")
results, alpha = calc_tsnr2(uncalibrated_frame,
fiberflat=fiberflat,
skymodel=skymodel,
fluxcalib=fluxcalib,
alpha_only=args.alpha_only)
frame.meta['TSNRALPH'] = alpha
comments = {k: "from calc_frame_tsnr" for k in results.keys()}
append_frame_scores(frame, results, comments, overwrite=True)
# record inputs
frame.meta['IN_FRAME'] = shorten_filename(args.infile)
frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat)
frame.meta['IN_SKY'] = shorten_filename(args.sky)
frame.meta['IN_CALIB'] = shorten_filename(args.calib)
# save output
write_frame(args.outfile, frame, units='10**-17 erg/(s cm2 Angstrom)')
log.info("successfully wrote %s" % args.outfile)