def exposure_fiberflat(channel, expid, metric, outfile=None):
""" Generate an Exposure level plot of a FiberFlat metric
Args:
channel: str, e.g. 'b', 'r', 'z'
expid: int
metric: str, allowed entires are: ['meanflux']
Returns:
"""
from desispec.io.meta import find_exposure_night, findfile
from desispec.io.frame import read_meta_frame, read_frame
from desispec.io.fiberflat import read_fiberflat
from desimodel.focalplane import fiber_area_arcsec2
log = get_logger()
# Find exposure
night = find_exposure_night(expid)
# Search for frames with the input channel
frame0 = findfile('frame', camera=channel+'0', night=night, expid=expid)
if not os.path.exists(frame0):
log.fatal("No Frame 0 for channel={:s} and expid={:d}".format(channel, expid))
# Confirm frame is a Flat
fmeta = read_meta_frame(frame0)
assert fmeta['FLAVOR'].strip() == 'flat'
# Load up all the frames
x,y,metrics = [],[],[]
for wedge in range(10):
# Load
frame_file = findfile('frame', camera=channel+'{:d}'.format(wedge), night=night, expid=expid)
fiber_file = findfile('fiberflat', camera=channel+'{:d}'.format(wedge), night=night, expid=expid)
try:
frame = read_frame(frame_file)
except:
continue
else:
fiberflat = read_fiberflat(fiber_file)
fibermap = frame.fibermap
gdp = fiberflat.mask == 0
# X,Y
x.append([fibermap['X_TARGET']])
y.append([fibermap['Y_TARGET']])
area = fiber_area_arcsec2(x[-1], y[-1])
mean_area = np.mean(area)
# Metric
if metric == 'meanflux':
mean_norm = np.mean(fiberflat.fiberflat*gdp,axis=1) / (area / mean_area)
metrics.append([mean_norm])
# Cocatenate
x = np.concatenate(x)
y = np.concatenate(y)
metrics = np.concatenate(metrics)
# Plot
if outfile is None:
outfile='qa_{:08d}_{:s}_fiberflat.png'.format(expid, channel)
exposure_map(x,y,metrics, mlbl='Mean Flux',
title='Mean Flux for Exposure {:08d}, Channel {:s}'.format(expid, channel),
outfile=outfile)
def make_frameqa(self, make_plots=False, clobber=True):
""" Work through the Production and make QA for all frames
Parameters:
make_plots: bool, optional
Remake the plots too?
clobber: bool, optional
Returns:
"""
# imports
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.fiberflat import read_fiberflat
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots
from desispec.io.fluxcalibration import read_stdstar_models
# Loop on nights
path_nights = glob.glob(self.specprod_dir + '/exposures/*')
nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights]
for night in nights:
for exposure in get_exposures(night,
specprod_dir=self.specprod_dir):
# Object only??
frames_dict = get_files(filetype=str('frame'),
night=night,
expid=exposure,
specprod_dir=self.specprod_dir)
for camera, frame_fil in frames_dict.items():
# Load frame
frame = read_frame(frame_fil)
spectro = int(frame.meta['CAMERA'][-1])
if frame.meta['FLAVOR'] in ['flat', 'arc']:
qatype = 'qa_calib'
else:
qatype = 'qa_data'
qafile = meta.findfile(qatype,
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
if (not clobber) & os.path.isfile(qafile):
log.info(
"qafile={:s} exists. Not over-writing. Consider clobber=True"
.format(qafile))
continue
# Load
qaframe = load_qa_frame(qafile,
frame,
flavor=frame.meta['FLAVOR'])
# Flat QA
if frame.meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile(
'fiberflat',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
fiberflat = read_fiberflat(fiberflat_fil)
qaframe.run_qa('FIBERFLAT', (frame, fiberflat),
clobber=clobber)
if make_plots:
# Do it
qafig = meta.findfile(
'qa_flat_fig',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
qa_plots.frame_fiberflat(qafig, qaframe, frame,
fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
skymodel = read_sky(sky_fil)
qaframe.run_qa('SKYSUB', (frame, skymodel))
if make_plots:
qafig = meta.findfile(
'qa_sky_fig',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
qa_plots.frame_skyres(qafig, frame, skymodel,
qaframe)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile(
'stdstars',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir,
spectrograph=spectro)
model_tuple = read_stdstar_models(stdstar_fil)
flux_fil = meta.findfile(
'calib',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
fluxcalib = read_flux_calibration(flux_fil)
qaframe.run_qa(
'FLUXCALIB',
(frame, fluxcalib, model_tuple)) #, indiv_stars))
if make_plots:
qafig = meta.findfile(
'qa_flux_fig',
night=night,
camera=camera,
expid=exposure,
specprod_dir=self.specprod_dir)
qa_plots.frame_fluxcalib(qafig, qaframe, frame,
fluxcalib, model_tuple)
# Write
write_qa_frame(qafile, qaframe)
def setup_pipeline(config):
"""
Given a configuration from QLF, this sets up a pipeline [pa,qa] and also returns a
conversion dictionary from the configuration dictionary so that Pipeline steps (PA) can
take them. This is required for runpipeline.
"""
import astropy.io.fits as fits
import desispec.io.fibermap as fibIO
import desispec.io.sky as skyIO
import desispec.io.fiberflat as ffIO
import desispec.fiberflat as ff
import desispec.io.image as imIO
import desispec.image as im
import desispec.io.frame as frIO
import desispec.frame as dframe
from desispec.quicklook import procalgs
from desispec.boxcar import do_boxcar
qlog=qllogger.QLLogger("QuickLook",20)
log=qlog.getlog()
if config is None:
return None
log.info("Reading Configuration")
if "RawImage" not in config:
log.critical("Config is missing \"RawImage\" key.")
sys.exit("Missing \"RawImage\" key.")
inpname=config["RawImage"]
if "FiberMap" not in config:
log.critical("Config is missing \"FiberMap\" key.")
sys.exit("Missing \"FiberMap\" key.")
fibname=config["FiberMap"]
proctype="Exposure"
if "Camera" in config:
camera=config["Camera"]
if "DataType" in config:
proctype=config["DataType"]
debuglevel=20
if "DebugLevel" in config:
debuglevel=config["DebugLevel"]
log.setLevel(debuglevel)
hbeat=QLHB.QLHeartbeat(log,config["Period"],config["Timeout"])
if config["Timeout"]> 200.0:
log.warning("Heartbeat timeout exceeding 200.0 seconds")
dumpintermediates=False
if "DumpIntermediates" in config:
dumpintermediates=config["DumpIntermediates"]
biasimage=None #- This will be the converted dictionary key
biasfile=None
if "BiasImage" in config:
biasfile=config["BiasImage"]
darkimage=None
darkfile=None
if "DarkImage" in config:
darkfile=config["DarkImage"]
pixelflatfile=None
pixflatimage=None
if "PixelFlat" in config:
pixelflatfile=config["PixelFlat"]
fiberflatimagefile=None
fiberflatimage=None
if "FiberFlatImage" in config:
fiberflatimagefile=config["FiberFlatImage"]
arclampimagefile=None
arclampimage=None
if "ArcLampImage" in config:
arclampimagefile=config["ArcLampImage"]
fiberflatfile=None
fiberflat=None
if "FiberFlatFile" in config:
fiberflatfile=config["FiberFlatFile"]
skyfile=None
skyimage=None
if "SkyFile" in config:
skyfile=config["SkyFile"]
psf=None
if "PSFFile" in config:
#from specter.psf import load_psf
import desispec.psf
psf=desispec.psf.PSF(config["PSFFile"])
#psf=load_psf(config["PSFFile"])
if "basePath" in config:
basePath=config["basePath"]
hbeat.start("Reading input file %s"%inpname)
inp=fits.open(inpname) #- reading raw image directly from astropy.io.fits
hbeat.start("Reading fiberMap file %s"%fibname)
fibfile,fibhdr=fibIO.read_fibermap(fibname,header=True)
convdict={"FiberMap":fibfile}
if psf is not None:
convdict["PSFFile"]=psf
if biasfile is not None:
hbeat.start("Reading Bias Image %s"%biasfile)
biasimage=imIO.read_image(biasfile)
convdict["BiasImage"]=biasimage
if darkfile is not None:
hbeat.start("Reading Dark Image %s"%darkfile)
darkimage=imIO.read_image(darkfile)
convdict["DarkImage"]=darkimage
if pixelflatfile:
hbeat.start("Reading PixelFlat Image %s"%pixelflatfile)
pixelflatimage=imIO.read_image(pixelflatfile)
convdict["PixelFlat"]=pixelflatimage
if fiberflatimagefile:
hbeat.start("Reading FiberFlat Image %s"%fiberflatimagefile)
fiberflatimage=imIO.read_image(fiberflatimagefile)
convdict["FiberFlatImage"]=fiberflatimage
if arclampimagefile:
hbeat.start("Reading ArcLampImage %s"%arclampimagefile)
arclampimage=imIO.read_image(arclampimagefile)
convdict["ArcLampImage"]=arclampimage
if fiberflatfile:
hbeat.start("Reading FiberFlat %s"%fiberflatfile)
fiberflat=ffIO.read_fiberflat(fiberflatfile)
convdict["FiberFlatFile"]=fiberflat
if skyfile:
hbeat.start("Reading SkyModel file %s"%skyfile)
skymodel=skyIO.read_sky(skyfile)
convdict["SkyFile"]=skymodel
if dumpintermediates:
convdict["DumpIntermediates"]=dumpintermediates
hbeat.stop("Finished reading all static files")
img=inp
convdict["rawimage"]=img
pipeline=[]
for step in config["PipeLine"]:
pa=getobject(step["PA"],log)
if len(pipeline) == 0:
if not pa.is_compatible(type(img)):
log.critical("Pipeline configuration is incorrect! check configuration %s %s"%(img,pa.is_compatible(img)))
sys.exit("Wrong pipeline configuration")
else:
if not pa.is_compatible(pipeline[-1][0].get_output_type()):
log.critical("Pipeline configuration is incorrect! check configuration")
log.critical("Can't connect input of %s to output of %s. Incompatible types"%(pa.name,pipeline[-1][0].name))
sys.exit("Wrong pipeline configuration")
qas=[]
for q in step["QAs"]:
qa=getobject(q,log)
if not qa.is_compatible(pa.get_output_type()):
log.warning("QA %s can not be used for output of %s. Skipping expecting %s got %s %s"%(qa.name,pa.name,qa.__inpType__,pa.get_output_type(),qa.is_compatible(pa.get_output_type())))
else:
qas.append(qa)
pipeline.append([pa,qas])
return pipeline,convdict
def make_frameqa(self, make_plots=False, clobber=True):
""" Work through the Production and make QA for all frames
Parameters:
make_plots: bool, optional
Remake the plots too?
clobber: bool, optional
Returns:
"""
# imports
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.fiberflat import read_fiberflat
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots
from desispec.io.fluxcalibration import read_stdstar_models
# Loop on nights
path_nights = glob.glob(self.specprod_dir+'/exposures/*')
nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights]
for night in nights:
for exposure in get_exposures(night, specprod_dir = self.specprod_dir):
# Object only??
frames_dict = get_files(filetype = str('frame'), night = night,
expid = exposure, specprod_dir = self.specprod_dir)
for camera,frame_fil in frames_dict.items():
# Load frame
frame = read_frame(frame_fil)
spectro = int(frame.meta['CAMERA'][-1])
if frame.meta['FLAVOR'] in ['flat','arc']:
qatype = 'qa_calib'
else:
qatype = 'qa_data'
qafile = meta.findfile(qatype, night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
if (not clobber) & os.path.isfile(qafile):
log.info("qafile={:s} exists. Not over-writing. Consider clobber=True".format(qafile))
continue
# Load
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame.meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
fiberflat = read_fiberflat(fiberflat_fil)
qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber)
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
skymodel = read_sky(sky_fil)
qaframe.run_qa('SKYSUB', (frame, skymodel))
if make_plots:
qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir,
spectrograph=spectro)
model_tuple=read_stdstar_models(stdstar_fil)
flux_fil = meta.findfile('calib', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
fluxcalib = read_flux_calibration(flux_fil)
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib, model_tuple))#, indiv_stars))
if make_plots:
qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir)
qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib, model_tuple)
# Write
write_qa_frame(qafile, qaframe)
def qaframe_from_frame(frame_file,
specprod_dir=None,
make_plots=False,
qaprod_dir=None,
output_dir=None,
clobber=True):
""" Generate a qaframe object from an input frame_file name (and night)
Write QA to disk
Will also make plots if directed
Args:
frame_file: str
specprod_dir: str, optional
qa_dir: str, optional -- Location of QA
make_plots: bool, optional
output_dir: str, optional
Returns:
"""
import glob
import os
from desispec.io import read_frame
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.qa import qafile_from_framefile
from desispec.io.frame import search_for_framefile
from desispec.io.fiberflat import read_fiberflat
from desispec.fiberflat import apply_fiberflat
from desispec.qa import qa_plots
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots_ql
if '/' in frame_file: # If present, assume full path is used here
pass
else: # Find the frame file in the desispec hierarchy?
frame_file = search_for_framefile(frame_file)
# Load frame
frame = read_frame(frame_file)
frame_meta = frame.meta
night = frame_meta['NIGHT'].strip()
camera = frame_meta['CAMERA'].strip()
expid = frame_meta['EXPID']
spectro = int(frame_meta['CAMERA'][-1])
# Filename
qafile, qatype = qafile_from_framefile(frame_file,
qaprod_dir=qaprod_dir,
output_dir=output_dir)
if os.path.isfile(qafile) and (not clobber):
write = False
else:
write = True
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame_meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try: # Backwards compatibility
fiberflat = read_fiberflat(fiberflat_fil)
except FileNotFoundError:
fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
path, basen = os.path.split(fiberflat_fil)
path, _ = os.path.split(path)
fiberflat_fil = os.path.join(path, basen)
fiberflat = read_fiberflat(fiberflat_fil)
if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber):
write = True
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig',
night=night,
camera=camera,
expid=expid,
qaprod_dir=qaprod_dir,
specprod_dir=specprod_dir,
outdir=output_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
fiberflat_fil = meta.findfile('fiberflatnight',
night=night,
camera=camera)
if not os.path.exists(fiberflat_fil):
# Backwards compatibility (for now)
dummy_fiberflat_fil = meta.findfile(
'fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir) # This is dummy
path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
fiberflat_files = glob.glob(
os.path.join(path, '*', 'fiberflat-' + camera + '*.fits'))
if len(fiberflat_files) == 0:
path = path.replace('exposures', 'calib2d')
path, _ = os.path.split(path) # Remove night
fiberflat_files = glob.glob(
os.path.join(path, 'fiberflat-' + camera + '*.fits'))
# Sort and take the first (same as old pipeline)
fiberflat_files.sort()
fiberflat_fil = fiberflat_files[0]
fiberflat = read_fiberflat(fiberflat_fil)
apply_fiberflat(frame, fiberflat)
# Load sky model and run
try:
skymodel = read_sky(sky_fil)
except FileNotFoundError:
warnings.warn(
"Sky file {:s} not found. Skipping..".format(sky_fil))
else:
if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber):
write = True
if make_plots:
qafig = meta.findfile('qa_sky_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
qafig2 = meta.findfile('qa_skychi_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
#qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)
# S/N QA on cframe
if qatype == 'qa_data':
# cframe
cframe_file = frame_file.replace('frame-', 'cframe-')
cframe = read_frame(cframe_file)
if qaframe.run_qa('S2N', (cframe, ), clobber=clobber):
write = True
# Figure?
if make_plots:
s2n_dict = copy.deepcopy(qaframe.qa_data['S2N'])
qafig = meta.findfile('qa_s2n_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
#badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist()
#sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE')
coeff = s2n_dict['METRICS']['FITCOEFF_TGT'] #[sci_idx]
# Add an item or two for the QL method
s2n_dict['CAMERA'] = camera
s2n_dict['EXPID'] = expid
s2n_dict['PANAME'] = 'SNRFit'
s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA']
s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC']
objlist = s2n_dict['METRICS']['OBJLIST']
# Deal with YAML list instead of ndarray
s2n_dict['METRICS']['MEDIAN_SNR'] = np.array(
s2n_dict['METRICS']['MEDIAN_SNR'])
# Generate
if (not os.path.isfile(qafig)) or clobber:
qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist,
[[]] * len(objlist), coeff)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
spectrograph=spectro)
# try:
# model_tuple=read_stdstar_models(stdstar_fil)
# except FileNotFoundError:
# warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil))
# else:
flux_fil = meta.findfile('calib',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try:
fluxcalib = read_flux_calibration(flux_fil)
except FileNotFoundError:
warnings.warn(
"Flux file {:s} not found. Skipping..".format(flux_fil))
else:
if qaframe.run_qa(
'FLUXCALIB', (frame, fluxcalib),
clobber=clobber): # , model_tuple))#, indiv_stars))
write = True
if make_plots:
qafig = meta.findfile('qa_flux_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir,
qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fluxcalib(qafig, qaframe, frame,
fluxcalib) # , model_tuple)
# Write
if write:
write_qa_frame(qafile, qaframe, verbose=True)
return qaframe
def setup_pipeline(config):
"""
Given a configuration from QLF, this sets up a pipeline [pa,qa] and also returns a
conversion dictionary from the configuration dictionary so that Pipeline steps (PA) can
take them. This is required for runpipeline.
"""
import astropy.io.fits as fits
import desispec.io.fibermap as fibIO
import desispec.io.sky as skyIO
import desispec.io.fiberflat as ffIO
import desispec.fiberflat as ff
import desispec.io.image as imIO
import desispec.image as im
import desispec.io.frame as frIO
import desispec.frame as dframe
from desispec.quicklook import procalgs
from desispec.boxcar import do_boxcar
qlog=qllogger.QLLogger("QuickLook",20)
log=qlog.getlog()
if config is None:
return None
log.info("Reading Configuration")
if "RawImage" not in config:
log.critical("Config is missing \"RawImage\" key.")
sys.exit("Missing \"RawImage\" key.")
inpname=config["RawImage"]
if "FiberMap" not in config:
log.critical("Config is missing \"FiberMap\" key.")
sys.exit("Missing \"FiberMap\" key.")
fibname=config["FiberMap"]
proctype="Exposure"
if "Camera" in config:
camera=config["Camera"]
if "DataType" in config:
proctype=config["DataType"]
debuglevel=20
if "DebugLevel" in config:
debuglevel=config["DebugLevel"]
log.setLevel(debuglevel)
hbeat=QLHB.QLHeartbeat(log,config["Period"],config["Timeout"])
if config["Timeout"]> 200.0:
log.warning("Heartbeat timeout exceeding 200.0 seconds")
dumpintermediates=False
if "DumpIntermediates" in config:
dumpintermediates=config["DumpIntermediates"]
biasimage=None #- This will be the converted dictionary key
biasfile=None
if "BiasImage" in config:
biasfile=config["BiasImage"]
darkimage=None
darkfile=None
if "DarkImage" in config:
darkfile=config["DarkImage"]
pixelflatfile=None
pixflatimage=None
if "PixelFlat" in config:
pixelflatfile=config["PixelFlat"]
fiberflatimagefile=None
fiberflatimage=None
if "FiberFlatImage" in config:
fiberflatimagefile=config["FiberFlatImage"]
arclampimagefile=None
arclampimage=None
if "ArcLampImage" in config:
arclampimagefile=config["ArcLampImage"]
fiberflatfile=None
fiberflat=None
if "FiberFlatFile" in config:
if config["Flavor"] == 'arcs':
pass
else:
fiberflatfile=config["FiberFlatFile"]
skyfile=None
skyimage=None
if "SkyFile" in config:
skyfile=config["SkyFile"]
psf=None
if config["Flavor"] == 'arcs':
if not os.path.exists(os.path.join(os.environ['QL_SPEC_REDUX'],'calib2d','psf',config["Night"])):
os.mkdir(os.path.join(os.environ['QL_SPEC_REDUX'],'calib2d','psf',config["Night"]))
pass
elif "PSFFile" in config:
#from specter.psf import load_psf
import desispec.psf
psf=desispec.psf.PSF(config["PSFFile"])
#psf=load_psf(config["PSFFile"])
if "basePath" in config:
basePath=config["basePath"]
hbeat.start("Reading input file {}".format(inpname))
inp=fits.open(inpname) #- reading raw image directly from astropy.io.fits
hbeat.start("Reading fiberMap file {}".format(fibname))
fibfile=fibIO.read_fibermap(fibname)
fibhdr=fibfile.meta
convdict={"FiberMap":fibfile}
if psf is not None:
convdict["PSFFile"]=psf
if biasfile is not None:
hbeat.start("Reading Bias Image {}".format(biasfile))
biasimage=imIO.read_image(biasfile)
convdict["BiasImage"]=biasimage
if darkfile is not None:
hbeat.start("Reading Dark Image {}".format(darkfile))
darkimage=imIO.read_image(darkfile)
convdict["DarkImage"]=darkimage
if pixelflatfile:
hbeat.start("Reading PixelFlat Image {}".format(pixelflatfile))
pixelflatimage=imIO.read_image(pixelflatfile)
convdict["PixelFlat"]=pixelflatimage
if fiberflatimagefile:
hbeat.start("Reading FiberFlat Image {}".format(fiberflatimagefile))
fiberflatimage=imIO.read_image(fiberflatimagefile)
convdict["FiberFlatImage"]=fiberflatimage
if arclampimagefile:
hbeat.start("Reading ArcLampImage {}".format(arclampimagefile))
arclampimage=imIO.read_image(arclampimagefile)
convdict["ArcLampImage"]=arclampimage
if fiberflatfile:
hbeat.start("Reading FiberFlat {}".format(fiberflatfile))
fiberflat=ffIO.read_fiberflat(fiberflatfile)
convdict["FiberFlatFile"]=fiberflat
if skyfile:
hbeat.start("Reading SkyModel file {}".format(skyfile))
skymodel=skyIO.read_sky(skyfile)
convdict["SkyFile"]=skymodel
if dumpintermediates:
convdict["DumpIntermediates"]=dumpintermediates
hbeat.stop("Finished reading all static files")
img=inp
convdict["rawimage"]=img
pipeline=[]
for step in config["PipeLine"]:
pa=getobject(step["PA"],log)
if len(pipeline) == 0:
if not pa.is_compatible(type(img)):
log.critical("Pipeline configuration is incorrect! check configuration {} {}".format(img,pa.is_compatible(img)))
sys.exit("Wrong pipeline configuration")
else:
if not pa.is_compatible(pipeline[-1][0].get_output_type()):
log.critical("Pipeline configuration is incorrect! check configuration")
log.critical("Can't connect input of {} to output of {}. Incompatible types".format(pa.name,pipeline[-1][0].name))
sys.exit("Wrong pipeline configuration")
qas=[]
for q in step["QAs"]:
qa=getobject(q,log)
if not qa.is_compatible(pa.get_output_type()):
log.warning("QA {} can not be used for output of {}. Skipping expecting {} got {} {}".format(qa.name,pa.name,qa.__inpType__,pa.get_output_type(),qa.is_compatible(pa.get_output_type())))
else:
qas.append(qa)
pipeline.append([pa,qas])
return pipeline,convdict
def qaframe_from_frame(frame_file,
specprod_dir=None,
make_plots=False,
qaprod_dir=None,
output_dir=None,
clobber=True):
""" Generate a qaframe object from an input frame_file name (and night)
Write QA to disk
Will also make plots if directed
Args:
frame_file: str
specprod_dir: str, optional
qa_dir: str, optional -- Location of QA
make_plots: bool, optional
output_dir: str, optional
Returns:
"""
import glob
import os
from desispec.io import read_frame
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.qa import qafile_from_framefile
from desispec.io.frame import search_for_framefile
from desispec.io.fiberflat import read_fiberflat
from desispec.fiberflat import apply_fiberflat
from desispec.qa import qa_plots
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
if '/' in frame_file: # If present, assume full path is used here
pass
else: # Find the frame file in the desispec hierarchy?
frame_file = search_for_framefile(frame_file)
# Load frame
frame = read_frame(frame_file)
frame_meta = frame.meta
night = frame_meta['NIGHT'].strip()
camera = frame_meta['CAMERA'].strip()
expid = frame_meta['EXPID']
spectro = int(frame_meta['CAMERA'][-1])
# Filename
qafile, qatype = qafile_from_framefile(frame_file,
qaprod_dir=qaprod_dir,
output_dir=output_dir)
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame_meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try: # Backwards compatibility
fiberflat = read_fiberflat(fiberflat_fil)
except FileNotFoundError:
fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
path, basen = os.path.split(fiberflat_fil)
path, _ = os.path.split(path)
fiberflat_fil = os.path.join(path, basen)
fiberflat = read_fiberflat(fiberflat_fil)
qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber)
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
fiberflat_fil = meta.findfile('fiberflatnight',
night=night,
camera=camera)
if not os.path.exists(fiberflat_fil):
# Backwards compatibility (for now)
dummy_fiberflat_fil = meta.findfile(
'fiberflat',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir) # This is dummy
path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
fiberflat_files = glob.glob(
os.path.join(path, '*', 'fiberflat-' + camera + '*.fits'))
if len(fiberflat_files) == 0:
path = path.replace('exposures', 'calib2d')
path, _ = os.path.split(path) # Remove night
fiberflat_files = glob.glob(
os.path.join(path, 'fiberflat-' + camera + '*.fits'))
# Sort and take the first (same as old pipeline)
fiberflat_files.sort()
fiberflat_fil = fiberflat_files[0]
fiberflat = read_fiberflat(fiberflat_fil)
apply_fiberflat(frame, fiberflat)
# Load sky model and run
try:
skymodel = read_sky(sky_fil)
except FileNotFoundError:
warnings.warn(
"Sky file {:s} not found. Skipping..".format(sky_fil))
else:
qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber)
if make_plots:
qafig = meta.findfile('qa_sky_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qafig2 = meta.findfile('qa_skychi_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
#qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
spectrograph=spectro)
# try:
# model_tuple=read_stdstar_models(stdstar_fil)
# except FileNotFoundError:
# warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil))
# else:
flux_fil = meta.findfile('calib',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir)
try:
fluxcalib = read_flux_calibration(flux_fil)
except FileNotFoundError:
warnings.warn(
"Flux file {:s} not found. Skipping..".format(flux_fil))
else:
qaframe.run_qa(
'FLUXCALIB',
(frame, fluxcalib)) # , model_tuple))#, indiv_stars))
if make_plots:
qafig = meta.findfile('qa_flux_fig',
night=night,
camera=camera,
expid=expid,
specprod_dir=specprod_dir,
outdir=output_dir)
qa_plots.frame_fluxcalib(qafig, qaframe, frame,
fluxcalib) # , model_tuple)
# Write
write_qa_frame(qafile, qaframe, verbose=True)
return qaframe
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None,
output_dir=None, clobber=True):
""" Generate a qaframe object from an input frame_file name (and night)
Write QA to disk
Will also make plots if directed
Args:
frame_file: str
specprod_dir: str, optional
qa_dir: str, optional -- Location of QA
make_plots: bool, optional
output_dir: str, optional
Returns:
"""
import glob
import os
from desispec.io import read_frame
from desispec.io import meta
from desispec.io.qa import load_qa_frame, write_qa_frame
from desispec.io.qa import qafile_from_framefile
from desispec.io.frame import search_for_framefile
from desispec.io.fiberflat import read_fiberflat
from desispec.fiberflat import apply_fiberflat
from desispec.qa import qa_plots
from desispec.io.sky import read_sky
from desispec.io.fluxcalibration import read_flux_calibration
from desispec.qa import qa_plots_ql
if '/' in frame_file: # If present, assume full path is used here
pass
else: # Find the frame file in the desispec hierarchy?
frame_file = search_for_framefile(frame_file)
# Load frame
frame = read_frame(frame_file)
frame_meta = frame.meta
night = frame_meta['NIGHT'].strip()
camera = frame_meta['CAMERA'].strip()
expid = frame_meta['EXPID']
spectro = int(frame_meta['CAMERA'][-1])
# Filename
qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir)
if os.path.isfile(qafile) and (not clobber):
write = False
else:
write = True
qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
# Flat QA
if frame_meta['FLAVOR'] in ['flat']:
fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir)
try: # Backwards compatibility
fiberflat = read_fiberflat(fiberflat_fil)
except FileNotFoundError:
fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
path, basen = os.path.split(fiberflat_fil)
path,_ = os.path.split(path)
fiberflat_fil = os.path.join(path, basen)
fiberflat = read_fiberflat(fiberflat_fil)
if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber):
write = True
if make_plots:
# Do it
qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid,
qaprod_dir=qaprod_dir, specprod_dir=specprod_dir, outdir=output_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
# SkySub QA
if qatype == 'qa_data':
sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir)
fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera)
if not os.path.exists(fiberflat_fil):
# Backwards compatibility (for now)
dummy_fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir) # This is dummy
path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
fiberflat_files = glob.glob(os.path.join(path,'*','fiberflat-'+camera+'*.fits'))
if len(fiberflat_files) == 0:
path = path.replace('exposures', 'calib2d')
path,_ = os.path.split(path) # Remove night
fiberflat_files = glob.glob(os.path.join(path,'fiberflat-'+camera+'*.fits'))
# Sort and take the first (same as old pipeline)
fiberflat_files.sort()
fiberflat_fil = fiberflat_files[0]
fiberflat = read_fiberflat(fiberflat_fil)
apply_fiberflat(frame, fiberflat)
# Load sky model and run
try:
skymodel = read_sky(sky_fil)
except FileNotFoundError:
warnings.warn("Sky file {:s} not found. Skipping..".format(sky_fil))
else:
if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber):
write=True
if make_plots:
qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
#qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)
# S/N QA on cframe
if qatype == 'qa_data':
# cframe
cframe_file = frame_file.replace('frame-', 'cframe-')
cframe = read_frame(cframe_file)
if qaframe.run_qa('S2N', (cframe,), clobber=clobber):
write=True
# Figure?
if make_plots:
s2n_dict = copy.deepcopy(qaframe.qa_data['S2N'])
qafig = meta.findfile('qa_s2n_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
#badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist()
#sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE')
coeff = s2n_dict['METRICS']['FITCOEFF_TGT']#[sci_idx]
# Add an item or two for the QL method
s2n_dict['CAMERA'] = camera
s2n_dict['EXPID'] = expid
s2n_dict['PANAME'] = 'SNRFit'
s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA']
s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC']
objlist = s2n_dict['METRICS']['OBJLIST']
# Deal with YAML list instead of ndarray
s2n_dict['METRICS']['MEDIAN_SNR'] = np.array(s2n_dict['METRICS']['MEDIAN_SNR'])
# Generate
if (not os.path.isfile(qafig)) or clobber:
qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist, [[]]*len(objlist), coeff)
# FluxCalib QA
if qatype == 'qa_data':
# Standard stars
stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir,
spectrograph=spectro)
# try:
# model_tuple=read_stdstar_models(stdstar_fil)
# except FileNotFoundError:
# warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil))
# else:
flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir)
try:
fluxcalib = read_flux_calibration(flux_fil)
except FileNotFoundError:
warnings.warn("Flux file {:s} not found. Skipping..".format(flux_fil))
else:
if qaframe.run_qa('FLUXCALIB', (frame, fluxcalib), clobber=clobber): # , model_tuple))#, indiv_stars))
write = True
if make_plots:
qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid,
specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
if (not os.path.isfile(qafig)) or clobber:
qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple)
# Write
if write:
write_qa_frame(qafile, qaframe, verbose=True)
return qaframe