def get_skyres(cframes, sub_sky=False): from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.sky import subtract_sky if isinstance(cframes, list): all_wave, all_flux, all_res, all_ivar = [], [], [], [] for cframe_file in cframes: wave, flux, res, ivar = get_skyres(cframe_file) # Concatenate and return all_wave.append(wave) all_flux.append(flux) all_res.append(res) all_ivar.append(ivar) return np.concatenate(all_wave), np.concatenate(all_flux), \ np.concatenate(all_res), np.concatenate(all_ivar) cframe = read_frame(cframes) if cframe.meta['FLAVOR'] in ['flat', 'arc']: raise ValueError("Bad flavor for exposure: {:s}".format(cframes)) # Sky sky_file = cframes.replace('cframe', 'sky') skymodel = read_sky(sky_file) if sub_sky: subtract_sky(cframe, skymodel) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] ivar = cframe.ivar[skyfibers] flux = skymodel.flux[skyfibers] # Residuals wave = np.outer(np.ones(flux.shape[0]), cframe.wave) # Return return wave.flatten(), flux.flatten(), res.flatten(), ivar.flatten()
def run(self,*args,**kwargs): if len(args) == 0 : raise qlexceptions.ParameterException("Missing input parameter") if not self.is_compatible(type(args[0])): raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0]))) input_frame=args[0] #- this must be flat field applied before sky subtraction in the pipeline if "SkyFile" in kwargs: from desispec.io.sky import read_sky skyfile=kwargs["SkyFile"] #- Read sky model file itself from an argument log.info("Using given sky file %s for subtraction"%skyfile) skymodel=read_sky(skyfile) else: if "Outskyfile" in kwargs: outskyfile=kwargs["Outskyfile"] else: outskyfile=None log.info("No sky file given. Computing sky first") from desispec.quicklook.quicksky import compute_sky fibermap=input_frame.fibermap skymodel=compute_sky(input_frame,fibermap) if outskyfile is not None: from desispec.io.sky import write_sky log.info("writing an output sky model file %s "%outskyfile) write_sky(outputfile,skymodel,input_frame.meta) #- now do the subtraction return self.run_pa(input_frame,skymodel)
def run(self,*args,**kwargs): if len(args) == 0 : raise qlexceptions.ParameterException("Missing input parameter") if not self.is_compatible(type(args[0])): raise qlexceptions.ParameterException("Incompatible input. Was expecting %s got %s"%(type(self.__inpType__),type(args[0]))) if "SkyFile" not in kwargs: raise qlexceptions.ParameterException("Need Skymodel file") input_frame=args[0] #- this must be flat field applied before sky subtraction in the pipeline skyfile=kwargs["SkyFile"] #- Read sky model file itself from an argument from desispec.io.sky import read_sky skymodel=read_sky(skyfile) return self.run_pa(input_frame,skymodel)
def make_frameqa(self, make_plots=False, clobber=True): """ Work through the Production and make QA for all frames Parameters: make_plots: bool, optional Remake the plots too? clobber: bool, optional Returns: """ # imports from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.fiberflat import read_fiberflat from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots from desispec.io.fluxcalibration import read_stdstar_models # Loop on nights path_nights = glob.glob(self.specprod_dir + '/exposures/*') nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights] for night in nights: for exposure in get_exposures(night, specprod_dir=self.specprod_dir): # Object only?? frames_dict = get_files(filetype=str('frame'), night=night, expid=exposure, specprod_dir=self.specprod_dir) for camera, frame_fil in frames_dict.items(): # Load frame frame = read_frame(frame_fil) spectro = int(frame.meta['CAMERA'][-1]) if frame.meta['FLAVOR'] in ['flat', 'arc']: qatype = 'qa_calib' else: qatype = 'qa_data' qafile = meta.findfile(qatype, night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) if (not clobber) & os.path.isfile(qafile): log.info( "qafile={:s} exists. Not over-writing. Consider clobber=True" .format(qafile)) continue # Load qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame.meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile( 'fiberflat', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fiberflat = read_fiberflat(fiberflat_fil) qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber) if make_plots: # Do it qafig = meta.findfile( 'qa_flat_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) skymodel = read_sky(sky_fil) qaframe.run_qa('SKYSUB', (frame, skymodel)) if make_plots: qafig = meta.findfile( 'qa_sky_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile( 'stdstars', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir, spectrograph=spectro) model_tuple = read_stdstar_models(stdstar_fil) flux_fil = meta.findfile( 'calib', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fluxcalib = read_flux_calibration(flux_fil) qaframe.run_qa( 'FLUXCALIB', (frame, fluxcalib, model_tuple)) #, indiv_stars)) if make_plots: qafig = meta.findfile( 'qa_flux_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib, model_tuple) # Write write_qa_frame(qafile, qaframe)
def setup_pipeline(config): """ Given a configuration from QLF, this sets up a pipeline [pa,qa] and also returns a conversion dictionary from the configuration dictionary so that Pipeline steps (PA) can take them. This is required for runpipeline. """ import astropy.io.fits as fits import desispec.io.fibermap as fibIO import desispec.io.sky as skyIO import desispec.io.fiberflat as ffIO import desispec.fiberflat as ff import desispec.io.image as imIO import desispec.image as im import desispec.io.frame as frIO import desispec.frame as dframe from desispec.quicklook import procalgs from desispec.boxcar import do_boxcar qlog=qllogger.QLLogger("QuickLook",20) log=qlog.getlog() if config is None: return None log.info("Reading Configuration") if "RawImage" not in config: log.critical("Config is missing \"RawImage\" key.") sys.exit("Missing \"RawImage\" key.") inpname=config["RawImage"] if "FiberMap" not in config: log.critical("Config is missing \"FiberMap\" key.") sys.exit("Missing \"FiberMap\" key.") fibname=config["FiberMap"] proctype="Exposure" if "Camera" in config: camera=config["Camera"] if "DataType" in config: proctype=config["DataType"] debuglevel=20 if "DebugLevel" in config: debuglevel=config["DebugLevel"] log.setLevel(debuglevel) hbeat=QLHB.QLHeartbeat(log,config["Period"],config["Timeout"]) if config["Timeout"]> 200.0: log.warning("Heartbeat timeout exceeding 200.0 seconds") dumpintermediates=False if "DumpIntermediates" in config: dumpintermediates=config["DumpIntermediates"] biasimage=None #- This will be the converted dictionary key biasfile=None if "BiasImage" in config: biasfile=config["BiasImage"] darkimage=None darkfile=None if "DarkImage" in config: darkfile=config["DarkImage"] pixelflatfile=None pixflatimage=None if "PixelFlat" in config: pixelflatfile=config["PixelFlat"] fiberflatimagefile=None fiberflatimage=None if "FiberFlatImage" in config: fiberflatimagefile=config["FiberFlatImage"] arclampimagefile=None arclampimage=None if "ArcLampImage" in config: arclampimagefile=config["ArcLampImage"] fiberflatfile=None fiberflat=None if "FiberFlatFile" in config: fiberflatfile=config["FiberFlatFile"] skyfile=None skyimage=None if "SkyFile" in config: skyfile=config["SkyFile"] psf=None if "PSFFile" in config: #from specter.psf import load_psf import desispec.psf psf=desispec.psf.PSF(config["PSFFile"]) #psf=load_psf(config["PSFFile"]) if "basePath" in config: basePath=config["basePath"] hbeat.start("Reading input file %s"%inpname) inp=fits.open(inpname) #- reading raw image directly from astropy.io.fits hbeat.start("Reading fiberMap file %s"%fibname) fibfile,fibhdr=fibIO.read_fibermap(fibname,header=True) convdict={"FiberMap":fibfile} if psf is not None: convdict["PSFFile"]=psf if biasfile is not None: hbeat.start("Reading Bias Image %s"%biasfile) biasimage=imIO.read_image(biasfile) convdict["BiasImage"]=biasimage if darkfile is not None: hbeat.start("Reading Dark Image %s"%darkfile) darkimage=imIO.read_image(darkfile) convdict["DarkImage"]=darkimage if pixelflatfile: hbeat.start("Reading PixelFlat Image %s"%pixelflatfile) pixelflatimage=imIO.read_image(pixelflatfile) convdict["PixelFlat"]=pixelflatimage if fiberflatimagefile: hbeat.start("Reading FiberFlat Image %s"%fiberflatimagefile) fiberflatimage=imIO.read_image(fiberflatimagefile) convdict["FiberFlatImage"]=fiberflatimage if arclampimagefile: hbeat.start("Reading ArcLampImage %s"%arclampimagefile) arclampimage=imIO.read_image(arclampimagefile) convdict["ArcLampImage"]=arclampimage if fiberflatfile: hbeat.start("Reading FiberFlat %s"%fiberflatfile) fiberflat=ffIO.read_fiberflat(fiberflatfile) convdict["FiberFlatFile"]=fiberflat if skyfile: hbeat.start("Reading SkyModel file %s"%skyfile) skymodel=skyIO.read_sky(skyfile) convdict["SkyFile"]=skymodel if dumpintermediates: convdict["DumpIntermediates"]=dumpintermediates hbeat.stop("Finished reading all static files") img=inp convdict["rawimage"]=img pipeline=[] for step in config["PipeLine"]: pa=getobject(step["PA"],log) if len(pipeline) == 0: if not pa.is_compatible(type(img)): log.critical("Pipeline configuration is incorrect! check configuration %s %s"%(img,pa.is_compatible(img))) sys.exit("Wrong pipeline configuration") else: if not pa.is_compatible(pipeline[-1][0].get_output_type()): log.critical("Pipeline configuration is incorrect! check configuration") log.critical("Can't connect input of %s to output of %s. Incompatible types"%(pa.name,pipeline[-1][0].name)) sys.exit("Wrong pipeline configuration") qas=[] for q in step["QAs"]: qa=getobject(q,log) if not qa.is_compatible(pa.get_output_type()): log.warning("QA %s can not be used for output of %s. Skipping expecting %s got %s %s"%(qa.name,pa.name,qa.__inpType__,pa.get_output_type(),qa.is_compatible(pa.get_output_type()))) else: qas.append(qa) pipeline.append([pa,qas]) return pipeline,convdict
def make_frameqa(self, make_plots=False, clobber=True): """ Work through the Production and make QA for all frames Parameters: make_plots: bool, optional Remake the plots too? clobber: bool, optional Returns: """ # imports from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.fiberflat import read_fiberflat from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots from desispec.io.fluxcalibration import read_stdstar_models # Loop on nights path_nights = glob.glob(self.specprod_dir+'/exposures/*') nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights] for night in nights: for exposure in get_exposures(night, specprod_dir = self.specprod_dir): # Object only?? frames_dict = get_files(filetype = str('frame'), night = night, expid = exposure, specprod_dir = self.specprod_dir) for camera,frame_fil in frames_dict.items(): # Load frame frame = read_frame(frame_fil) spectro = int(frame.meta['CAMERA'][-1]) if frame.meta['FLAVOR'] in ['flat','arc']: qatype = 'qa_calib' else: qatype = 'qa_data' qafile = meta.findfile(qatype, night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) if (not clobber) & os.path.isfile(qafile): log.info("qafile={:s} exists. Not over-writing. Consider clobber=True".format(qafile)) continue # Load qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame.meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fiberflat = read_fiberflat(fiberflat_fil) qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber) if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) skymodel = read_sky(sky_fil) qaframe.run_qa('SKYSUB', (frame, skymodel)) if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir, spectrograph=spectro) model_tuple=read_stdstar_models(stdstar_fil) flux_fil = meta.findfile('calib', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fluxcalib = read_flux_calibration(flux_fil) qaframe.run_qa('FLUXCALIB', (frame, fluxcalib, model_tuple))#, indiv_stars)) if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib, model_tuple) # Write write_qa_frame(qafile, qaframe)
def main(args): # imports import glob from desispec.io import findfile from desispec.io import get_exposures from desispec.io import get_files from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.qa.qa_plots import skysub_resid import copy import pdb # Log log = get_logger() log.info("starting") # Exposures? if args.expids is not None: expids = [int(iarg) for iarg in args.expids.split(',')] else: expids = 'all' # Nights? if args.nights is not None: gdnights = [iarg for iarg in args.nights.split(',')] else: gdnights = 'all' # Channels? if args.channels is not None: gdchannels = [iarg for iarg in args.channels.split(',')] else: gdchannels = 'all' # Sky dict sky_dict = dict(wave=[], skyflux=[], res=[], count=0) channel_dict = dict( b=copy.deepcopy(sky_dict), r=copy.deepcopy(sky_dict), z=copy.deepcopy(sky_dict), ) # Loop on nights path_nights = glob.glob(args.specprod_dir + '/exposures/*') nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights] for night in nights: if gdnights == 'all': pass else: if night not in gdnights: continue # Get em for exposure in get_exposures(night, specprod_dir=args.specprod_dir): # Check against input expids if expids == 'all': pass else: if exposure not in expids: continue # Get em frames_dict = get_files(filetype=str('cframe'), night=night, expid=exposure, specprod_dir=args.specprod_dir) for camera, cframe_fil in frames_dict.items(): channel = camera[0] # Check against input if gdchannels == 'all': pass else: if channel not in gdchannels: continue # Load frame log.info('Loading {:s}'.format(cframe_fil)) cframe = read_frame(cframe_fil) if cframe.meta['FLAVOR'] in ['flat', 'arc']: # Probably can't happen continue # Sky sky_file = findfile(str('sky'), night=night, camera=camera, expid=exposure, specprod_dir=args.specprod_dir) skymodel = read_sky(sky_file) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] flux = skymodel.flux[skyfibers] # Residuals tmp = np.outer(np.ones(flux.shape[0]), cframe.wave) # Append #from xastropy.xutils import xdebug as xdb #xdb.set_trace() channel_dict[channel]['wave'].append(tmp.flatten()) channel_dict[channel]['skyflux'].append( np.log10(np.maximum(flux.flatten(), 1e-1))) channel_dict[channel]['res'].append(res.flatten()) channel_dict[channel]['count'] += 1 # Figure for channel in ['b', 'r', 'z']: if channel_dict[channel]['count'] > 0: sky_wave = np.concatenate(channel_dict[channel]['wave']) sky_flux = np.concatenate(channel_dict[channel]['skyflux']) sky_res = np.concatenate(channel_dict[channel]['res']) # Plot skysub_resid(sky_wave, sky_flux, sky_res, outfile='tmp{:s}.png'.format(channel))
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None, output_dir=None, clobber=True): """ Generate a qaframe object from an input frame_file name (and night) Write QA to disk Will also make plots if directed Args: frame_file: str specprod_dir: str, optional qa_dir: str, optional -- Location of QA make_plots: bool, optional output_dir: str, optional Returns: """ import glob import os from desispec.io import read_frame from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.qa import qafile_from_framefile from desispec.io.frame import search_for_framefile from desispec.io.fiberflat import read_fiberflat from desispec.fiberflat import apply_fiberflat from desispec.qa import qa_plots from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots_ql if '/' in frame_file: # If present, assume full path is used here pass else: # Find the frame file in the desispec hierarchy? frame_file = search_for_framefile(frame_file) # Load frame frame = read_frame(frame_file) frame_meta = frame.meta night = frame_meta['NIGHT'].strip() camera = frame_meta['CAMERA'].strip() expid = frame_meta['EXPID'] spectro = int(frame_meta['CAMERA'][-1]) # Filename qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir) if os.path.isfile(qafile) and (not clobber): write = False else: write = True qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame_meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: # Backwards compatibility fiberflat = read_fiberflat(fiberflat_fil) except FileNotFoundError: fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d') path, basen = os.path.split(fiberflat_fil) path, _ = os.path.split(path) fiberflat_fil = os.path.join(path, basen) fiberflat = read_fiberflat(fiberflat_fil) if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber): write = True if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid, qaprod_dir=qaprod_dir, specprod_dir=specprod_dir, outdir=output_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera) if not os.path.exists(fiberflat_fil): # Backwards compatibility (for now) dummy_fiberflat_fil = meta.findfile( 'fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) # This is dummy path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil)) fiberflat_files = glob.glob( os.path.join(path, '*', 'fiberflat-' + camera + '*.fits')) if len(fiberflat_files) == 0: path = path.replace('exposures', 'calib2d') path, _ = os.path.split(path) # Remove night fiberflat_files = glob.glob( os.path.join(path, 'fiberflat-' + camera + '*.fits')) # Sort and take the first (same as old pipeline) fiberflat_files.sort() fiberflat_fil = fiberflat_files[0] fiberflat = read_fiberflat(fiberflat_fil) apply_fiberflat(frame, fiberflat) # Load sky model and run try: skymodel = read_sky(sky_fil) except FileNotFoundError: warnings.warn( "Sky file {:s} not found. Skipping..".format(sky_fil)) else: if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber): write = True if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) #qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe) # S/N QA on cframe if qatype == 'qa_data': # cframe cframe_file = frame_file.replace('frame-', 'cframe-') cframe = read_frame(cframe_file) if qaframe.run_qa('S2N', (cframe, ), clobber=clobber): write = True # Figure? if make_plots: s2n_dict = copy.deepcopy(qaframe.qa_data['S2N']) qafig = meta.findfile('qa_s2n_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) #badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist() #sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE') coeff = s2n_dict['METRICS']['FITCOEFF_TGT'] #[sci_idx] # Add an item or two for the QL method s2n_dict['CAMERA'] = camera s2n_dict['EXPID'] = expid s2n_dict['PANAME'] = 'SNRFit' s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA'] s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC'] objlist = s2n_dict['METRICS']['OBJLIST'] # Deal with YAML list instead of ndarray s2n_dict['METRICS']['MEDIAN_SNR'] = np.array( s2n_dict['METRICS']['MEDIAN_SNR']) # Generate if (not os.path.isfile(qafig)) or clobber: qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist, [[]] * len(objlist), coeff) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, spectrograph=spectro) # try: # model_tuple=read_stdstar_models(stdstar_fil) # except FileNotFoundError: # warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil)) # else: flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: fluxcalib = read_flux_calibration(flux_fil) except FileNotFoundError: warnings.warn( "Flux file {:s} not found. Skipping..".format(flux_fil)) else: if qaframe.run_qa( 'FLUXCALIB', (frame, fluxcalib), clobber=clobber): # , model_tuple))#, indiv_stars)) write = True if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple) # Write if write: write_qa_frame(qafile, qaframe, verbose=True) return qaframe
def setup_pipeline(config): """ Given a configuration from QLF, this sets up a pipeline [pa,qa] and also returns a conversion dictionary from the configuration dictionary so that Pipeline steps (PA) can take them. This is required for runpipeline. """ import astropy.io.fits as fits import desispec.io.fibermap as fibIO import desispec.io.sky as skyIO import desispec.io.fiberflat as ffIO import desispec.fiberflat as ff import desispec.io.image as imIO import desispec.image as im import desispec.io.frame as frIO import desispec.frame as dframe from desispec.quicklook import procalgs from desispec.boxcar import do_boxcar qlog=qllogger.QLLogger("QuickLook",20) log=qlog.getlog() if config is None: return None log.info("Reading Configuration") if "RawImage" not in config: log.critical("Config is missing \"RawImage\" key.") sys.exit("Missing \"RawImage\" key.") inpname=config["RawImage"] if "FiberMap" not in config: log.critical("Config is missing \"FiberMap\" key.") sys.exit("Missing \"FiberMap\" key.") fibname=config["FiberMap"] proctype="Exposure" if "Camera" in config: camera=config["Camera"] if "DataType" in config: proctype=config["DataType"] debuglevel=20 if "DebugLevel" in config: debuglevel=config["DebugLevel"] log.setLevel(debuglevel) hbeat=QLHB.QLHeartbeat(log,config["Period"],config["Timeout"]) if config["Timeout"]> 200.0: log.warning("Heartbeat timeout exceeding 200.0 seconds") dumpintermediates=False if "DumpIntermediates" in config: dumpintermediates=config["DumpIntermediates"] biasimage=None #- This will be the converted dictionary key biasfile=None if "BiasImage" in config: biasfile=config["BiasImage"] darkimage=None darkfile=None if "DarkImage" in config: darkfile=config["DarkImage"] pixelflatfile=None pixflatimage=None if "PixelFlat" in config: pixelflatfile=config["PixelFlat"] fiberflatimagefile=None fiberflatimage=None if "FiberFlatImage" in config: fiberflatimagefile=config["FiberFlatImage"] arclampimagefile=None arclampimage=None if "ArcLampImage" in config: arclampimagefile=config["ArcLampImage"] fiberflatfile=None fiberflat=None if "FiberFlatFile" in config: if config["Flavor"] == 'arcs': pass else: fiberflatfile=config["FiberFlatFile"] skyfile=None skyimage=None if "SkyFile" in config: skyfile=config["SkyFile"] psf=None if config["Flavor"] == 'arcs': if not os.path.exists(os.path.join(os.environ['QL_SPEC_REDUX'],'calib2d','psf',config["Night"])): os.mkdir(os.path.join(os.environ['QL_SPEC_REDUX'],'calib2d','psf',config["Night"])) pass elif "PSFFile" in config: #from specter.psf import load_psf import desispec.psf psf=desispec.psf.PSF(config["PSFFile"]) #psf=load_psf(config["PSFFile"]) if "basePath" in config: basePath=config["basePath"] hbeat.start("Reading input file {}".format(inpname)) inp=fits.open(inpname) #- reading raw image directly from astropy.io.fits hbeat.start("Reading fiberMap file {}".format(fibname)) fibfile=fibIO.read_fibermap(fibname) fibhdr=fibfile.meta convdict={"FiberMap":fibfile} if psf is not None: convdict["PSFFile"]=psf if biasfile is not None: hbeat.start("Reading Bias Image {}".format(biasfile)) biasimage=imIO.read_image(biasfile) convdict["BiasImage"]=biasimage if darkfile is not None: hbeat.start("Reading Dark Image {}".format(darkfile)) darkimage=imIO.read_image(darkfile) convdict["DarkImage"]=darkimage if pixelflatfile: hbeat.start("Reading PixelFlat Image {}".format(pixelflatfile)) pixelflatimage=imIO.read_image(pixelflatfile) convdict["PixelFlat"]=pixelflatimage if fiberflatimagefile: hbeat.start("Reading FiberFlat Image {}".format(fiberflatimagefile)) fiberflatimage=imIO.read_image(fiberflatimagefile) convdict["FiberFlatImage"]=fiberflatimage if arclampimagefile: hbeat.start("Reading ArcLampImage {}".format(arclampimagefile)) arclampimage=imIO.read_image(arclampimagefile) convdict["ArcLampImage"]=arclampimage if fiberflatfile: hbeat.start("Reading FiberFlat {}".format(fiberflatfile)) fiberflat=ffIO.read_fiberflat(fiberflatfile) convdict["FiberFlatFile"]=fiberflat if skyfile: hbeat.start("Reading SkyModel file {}".format(skyfile)) skymodel=skyIO.read_sky(skyfile) convdict["SkyFile"]=skymodel if dumpintermediates: convdict["DumpIntermediates"]=dumpintermediates hbeat.stop("Finished reading all static files") img=inp convdict["rawimage"]=img pipeline=[] for step in config["PipeLine"]: pa=getobject(step["PA"],log) if len(pipeline) == 0: if not pa.is_compatible(type(img)): log.critical("Pipeline configuration is incorrect! check configuration {} {}".format(img,pa.is_compatible(img))) sys.exit("Wrong pipeline configuration") else: if not pa.is_compatible(pipeline[-1][0].get_output_type()): log.critical("Pipeline configuration is incorrect! check configuration") log.critical("Can't connect input of {} to output of {}. Incompatible types".format(pa.name,pipeline[-1][0].name)) sys.exit("Wrong pipeline configuration") qas=[] for q in step["QAs"]: qa=getobject(q,log) if not qa.is_compatible(pa.get_output_type()): log.warning("QA {} can not be used for output of {}. Skipping expecting {} got {} {}".format(qa.name,pa.name,qa.__inpType__,pa.get_output_type(),qa.is_compatible(pa.get_output_type()))) else: qas.append(qa) pipeline.append([pa,qas]) return pipeline,convdict
def get_skyres(cframes, sub_sky=False, flatten=True): """ Args: cframes: str or list Single cframe or a list of them sub_sky: bool, optional Subtract the sky? This should probably not be done flatten: bool, optional Return a flat, 1D array for each variable combine: bool, optional combine the individual sky fibers? Median 'smash' Returns: wave : ndarray flux : ndarray res : ndarray ivar : ndarray """ from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.sky import subtract_sky if isinstance(cframes, list): all_wave, all_flux, all_res, all_ivar = [], [], [], [] for cframe_file in cframes: wave, flux, res, ivar = get_skyres(cframe_file, flatten=flatten) # Save all_wave.append(wave) all_flux.append(flux) all_res.append(res) all_ivar.append(ivar) # Concatenate -- Shape is preserved (nfibers, npix) twave = np.concatenate(all_wave) tflux = np.concatenate(all_flux) tres = np.concatenate(all_res) tivar = np.concatenate(all_ivar) # Return return twave, tflux, tres, tivar cframe = read_frame(cframes, skip_resolution=True) if cframe.meta['FLAVOR'] in ['flat', 'arc']: raise ValueError("Bad flavor for exposure: {:s}".format(cframes)) # Sky sky_file = cframes.replace('cframe', 'sky') skymodel = read_sky(sky_file) if sub_sky: subtract_sky(cframe, skymodel) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] # Flux calibrated ivar = cframe.ivar[skyfibers] # Flux calibrated flux = skymodel.flux[skyfibers] # Residuals; not flux calibrated! wave = np.outer(np.ones(flux.shape[0]), cframe.wave) # Combine? ''' if combine: res = np.median(res, axis=0) ivar = np.median(ivar, axis=0) flux = np.median(flux, axis=0) wave = np.median(wave, axis=0) ''' # Return if flatten: return wave.flatten(), flux.flatten(), res.flatten(), ivar.flatten() else: return wave, flux, res, ivar
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None, output_dir=None, clobber=True): """ Generate a qaframe object from an input frame_file name (and night) Write QA to disk Will also make plots if directed Args: frame_file: str specprod_dir: str, optional qa_dir: str, optional -- Location of QA make_plots: bool, optional output_dir: str, optional Returns: """ import glob import os from desispec.io import read_frame from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.qa import qafile_from_framefile from desispec.io.frame import search_for_framefile from desispec.io.fiberflat import read_fiberflat from desispec.fiberflat import apply_fiberflat from desispec.qa import qa_plots from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration if '/' in frame_file: # If present, assume full path is used here pass else: # Find the frame file in the desispec hierarchy? frame_file = search_for_framefile(frame_file) # Load frame frame = read_frame(frame_file) frame_meta = frame.meta night = frame_meta['NIGHT'].strip() camera = frame_meta['CAMERA'].strip() expid = frame_meta['EXPID'] spectro = int(frame_meta['CAMERA'][-1]) # Filename qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir) qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame_meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: # Backwards compatibility fiberflat = read_fiberflat(fiberflat_fil) except FileNotFoundError: fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d') path, basen = os.path.split(fiberflat_fil) path, _ = os.path.split(path) fiberflat_fil = os.path.join(path, basen) fiberflat = read_fiberflat(fiberflat_fil) qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber) if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera) if not os.path.exists(fiberflat_fil): # Backwards compatibility (for now) dummy_fiberflat_fil = meta.findfile( 'fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) # This is dummy path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil)) fiberflat_files = glob.glob( os.path.join(path, '*', 'fiberflat-' + camera + '*.fits')) if len(fiberflat_files) == 0: path = path.replace('exposures', 'calib2d') path, _ = os.path.split(path) # Remove night fiberflat_files = glob.glob( os.path.join(path, 'fiberflat-' + camera + '*.fits')) # Sort and take the first (same as old pipeline) fiberflat_files.sort() fiberflat_fil = fiberflat_files[0] fiberflat = read_fiberflat(fiberflat_fil) apply_fiberflat(frame, fiberflat) # Load sky model and run try: skymodel = read_sky(sky_fil) except FileNotFoundError: warnings.warn( "Sky file {:s} not found. Skipping..".format(sky_fil)) else: qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber) if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) #qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, spectrograph=spectro) # try: # model_tuple=read_stdstar_models(stdstar_fil) # except FileNotFoundError: # warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil)) # else: flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: fluxcalib = read_flux_calibration(flux_fil) except FileNotFoundError: warnings.warn( "Flux file {:s} not found. Skipping..".format(flux_fil)) else: qaframe.run_qa( 'FLUXCALIB', (frame, fluxcalib)) # , model_tuple))#, indiv_stars)) if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple) # Write write_qa_frame(qafile, qaframe, verbose=True) return qaframe
def main(args) : # imports import glob from desispec.io import findfile from desispec.io import get_exposures from desispec.io import get_files from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.qa.qa_plots import skysub_resid import copy import pdb # Log log=get_logger() log.info("starting") # Exposures? if args.expids is not None: expids = [int(iarg) for iarg in args.expids.split(',')] else: expids = 'all' # Nights? if args.nights is not None: gdnights = [iarg for iarg in args.nights.split(',')] else: gdnights = 'all' # Channels? if args.channels is not None: gdchannels = [iarg for iarg in args.channels.split(',')] else: gdchannels = 'all' # Sky dict sky_dict = dict(wave=[], skyflux=[], res=[], count=0) channel_dict = dict(b=copy.deepcopy(sky_dict), r=copy.deepcopy(sky_dict), z=copy.deepcopy(sky_dict), ) # Loop on nights path_nights = glob.glob(args.specprod_dir+'/exposures/*') nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights] for night in nights: if gdnights == 'all': pass else: if night not in gdnights: continue # Get em for exposure in get_exposures(night, specprod_dir = args.specprod_dir): # Check against input expids if expids == 'all': pass else: if exposure not in expids: continue # Get em frames_dict = get_files(filetype=str('cframe'), night=night, expid=exposure, specprod_dir=args.specprod_dir) for camera, cframe_fil in frames_dict.items(): channel = camera[0] # Check against input if gdchannels == 'all': pass else: if channel not in gdchannels: continue # Load frame log.info('Loading {:s}'.format(cframe_fil)) cframe = read_frame(cframe_fil) if cframe.meta['FLAVOR'] in ['flat','arc']: # Probably can't happen continue # Sky sky_file = findfile(str('sky'), night=night, camera=camera, expid=exposure, specprod_dir=args.specprod_dir) skymodel = read_sky(sky_file) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] flux = skymodel.flux[skyfibers] # Residuals tmp = np.outer(np.ones(flux.shape[0]), cframe.wave) # Append #from xastropy.xutils import xdebug as xdb #xdb.set_trace() channel_dict[channel]['wave'].append(tmp.flatten()) channel_dict[channel]['skyflux'].append( np.log10(np.maximum(flux.flatten(),1e-1))) channel_dict[channel]['res'].append(res.flatten()) channel_dict[channel]['count'] += 1 # Figure for channel in ['b', 'r', 'z']: if channel_dict[channel]['count'] > 0: sky_wave = np.concatenate(channel_dict[channel]['wave']) sky_flux = np.concatenate(channel_dict[channel]['skyflux']) sky_res = np.concatenate(channel_dict[channel]['res']) # Plot skysub_resid(sky_wave, sky_flux, sky_res, outfile='tmp{:s}.png'.format(channel))
def get_skyres(cframes, sub_sky=False, flatten=True): """ Args: cframes: str or list Single cframe or a list of them sub_sky: bool, optional Subtract the sky? This should probably not be done flatten: bool, optional Return a flat, 1D array for each variable combine: bool, optional combine the individual sky fibers? Median 'smash' Returns: wave : ndarray flux : ndarray res : ndarray ivar : ndarray """ from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.sky import subtract_sky if isinstance(cframes,list): all_wave, all_flux, all_res, all_ivar = [], [], [], [] for cframe_file in cframes: wave, flux, res, ivar = get_skyres(cframe_file, flatten=flatten) # Save all_wave.append(wave) all_flux.append(flux) all_res.append(res) all_ivar.append(ivar) # Concatenate -- Shape is preserved (nfibers, npix) twave = np.concatenate(all_wave) tflux = np.concatenate(all_flux) tres = np.concatenate(all_res) tivar = np.concatenate(all_ivar) # Return return twave, tflux, tres, tivar cframe = read_frame(cframes, skip_resolution=True) if cframe.meta['FLAVOR'] in ['flat','arc']: raise ValueError("Bad flavor for exposure: {:s}".format(cframes)) # Sky sky_file = cframes.replace('cframe', 'sky') skymodel = read_sky(sky_file) if sub_sky: subtract_sky(cframe, skymodel) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] # Flux calibrated ivar = cframe.ivar[skyfibers] # Flux calibrated flux = skymodel.flux[skyfibers] # Residuals; not flux calibrated! wave = np.outer(np.ones(flux.shape[0]), cframe.wave) # Combine? ''' if combine: res = np.median(res, axis=0) ivar = np.median(ivar, axis=0) flux = np.median(flux, axis=0) wave = np.median(wave, axis=0) ''' # Return if flatten: return wave.flatten(), flux.flatten(), res.flatten(), ivar.flatten() else: return wave, flux, res, ivar
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None, output_dir=None, clobber=True): """ Generate a qaframe object from an input frame_file name (and night) Write QA to disk Will also make plots if directed Args: frame_file: str specprod_dir: str, optional qa_dir: str, optional -- Location of QA make_plots: bool, optional output_dir: str, optional Returns: """ import glob import os from desispec.io import read_frame from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.qa import qafile_from_framefile from desispec.io.frame import search_for_framefile from desispec.io.fiberflat import read_fiberflat from desispec.fiberflat import apply_fiberflat from desispec.qa import qa_plots from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots_ql if '/' in frame_file: # If present, assume full path is used here pass else: # Find the frame file in the desispec hierarchy? frame_file = search_for_framefile(frame_file) # Load frame frame = read_frame(frame_file) frame_meta = frame.meta night = frame_meta['NIGHT'].strip() camera = frame_meta['CAMERA'].strip() expid = frame_meta['EXPID'] spectro = int(frame_meta['CAMERA'][-1]) # Filename qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir) if os.path.isfile(qafile) and (not clobber): write = False else: write = True qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame_meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: # Backwards compatibility fiberflat = read_fiberflat(fiberflat_fil) except FileNotFoundError: fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d') path, basen = os.path.split(fiberflat_fil) path,_ = os.path.split(path) fiberflat_fil = os.path.join(path, basen) fiberflat = read_fiberflat(fiberflat_fil) if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber): write = True if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid, qaprod_dir=qaprod_dir, specprod_dir=specprod_dir, outdir=output_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera) if not os.path.exists(fiberflat_fil): # Backwards compatibility (for now) dummy_fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) # This is dummy path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil)) fiberflat_files = glob.glob(os.path.join(path,'*','fiberflat-'+camera+'*.fits')) if len(fiberflat_files) == 0: path = path.replace('exposures', 'calib2d') path,_ = os.path.split(path) # Remove night fiberflat_files = glob.glob(os.path.join(path,'fiberflat-'+camera+'*.fits')) # Sort and take the first (same as old pipeline) fiberflat_files.sort() fiberflat_fil = fiberflat_files[0] fiberflat = read_fiberflat(fiberflat_fil) apply_fiberflat(frame, fiberflat) # Load sky model and run try: skymodel = read_sky(sky_fil) except FileNotFoundError: warnings.warn("Sky file {:s} not found. Skipping..".format(sky_fil)) else: if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber): write=True if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) #qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe) # S/N QA on cframe if qatype == 'qa_data': # cframe cframe_file = frame_file.replace('frame-', 'cframe-') cframe = read_frame(cframe_file) if qaframe.run_qa('S2N', (cframe,), clobber=clobber): write=True # Figure? if make_plots: s2n_dict = copy.deepcopy(qaframe.qa_data['S2N']) qafig = meta.findfile('qa_s2n_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) #badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist() #sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE') coeff = s2n_dict['METRICS']['FITCOEFF_TGT']#[sci_idx] # Add an item or two for the QL method s2n_dict['CAMERA'] = camera s2n_dict['EXPID'] = expid s2n_dict['PANAME'] = 'SNRFit' s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA'] s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC'] objlist = s2n_dict['METRICS']['OBJLIST'] # Deal with YAML list instead of ndarray s2n_dict['METRICS']['MEDIAN_SNR'] = np.array(s2n_dict['METRICS']['MEDIAN_SNR']) # Generate if (not os.path.isfile(qafig)) or clobber: qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist, [[]]*len(objlist), coeff) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, spectrograph=spectro) # try: # model_tuple=read_stdstar_models(stdstar_fil) # except FileNotFoundError: # warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil)) # else: flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: fluxcalib = read_flux_calibration(flux_fil) except FileNotFoundError: warnings.warn("Flux file {:s} not found. Skipping..".format(flux_fil)) else: if qaframe.run_qa('FLUXCALIB', (frame, fluxcalib), clobber=clobber): # , model_tuple))#, indiv_stars)) write = True if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple) # Write if write: write_qa_frame(qafile, qaframe, verbose=True) return qaframe