def main(args) : log=get_logger() log.info("read frame") # read frame frame = read_frame(args.infile) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel=read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux,model_wave,model_fibers=read_stdstar_models(args.models) # check that the model_fibers are actually standard stars fibermap = frame.fibermap model_fibers = model_fibers%500 if np.any(fibermap['OBJTYPE'][model_fibers] != 'STD'): for i in model_fibers: log.error("inconsistency with spectrum %d, OBJTYPE='%s' in fibermap"%(i,fibermap["OBJTYPE"][i])) sys.exit(12) fluxcalib = compute_flux_calibration(frame, model_wave, model_flux) # QA if (args.qafile is not None): log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run #import pdb; pdb.set_trace() qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s"%args.outfile)
def test_qa_frame_plot(self): from desispec.qa import qa_plots from desispec.qa import qa_frame # Frame frame = get_frame_data(500) # Load calib fluxcalib = get_calib_from_frame(frame) # QA Frame tdict = {} tdict['20190829'] = {} dint = 20 tdict['20190829'][dint] = {} tdict['20190829'][dint]['flavor'] = 'science' tdict['20190829'][dint]['b'] = {} tdict['20190829'][dint]['b']['FLUXCALIB'] = {} tdict['20190829'][dint]['b']['FLUXCALIB']['METRICS'] = {} tdict['20190829'][dint]['b']['FLUXCALIB']['METRICS']['BLAH'] = 1 qaframe = qa_frame.QA_Frame(tdict) # Plot qa_plots.frame_fluxcalib(self.frame_pdf, qaframe, frame, fluxcalib)
def make_frameqa(self, make_plots=False, clobber=True): """ Work through the Production and make QA for all frames Parameters: make_plots: bool, optional Remake the plots too? clobber: bool, optional Returns: """ # imports from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.fiberflat import read_fiberflat from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots from desispec.io.fluxcalibration import read_stdstar_models # Loop on nights path_nights = glob.glob(self.specprod_dir + '/exposures/*') nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights] for night in nights: for exposure in get_exposures(night, specprod_dir=self.specprod_dir): # Object only?? frames_dict = get_files(filetype=str('frame'), night=night, expid=exposure, specprod_dir=self.specprod_dir) for camera, frame_fil in frames_dict.items(): # Load frame frame = read_frame(frame_fil) spectro = int(frame.meta['CAMERA'][-1]) if frame.meta['FLAVOR'] in ['flat', 'arc']: qatype = 'qa_calib' else: qatype = 'qa_data' qafile = meta.findfile(qatype, night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) if (not clobber) & os.path.isfile(qafile): log.info( "qafile={:s} exists. Not over-writing. Consider clobber=True" .format(qafile)) continue # Load qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame.meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile( 'fiberflat', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fiberflat = read_fiberflat(fiberflat_fil) qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber) if make_plots: # Do it qafig = meta.findfile( 'qa_flat_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) skymodel = read_sky(sky_fil) qaframe.run_qa('SKYSUB', (frame, skymodel)) if make_plots: qafig = meta.findfile( 'qa_sky_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile( 'stdstars', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir, spectrograph=spectro) model_tuple = read_stdstar_models(stdstar_fil) flux_fil = meta.findfile( 'calib', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fluxcalib = read_flux_calibration(flux_fil) qaframe.run_qa( 'FLUXCALIB', (frame, fluxcalib, model_tuple)) #, indiv_stars)) if make_plots: qafig = meta.findfile( 'qa_flux_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib, model_tuple) # Write write_qa_frame(qafile, qaframe)
def make_frameqa(self, make_plots=False, clobber=True): """ Work through the Production and make QA for all frames Parameters: make_plots: bool, optional Remake the plots too? clobber: bool, optional Returns: """ # imports from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.fiberflat import read_fiberflat from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots from desispec.io.fluxcalibration import read_stdstar_models # Loop on nights path_nights = glob.glob(self.specprod_dir+'/exposures/*') nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights] for night in nights: for exposure in get_exposures(night, specprod_dir = self.specprod_dir): # Object only?? frames_dict = get_files(filetype = str('frame'), night = night, expid = exposure, specprod_dir = self.specprod_dir) for camera,frame_fil in frames_dict.items(): # Load frame frame = read_frame(frame_fil) spectro = int(frame.meta['CAMERA'][-1]) if frame.meta['FLAVOR'] in ['flat','arc']: qatype = 'qa_calib' else: qatype = 'qa_data' qafile = meta.findfile(qatype, night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) if (not clobber) & os.path.isfile(qafile): log.info("qafile={:s} exists. Not over-writing. Consider clobber=True".format(qafile)) continue # Load qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame.meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fiberflat = read_fiberflat(fiberflat_fil) qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber) if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) skymodel = read_sky(sky_fil) qaframe.run_qa('SKYSUB', (frame, skymodel)) if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir, spectrograph=spectro) model_tuple=read_stdstar_models(stdstar_fil) flux_fil = meta.findfile('calib', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fluxcalib = read_flux_calibration(flux_fil) qaframe.run_qa('FLUXCALIB', (frame, fluxcalib, model_tuple))#, indiv_stars)) if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib, model_tuple) # Write write_qa_frame(qafile, qaframe)
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None, output_dir=None, clobber=True): """ Generate a qaframe object from an input frame_file name (and night) Write QA to disk Will also make plots if directed Args: frame_file: str specprod_dir: str, optional qa_dir: str, optional -- Location of QA make_plots: bool, optional output_dir: str, optional Returns: """ import glob import os from desispec.io import read_frame from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.qa import qafile_from_framefile from desispec.io.frame import search_for_framefile from desispec.io.fiberflat import read_fiberflat from desispec.fiberflat import apply_fiberflat from desispec.qa import qa_plots from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots_ql if '/' in frame_file: # If present, assume full path is used here pass else: # Find the frame file in the desispec hierarchy? frame_file = search_for_framefile(frame_file) # Load frame frame = read_frame(frame_file) frame_meta = frame.meta night = frame_meta['NIGHT'].strip() camera = frame_meta['CAMERA'].strip() expid = frame_meta['EXPID'] spectro = int(frame_meta['CAMERA'][-1]) # Filename qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir) if os.path.isfile(qafile) and (not clobber): write = False else: write = True qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame_meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: # Backwards compatibility fiberflat = read_fiberflat(fiberflat_fil) except FileNotFoundError: fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d') path, basen = os.path.split(fiberflat_fil) path, _ = os.path.split(path) fiberflat_fil = os.path.join(path, basen) fiberflat = read_fiberflat(fiberflat_fil) if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber): write = True if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid, qaprod_dir=qaprod_dir, specprod_dir=specprod_dir, outdir=output_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera) if not os.path.exists(fiberflat_fil): # Backwards compatibility (for now) dummy_fiberflat_fil = meta.findfile( 'fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) # This is dummy path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil)) fiberflat_files = glob.glob( os.path.join(path, '*', 'fiberflat-' + camera + '*.fits')) if len(fiberflat_files) == 0: path = path.replace('exposures', 'calib2d') path, _ = os.path.split(path) # Remove night fiberflat_files = glob.glob( os.path.join(path, 'fiberflat-' + camera + '*.fits')) # Sort and take the first (same as old pipeline) fiberflat_files.sort() fiberflat_fil = fiberflat_files[0] fiberflat = read_fiberflat(fiberflat_fil) apply_fiberflat(frame, fiberflat) # Load sky model and run try: skymodel = read_sky(sky_fil) except FileNotFoundError: warnings.warn( "Sky file {:s} not found. Skipping..".format(sky_fil)) else: if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber): write = True if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) #qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe) # S/N QA on cframe if qatype == 'qa_data': # cframe cframe_file = frame_file.replace('frame-', 'cframe-') cframe = read_frame(cframe_file) if qaframe.run_qa('S2N', (cframe, ), clobber=clobber): write = True # Figure? if make_plots: s2n_dict = copy.deepcopy(qaframe.qa_data['S2N']) qafig = meta.findfile('qa_s2n_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) #badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist() #sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE') coeff = s2n_dict['METRICS']['FITCOEFF_TGT'] #[sci_idx] # Add an item or two for the QL method s2n_dict['CAMERA'] = camera s2n_dict['EXPID'] = expid s2n_dict['PANAME'] = 'SNRFit' s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA'] s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC'] objlist = s2n_dict['METRICS']['OBJLIST'] # Deal with YAML list instead of ndarray s2n_dict['METRICS']['MEDIAN_SNR'] = np.array( s2n_dict['METRICS']['MEDIAN_SNR']) # Generate if (not os.path.isfile(qafig)) or clobber: qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist, [[]] * len(objlist), coeff) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, spectrograph=spectro) # try: # model_tuple=read_stdstar_models(stdstar_fil) # except FileNotFoundError: # warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil)) # else: flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: fluxcalib = read_flux_calibration(flux_fil) except FileNotFoundError: warnings.warn( "Flux file {:s} not found. Skipping..".format(flux_fil)) else: if qaframe.run_qa( 'FLUXCALIB', (frame, fluxcalib), clobber=clobber): # , model_tuple))#, indiv_stars)) write = True if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple) # Write if write: write_qa_frame(qafile, qaframe, verbose=True) return qaframe
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None, output_dir=None, clobber=True): """ Generate a qaframe object from an input frame_file name (and night) Write QA to disk Will also make plots if directed Args: frame_file: str specprod_dir: str, optional qa_dir: str, optional -- Location of QA make_plots: bool, optional output_dir: str, optional Returns: """ import glob import os from desispec.io import read_frame from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.qa import qafile_from_framefile from desispec.io.frame import search_for_framefile from desispec.io.fiberflat import read_fiberflat from desispec.fiberflat import apply_fiberflat from desispec.qa import qa_plots from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration if '/' in frame_file: # If present, assume full path is used here pass else: # Find the frame file in the desispec hierarchy? frame_file = search_for_framefile(frame_file) # Load frame frame = read_frame(frame_file) frame_meta = frame.meta night = frame_meta['NIGHT'].strip() camera = frame_meta['CAMERA'].strip() expid = frame_meta['EXPID'] spectro = int(frame_meta['CAMERA'][-1]) # Filename qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir) qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame_meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: # Backwards compatibility fiberflat = read_fiberflat(fiberflat_fil) except FileNotFoundError: fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d') path, basen = os.path.split(fiberflat_fil) path, _ = os.path.split(path) fiberflat_fil = os.path.join(path, basen) fiberflat = read_fiberflat(fiberflat_fil) qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber) if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera) if not os.path.exists(fiberflat_fil): # Backwards compatibility (for now) dummy_fiberflat_fil = meta.findfile( 'fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) # This is dummy path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil)) fiberflat_files = glob.glob( os.path.join(path, '*', 'fiberflat-' + camera + '*.fits')) if len(fiberflat_files) == 0: path = path.replace('exposures', 'calib2d') path, _ = os.path.split(path) # Remove night fiberflat_files = glob.glob( os.path.join(path, 'fiberflat-' + camera + '*.fits')) # Sort and take the first (same as old pipeline) fiberflat_files.sort() fiberflat_fil = fiberflat_files[0] fiberflat = read_fiberflat(fiberflat_fil) apply_fiberflat(frame, fiberflat) # Load sky model and run try: skymodel = read_sky(sky_fil) except FileNotFoundError: warnings.warn( "Sky file {:s} not found. Skipping..".format(sky_fil)) else: qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber) if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) #qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, spectrograph=spectro) # try: # model_tuple=read_stdstar_models(stdstar_fil) # except FileNotFoundError: # warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil)) # else: flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: fluxcalib = read_flux_calibration(flux_fil) except FileNotFoundError: warnings.warn( "Flux file {:s} not found. Skipping..".format(flux_fil)) else: qaframe.run_qa( 'FLUXCALIB', (frame, fluxcalib)) # , model_tuple))#, indiv_stars)) if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple) # Write write_qa_frame(qafile, qaframe, verbose=True) return qaframe
def main(args) : log=get_logger() cmd = ['desi_compute_fluxcalibration',] for key, value in args.__dict__.items(): if value is not None: cmd += ['--'+key, str(value)] cmd = ' '.join(cmd) log.info(cmd) log.info("read frame") # read frame frame = read_frame(args.infile) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel=read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux,model_wave,model_fibers,model_metadata=read_stdstar_models(args.models) if args.chi2cut > 0 : ok = np.where(model_metadata["CHI2DOF"]<args.chi2cut)[0] if ok.size == 0 : log.error("chi2cut has discarded all stars") sys.exit(12) nstars=model_flux.shape[0] nbad=nstars-ok.size if nbad>0 : log.warning("discarding %d star(s) out of %d because of chi2cut"%(nbad,nstars)) model_flux=model_flux[ok] model_fibers=model_fibers[ok] model_metadata=model_metadata[:][ok] if args.delta_color_cut > 0 : ok = np.where(np.abs(model_metadata["MODEL_G-R"]-model_metadata["DATA_G-R"])<args.delta_color_cut)[0] nstars=model_flux.shape[0] nbad=nstars-ok.size if nbad>0 : log.warning("discarding %d star(s) out of %d because |delta_color|>%f"%(nbad,nstars,args.delta_color_cut)) model_flux=model_flux[ok] model_fibers=model_fibers[ok] model_metadata=model_metadata[:][ok] # automatically reject stars that ar chi2 outliers if args.chi2cut_nsig > 0 : mchi2=np.median(model_metadata["CHI2DOF"]) rmschi2=np.std(model_metadata["CHI2DOF"]) maxchi2=mchi2+args.chi2cut_nsig*rmschi2 ok=np.where(model_metadata["CHI2DOF"]<=maxchi2)[0] nstars=model_flux.shape[0] nbad=nstars-ok.size if nbad>0 : log.warning("discarding %d star(s) out of %d because reduced chi2 outliers (at %d sigma, giving rchi2<%f )"%(nbad,nstars,args.chi2cut_nsig,maxchi2)) model_flux=model_flux[ok] model_fibers=model_fibers[ok] model_metadata=model_metadata[:][ok] # check that the model_fibers are actually standard stars fibermap = frame.fibermap ## check whether star fibers from args.models are consistent with fibers from fibermap ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit fibermap_std_indices = np.where(isStdStar(fibermap['DESI_TARGET']))[0] if np.any(~np.in1d(model_fibers%500, fibermap_std_indices)): for i in model_fibers%500: log.error("inconsistency with spectrum {}, OBJTYPE='{}', DESI_TARGET={} in fibermap".format( (i, fibermap["OBJTYPE"][i], fibermap["DESI_TARGET"][i]))) sys.exit(12) fluxcalib = compute_flux_calibration(frame, model_wave, model_flux, model_fibers%500) # QA if (args.qafile is not None): log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run #import pdb; pdb.set_trace() qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s"%args.outfile)
def main(args): log = get_logger() log.info("read frame") # read frame frame = read_frame(args.infile) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel = read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux, model_wave, model_fibers = read_stdstar_models(args.models) model_tuple = model_flux, model_wave, model_fibers # check that the model_fibers are actually standard stars fibermap = frame.fibermap model_fibers = model_fibers % 500 if np.any(fibermap['OBJTYPE'][model_fibers] != 'STD'): for i in model_fibers: log.error( "inconsistency with spectrum %d, OBJTYPE='%s' in fibermap" % (i, fibermap["OBJTYPE"][i])) sys.exit(12) #fluxcalib, indiv_stars = compute_flux_calibration(frame, model_wave, model_flux) fluxcalib = compute_flux_calibration(frame, model_wave, model_flux) # QA if (args.qafile is not None): log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run qaframe.run_qa('FLUXCALIB', (frame, fluxcalib, model_tuple)) #, indiv_stars)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib, model_tuple) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s" % args.outfile)
def main(args): log = get_logger() cmd = [ 'desi_compute_fluxcalibration', ] for key, value in args.__dict__.items(): if value is not None: cmd += ['--' + key, str(value)] cmd = ' '.join(cmd) log.info(cmd) log.info("read frame") # read frame frame = read_frame(args.infile) # Set fibermask flagged spectra to have 0 flux and variance frame = get_fiberbitmasked_frame(frame, bitmask='flux', ivar_framemask=True) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel = read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models( args.models) ok = np.ones(len(model_metadata), dtype=bool) if args.chi2cut > 0: log.info("Apply cut CHI2DOF<{}".format(args.chi2cut)) ok &= (model_metadata["CHI2DOF"] < args.chi2cut) if args.delta_color_cut > 0: log.info("Apply cut |delta color|<{}".format(args.delta_color_cut)) ok &= (np.abs(model_metadata["MODEL_G-R"] - model_metadata["DATA_G-R"]) < args.delta_color_cut) if args.min_color is not None: log.info("Apply cut DATA_G-R>{}".format(args.min_color)) ok &= (model_metadata["DATA_G-R"] > args.min_color) if args.chi2cut_nsig > 0: # automatically reject stars that ar chi2 outliers mchi2 = np.median(model_metadata["CHI2DOF"]) rmschi2 = np.std(model_metadata["CHI2DOF"]) maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2 log.info("Apply cut CHI2DOF<{} based on chi2cut_nsig={}".format( maxchi2, args.chi2cut_nsig)) ok &= (model_metadata["CHI2DOF"] <= maxchi2) ok = np.where(ok)[0] if ok.size == 0: log.error("cuts discarded all stars") sys.exit(12) nstars = model_flux.shape[0] nbad = nstars - ok.size if nbad > 0: log.warning("discarding %d star(s) out of %d because of cuts" % (nbad, nstars)) model_flux = model_flux[ok] model_fibers = model_fibers[ok] model_metadata = model_metadata[:][ok] # check that the model_fibers are actually standard stars fibermap = frame.fibermap ## check whether star fibers from args.models are consistent with fibers from fibermap ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit fibermap_std_indices = np.where(isStdStar(fibermap))[0] if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)): target_colnames, target_masks, survey = main_cmx_or_sv(fibermap) colname = target_colnames[0] for i in model_fibers % 500: log.error( "inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap" .format(i, fibermap["OBJTYPE"][i], colname, fibermap[colname][i])) sys.exit(12) # Make sure the fibers of interest aren't entirely masked. if np.sum( np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) == frame.nwave) == len(model_fibers): log.warning('All standard-star spectra are masked!') return fluxcalib = compute_flux_calibration( frame, model_wave, model_flux, model_fibers % 500, highest_throughput_nstars=args.highest_throughput) # QA if (args.qafile is not None): log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame_meta=frame.meta, flavor=frame.meta['FLAVOR']) # Run #import pdb; pdb.set_trace() qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s" % args.outfile)
def main(args): log = get_logger() log.info("read frame") # read frame frame = read_frame(args.infile) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel = read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models( args.models) if args.chi2cut > 0: ok = np.where(model_metadata["CHI2DOF"] < args.chi2cut)[0] if ok.size == 0: log.error("chi2cut has discarded all stars") sys.exit(12) nstars = model_flux.shape[0] nbad = nstars - ok.size if nbad > 0: log.warning("discarding %d star(s) out of %d because of chi2cut" % (nbad, nstars)) model_flux = model_flux[ok] model_fibers = model_fibers[ok] model_metadata = model_metadata[:][ok] if args.delta_color_cut > 0: ok = np.where( np.abs(model_metadata["MODEL_G-R"] - model_metadata["DATA_G-R"]) < args.delta_color_cut)[0] nstars = model_flux.shape[0] nbad = nstars - ok.size if nbad > 0: log.warning( "discarding %d star(s) out of %d because |delta_color|>%f" % (nbad, nstars, args.delta_color_cut)) model_flux = model_flux[ok] model_fibers = model_fibers[ok] model_metadata = model_metadata[:][ok] # automatically reject stars that ar chi2 outliers if args.chi2cut_nsig > 0: mchi2 = np.median(model_metadata["CHI2DOF"]) rmschi2 = np.std(model_metadata["CHI2DOF"]) maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2 ok = np.where(model_metadata["CHI2DOF"] <= maxchi2)[0] nstars = model_flux.shape[0] nbad = nstars - ok.size if nbad > 0: log.warning( "discarding %d star(s) out of %d because reduced chi2 outliers (at %d sigma, giving rchi2<%f )" % (nbad, nstars, args.chi2cut_nsig, maxchi2)) model_flux = model_flux[ok] model_fibers = model_fibers[ok] model_metadata = model_metadata[:][ok] # check that the model_fibers are actually standard stars fibermap = frame.fibermap ## check whether star fibers from args.models are consistent with fibers from fibermap ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit w = np.where(fibermap["OBJTYPE"][model_fibers % 500] != 'STD')[0] if len(w) > 0: for i in model_fibers % 500: log.error( "inconsistency with spectrum %d, OBJTYPE='%s' in fibermap" % (i, fibermap["OBJTYPE"][i])) sys.exit(12) fluxcalib = compute_flux_calibration(frame, model_wave, model_flux, model_fibers % 500) # QA if (args.qafile is not None): log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run #import pdb; pdb.set_trace() qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s" % args.outfile)
def main(args): log = get_logger() cmd = [ 'desi_compute_fluxcalibration', ] for key, value in args.__dict__.items(): if value is not None: cmd += ['--' + key, str(value)] cmd = ' '.join(cmd) log.info(cmd) log.info("read frame") # read frame frame = read_frame(args.infile) # Set fibermask flagged spectra to have 0 flux and variance frame = get_fiberbitmasked_frame(frame, bitmask='flux', ivar_framemask=True) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel = read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models( args.models) ok = np.ones(len(model_metadata), dtype=bool) if args.chi2cut > 0: log.info("apply cut CHI2DOF<{}".format(args.chi2cut)) good = (model_metadata["CHI2DOF"] < args.chi2cut) bad = ~good ok &= good if np.any(bad): log.info(" discard {} stars with CHI2DOF= {}".format( np.sum(bad), list(model_metadata["CHI2DOF"][bad]))) legacy_filters = ('G-R', 'R-Z') gaia_filters = ('GAIA-BP-RP', 'GAIA-G-RP') model_column_list = model_metadata.columns.names if args.color is None: if 'MODEL_G-R' in model_column_list: color = 'G-R' elif 'MODEL_GAIA-BP-RP' in model_column_list: log.info('Using Gaia filters') color = 'GAIA-BP-RP' else: log.error( "Can't find either G-R or BP-RP color in the model file.") sys.exit(15) else: if args.color not in legacy_filters and args.color not in gaia_filters: log.error( 'Color name {} is not allowed, must be one of {} {}'.format( args.color, legacy_filters, gaia_filters)) sys.exit(14) color = args.color if color not in model_column_list: # This should't happen log.error( 'The color {} was not computed in the models'.format(color)) sys.exit(16) if args.delta_color_cut > 0: log.info("apply cut |delta color|<{}".format(args.delta_color_cut)) good = (np.abs(model_metadata["MODEL_" + color] - model_metadata["DATA_" + color]) < args.delta_color_cut) bad = ok & (~good) ok &= good if np.any(bad): vals = model_metadata["MODEL_" + color][bad] - model_metadata["DATA_" + color][bad] log.info(" discard {} stars with dcolor= {}".format( np.sum(bad), list(vals))) if args.min_color is not None: log.info("apply cut DATA_{}>{}".format(color, args.min_color)) good = (model_metadata["DATA_{}".format(color)] > args.min_color) bad = ok & (~good) ok &= good if np.any(bad): vals = model_metadata["DATA_{}".format(color)][bad] log.info(" discard {} stars with {}= {}".format( np.sum(bad), color, list(vals))) if args.chi2cut_nsig > 0: # automatically reject stars that ar chi2 outliers mchi2 = np.median(model_metadata["CHI2DOF"]) rmschi2 = np.std(model_metadata["CHI2DOF"]) maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2 log.info("apply cut CHI2DOF<{} based on chi2cut_nsig={}".format( maxchi2, args.chi2cut_nsig)) good = (model_metadata["CHI2DOF"] <= maxchi2) bad = ok & (~good) ok &= good if np.any(bad): log.info(" discard {} stars with CHI2DOF={}".format( np.sum(bad), list(model_metadata["CHI2DOF"][bad]))) ok = np.where(ok)[0] if ok.size == 0: log.error("selection cuts discarded all stars") sys.exit(12) nstars = model_flux.shape[0] nbad = nstars - ok.size if nbad > 0: log.warning("discarding %d star(s) out of %d because of cuts" % (nbad, nstars)) model_flux = model_flux[ok] model_fibers = model_fibers[ok] model_metadata = model_metadata[:][ok] # check that the model_fibers are actually standard stars fibermap = frame.fibermap ## check whether star fibers from args.models are consistent with fibers from fibermap ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit fibermap_std_indices = np.where(isStdStar(fibermap))[0] if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)): target_colnames, target_masks, survey = main_cmx_or_sv(fibermap) colname = target_colnames[0] for i in model_fibers % 500: log.error( "inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap" .format(i, fibermap["OBJTYPE"][i], colname, fibermap[colname][i])) sys.exit(12) # Make sure the fibers of interest aren't entirely masked. if np.sum( np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) == frame.nwave) == len(model_fibers): log.warning('All standard-star spectra are masked!') return fluxcalib = compute_flux_calibration( frame, model_wave, model_flux, model_fibers % 500, highest_throughput_nstars=args.highest_throughput, exposure_seeing_fwhm=args.seeing_fwhm) # QA if (args.qafile is not None): from desispec.io import write_qa_frame from desispec.io.qa import load_qa_frame from desispec.qa import qa_plots log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame_meta=frame.meta, flavor=frame.meta['FLAVOR']) # Run #import pdb; pdb.set_trace() qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib) # record inputs frame.meta['IN_FRAME'] = shorten_filename(args.infile) frame.meta['IN_SKY'] = shorten_filename(args.sky) frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat) frame.meta['STDMODEL'] = shorten_filename(args.models) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s" % args.outfile)
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None, output_dir=None, clobber=True): """ Generate a qaframe object from an input frame_file name (and night) Write QA to disk Will also make plots if directed Args: frame_file: str specprod_dir: str, optional qa_dir: str, optional -- Location of QA make_plots: bool, optional output_dir: str, optional Returns: """ import glob import os from desispec.io import read_frame from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.qa import qafile_from_framefile from desispec.io.frame import search_for_framefile from desispec.io.fiberflat import read_fiberflat from desispec.fiberflat import apply_fiberflat from desispec.qa import qa_plots from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots_ql if '/' in frame_file: # If present, assume full path is used here pass else: # Find the frame file in the desispec hierarchy? frame_file = search_for_framefile(frame_file) # Load frame frame = read_frame(frame_file) frame_meta = frame.meta night = frame_meta['NIGHT'].strip() camera = frame_meta['CAMERA'].strip() expid = frame_meta['EXPID'] spectro = int(frame_meta['CAMERA'][-1]) # Filename qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir) if os.path.isfile(qafile) and (not clobber): write = False else: write = True qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame_meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: # Backwards compatibility fiberflat = read_fiberflat(fiberflat_fil) except FileNotFoundError: fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d') path, basen = os.path.split(fiberflat_fil) path,_ = os.path.split(path) fiberflat_fil = os.path.join(path, basen) fiberflat = read_fiberflat(fiberflat_fil) if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber): write = True if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid, qaprod_dir=qaprod_dir, specprod_dir=specprod_dir, outdir=output_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera) if not os.path.exists(fiberflat_fil): # Backwards compatibility (for now) dummy_fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) # This is dummy path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil)) fiberflat_files = glob.glob(os.path.join(path,'*','fiberflat-'+camera+'*.fits')) if len(fiberflat_files) == 0: path = path.replace('exposures', 'calib2d') path,_ = os.path.split(path) # Remove night fiberflat_files = glob.glob(os.path.join(path,'fiberflat-'+camera+'*.fits')) # Sort and take the first (same as old pipeline) fiberflat_files.sort() fiberflat_fil = fiberflat_files[0] fiberflat = read_fiberflat(fiberflat_fil) apply_fiberflat(frame, fiberflat) # Load sky model and run try: skymodel = read_sky(sky_fil) except FileNotFoundError: warnings.warn("Sky file {:s} not found. Skipping..".format(sky_fil)) else: if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber): write=True if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) #qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe) # S/N QA on cframe if qatype == 'qa_data': # cframe cframe_file = frame_file.replace('frame-', 'cframe-') cframe = read_frame(cframe_file) if qaframe.run_qa('S2N', (cframe,), clobber=clobber): write=True # Figure? if make_plots: s2n_dict = copy.deepcopy(qaframe.qa_data['S2N']) qafig = meta.findfile('qa_s2n_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) #badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist() #sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE') coeff = s2n_dict['METRICS']['FITCOEFF_TGT']#[sci_idx] # Add an item or two for the QL method s2n_dict['CAMERA'] = camera s2n_dict['EXPID'] = expid s2n_dict['PANAME'] = 'SNRFit' s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA'] s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC'] objlist = s2n_dict['METRICS']['OBJLIST'] # Deal with YAML list instead of ndarray s2n_dict['METRICS']['MEDIAN_SNR'] = np.array(s2n_dict['METRICS']['MEDIAN_SNR']) # Generate if (not os.path.isfile(qafig)) or clobber: qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist, [[]]*len(objlist), coeff) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, spectrograph=spectro) # try: # model_tuple=read_stdstar_models(stdstar_fil) # except FileNotFoundError: # warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil)) # else: flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: fluxcalib = read_flux_calibration(flux_fil) except FileNotFoundError: warnings.warn("Flux file {:s} not found. Skipping..".format(flux_fil)) else: if qaframe.run_qa('FLUXCALIB', (frame, fluxcalib), clobber=clobber): # , model_tuple))#, indiv_stars)) write = True if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple) # Write if write: write_qa_frame(qafile, qaframe, verbose=True) return qaframe