def test_subtract_sky_with_gradient_using_compute_non_uniform_sky(self): spectra = self._get_spectra(with_gradient=True) sky = compute_sky(spectra,angular_variation_deg=1,chromatic_variation_deg=-1,add_variance=self.add_variance) subtract_sky(spectra, sky) #- allow some slop in the sky subtraction self.assertTrue(np.allclose(spectra.flux, 0, rtol=1e-3, atol=1e-3))
def test_subtract_sky_with_gradient_using_compute_polynomial_times_sky(self): spectra = self._get_spectra(with_gradient=True) sky = compute_sky(spectra,angular_variation_deg=1,chromatic_variation_deg=1,add_variance=self.add_variance) subtract_sky(spectra, sky) #- allow some slop in the sky subtraction self.assertTrue(np.allclose(spectra.flux, 0, rtol=1e-3, atol=1e-3)) # this is not exact, it's an iterative fit
def run_sky_subtraction(night, expid, cameras, basedir, nsky_list, reps=None): '''Runs sky subtraction with new sky models and frame files. Args: night: YYYYMMDD (float) expid: exposure id without padding zeros (float) cameras: list of cameras corresponding to frame and sky files in given directory, example ['r3', 'z3', 'b3'] (list or array) basedir: where to look for frame files nsky_list: list with different numbers of fibers frame files and sky files were generated with. Options: rep: number of different frame/sky files for each camera and nsky combination. Default is 5 ''' if reps == None: reps = 5 for cam in cameras: framefile = desispec.io.findfile('frame', night, expid, camera=cam) header = fitsio.read_header(framefile) fiberflatfile = findcalibfile([ header, ], 'FIBERFLAT') fiberflat = desispec.io.read_fiberflat(fiberflatfile) for N in range(reps): for n in nsky_list: newframefile = basedir + '/frame-{}-{:08d}-{}-{}.fits'.format( cam, expid, n, N) skyfile = basedir + '/sky-{}-{:08d}-{}-{}.fits'.format( cam, expid, n, N) sframe = desispec.io.read_frame(newframefile) sky = desispec.io.read_sky(skyfile) apply_fiberflat(sframe, fiberflat) subtract_sky(sframe, sky) sframefile = basedir + '/sframe-{}-{:08d}-{}-{}.fits'.format( cam, expid, n, N) desispec.io.write_frame(sframefile, sframe)
def test_subtract_sky_with_gradient_using_compute_polynomial_times_sky(self): spectra = self._get_spectra(with_gradient=True) sky = compute_sky(spectra,angular_variation_deg=1,chromatic_variation_deg=1,add_variance=self.add_variance) subtract_sky(spectra, sky) #- allow some slop in the sky subtraction self.assertTrue(np.allclose(spectra.flux, 0, rtol=1e-3, atol=1.)) # this is not exact, it's an iterative fit
def get_skyres(cframes, sub_sky=False): from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.sky import subtract_sky if isinstance(cframes, list): all_wave, all_flux, all_res, all_ivar = [], [], [], [] for cframe_file in cframes: wave, flux, res, ivar = get_skyres(cframe_file) # Concatenate and return all_wave.append(wave) all_flux.append(flux) all_res.append(res) all_ivar.append(ivar) return np.concatenate(all_wave), np.concatenate(all_flux), \ np.concatenate(all_res), np.concatenate(all_ivar) cframe = read_frame(cframes) if cframe.meta['FLAVOR'] in ['flat', 'arc']: raise ValueError("Bad flavor for exposure: {:s}".format(cframes)) # Sky sky_file = cframes.replace('cframe', 'sky') skymodel = read_sky(sky_file) if sub_sky: subtract_sky(cframe, skymodel) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] ivar = cframe.ivar[skyfibers] flux = skymodel.flux[skyfibers] # Residuals wave = np.outer(np.ones(flux.shape[0]), cframe.wave) # Return return wave.flatten(), flux.flatten(), res.flatten(), ivar.flatten()
def main(args) : log=get_logger() log.info("read frame") # read frame frame = read_frame(args.infile) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel=read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux,model_wave,model_fibers=read_stdstar_models(args.models) # check that the model_fibers are actually standard stars fibermap = frame.fibermap model_fibers = model_fibers%500 if np.any(fibermap['OBJTYPE'][model_fibers] != 'STD'): for i in model_fibers: log.error("inconsistency with spectrum %d, OBJTYPE='%s' in fibermap"%(i,fibermap["OBJTYPE"][i])) sys.exit(12) fluxcalib = compute_flux_calibration(frame, model_wave, model_flux) # QA if (args.qafile is not None): log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run #import pdb; pdb.set_trace() qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s"%args.outfile)
def main() : parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--infile', type = str, default = None, required=True, help = 'path of DESI exposure frame fits file') parser.add_argument('--fiberflat', type = str, default = None, help = 'path of DESI fiberflat fits file') parser.add_argument('--sky', type = str, default = None, help = 'path of DESI sky fits file') parser.add_argument('--calib', type = str, default = None, help = 'path of DESI calibration fits file') parser.add_argument('--outfile', type = str, default = None, required=True, help = 'path of DESI sky fits file') # add calibration here when exists args = parser.parse_args() log = get_logger() if (args.fiberflat is None) and (args.sky is None) and (args.calib is None): log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?') sys.exit(12) frame = read_frame(args.infile) if args.fiberflat!=None : log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat to sky fibers apply_fiberflat(frame, fiberflat) if args.sky!=None : log.info("subtract sky") # read sky skymodel=read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) if args.calib!=None : log.info("calibrate") # read calibration fluxcalib=read_flux_calibration(args.calib) # apply calibration apply_flux_calibration(frame, fluxcalib) # save output write_frame(args.outfile, frame) log.info("successfully wrote %s"%args.outfile)
def run_pa(self,input_frame,skymodel,dump=False,dumpfile=None): from desispec.quicklook.quicksky import subtract_sky sframe=subtract_sky(input_frame,skymodel) if dump and dumpfile is not None: from desispec import io night = sframe.meta['NIGHT'] expid = sframe.meta['EXPID'] io.write_frame(dumpfile, sframe) log.info("Wrote intermediate file %s after %s"%(dumpfile,self.name)) return sframe
def run_pa(self,input_frame,skymodel,dump=False,dumpfile=None): from desispec.quicklook.quicksky import subtract_sky sframe=subtract_sky(input_frame,skymodel) if dump and dumpfile is not None: from desispec import io night = sframe.meta['NIGHT'] expid = sframe.meta['EXPID'] io.write_frame(dumpfile, sframe) log.info("Wrote intermediate file %s after %s"%(dumpfile,self.name)) return (sframe,skymodel)
def main(args): log = get_logger() if (args.fiberflat is None) and (args.sky is None) and (args.calib is None): log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?') sys.exit(12) frame = read_frame(args.infile) if args.fiberflat!=None : log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat to sky fibers apply_fiberflat(frame, fiberflat) if args.sky!=None : log.info("subtract sky") # read sky skymodel=read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) if args.calib!=None : log.info("calibrate") # read calibration fluxcalib=read_flux_calibration(args.calib) # apply calibration apply_flux_calibration(frame, fluxcalib) # save output write_frame(args.outfile, frame, units='1e-17 erg/(s cm2 A)') log.info("successfully wrote %s"%args.outfile)
def main() : parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--infile', type = str, default = None, required=True, help = 'path of DESI exposure frame fits file') parser.add_argument('--fibermap', type = str, default = None, required=True, help = 'path of DESI exposure frame fits file') parser.add_argument('--fiberflat', type = str, default = None, required=True, help = 'path of DESI fiberflat fits file') parser.add_argument('--sky', type = str, default = None, required=True, help = 'path of DESI sky fits file') parser.add_argument('--models', type = str, default = None, required=True, help = 'path of spetro-photometric stellar spectra fits file') parser.add_argument('--outfile', type = str, default = None, required=True, help = 'path of DESI flux calbration fits file') args = parser.parse_args() log=get_logger() log.info("read frame") # read frame frame = read_frame(args.infile) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel=read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux,model_wave,model_fibers=read_stdstar_models(args.models) # select fibers SPECMIN=frame.header["SPECMIN"] SPECMAX=frame.header["SPECMAX"] selec=np.where((model_fibers>=SPECMIN)&(model_fibers<=SPECMAX))[0] if selec.size == 0 : log.error("no stellar models for this spectro") sys.exit(12) fibers=model_fibers[selec]-frame.header["SPECMIN"] log.info("star fibers= %s"%str(fibers)) table = read_fibermap(args.fibermap) bad=np.where(table["OBJTYPE"][fibers]!="STD")[0] if bad.size > 0 : for fiber in fibers[bad] : log.error("inconsistency with fiber %d, OBJTYPE='%s' in fibermap"%(fiber,table["OBJTYPE"][fiber])) sys.exit(12) fluxcalib = compute_flux_calibration(frame, fibers, model_wave, model_flux) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.header) log.info("successfully wrote %s"%args.outfile)
def get_skyres(cframes, sub_sky=False, flatten=True): """ Args: cframes: str or list Single cframe or a list of them sub_sky: bool, optional Subtract the sky? This should probably not be done flatten: bool, optional Return a flat, 1D array for each variable combine: bool, optional combine the individual sky fibers? Median 'smash' Returns: wave : ndarray flux : ndarray res : ndarray ivar : ndarray """ from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.sky import subtract_sky if isinstance(cframes,list): all_wave, all_flux, all_res, all_ivar = [], [], [], [] for cframe_file in cframes: wave, flux, res, ivar = get_skyres(cframe_file, flatten=flatten) # Save all_wave.append(wave) all_flux.append(flux) all_res.append(res) all_ivar.append(ivar) # Concatenate -- Shape is preserved (nfibers, npix) twave = np.concatenate(all_wave) tflux = np.concatenate(all_flux) tres = np.concatenate(all_res) tivar = np.concatenate(all_ivar) # Return return twave, tflux, tres, tivar cframe = read_frame(cframes, skip_resolution=True) if cframe.meta['FLAVOR'] in ['flat','arc']: raise ValueError("Bad flavor for exposure: {:s}".format(cframes)) # Sky sky_file = cframes.replace('cframe', 'sky') skymodel = read_sky(sky_file) if sub_sky: subtract_sky(cframe, skymodel) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] # Flux calibrated ivar = cframe.ivar[skyfibers] # Flux calibrated flux = skymodel.flux[skyfibers] # Residuals; not flux calibrated! wave = np.outer(np.ones(flux.shape[0]), cframe.wave) # Combine? ''' if combine: res = np.median(res, axis=0) ivar = np.median(ivar, axis=0) flux = np.median(flux, axis=0) wave = np.median(wave, axis=0) ''' # Return if flatten: return wave.flatten(), flux.flatten(), res.flatten(), ivar.flatten() else: return wave, flux, res, ivar
def test_subtract_sky(self): spectra = self._get_spectra() sky = compute_sky(spectra,add_variance=self.add_variance) subtract_sky(spectra, sky) #- allow some slop in the sky subtraction self.assertTrue(np.allclose(spectra.flux, 0, rtol=1e-5, atol=1e-6))
def main(args): log = get_logger() cmd = [ 'desi_compute_fluxcalibration', ] for key, value in args.__dict__.items(): if value is not None: cmd += ['--' + key, str(value)] cmd = ' '.join(cmd) log.info(cmd) log.info("read frame") # read frame frame = read_frame(args.infile) # Set fibermask flagged spectra to have 0 flux and variance frame = get_fiberbitmasked_frame(frame, bitmask='flux', ivar_framemask=True) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel = read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models( args.models) ok = np.ones(len(model_metadata), dtype=bool) if args.chi2cut > 0: log.info("apply cut CHI2DOF<{}".format(args.chi2cut)) good = (model_metadata["CHI2DOF"] < args.chi2cut) bad = ~good ok &= good if np.any(bad): log.info(" discard {} stars with CHI2DOF= {}".format( np.sum(bad), list(model_metadata["CHI2DOF"][bad]))) legacy_filters = ('G-R', 'R-Z') gaia_filters = ('GAIA-BP-RP', 'GAIA-G-RP') model_column_list = model_metadata.columns.names if args.color is None: if 'MODEL_G-R' in model_column_list: color = 'G-R' elif 'MODEL_GAIA-BP-RP' in model_column_list: log.info('Using Gaia filters') color = 'GAIA-BP-RP' else: log.error( "Can't find either G-R or BP-RP color in the model file.") sys.exit(15) else: if args.color not in legacy_filters and args.color not in gaia_filters: log.error( 'Color name {} is not allowed, must be one of {} {}'.format( args.color, legacy_filters, gaia_filters)) sys.exit(14) color = args.color if color not in model_column_list: # This should't happen log.error( 'The color {} was not computed in the models'.format(color)) sys.exit(16) if args.delta_color_cut > 0: log.info("apply cut |delta color|<{}".format(args.delta_color_cut)) good = (np.abs(model_metadata["MODEL_" + color] - model_metadata["DATA_" + color]) < args.delta_color_cut) bad = ok & (~good) ok &= good if np.any(bad): vals = model_metadata["MODEL_" + color][bad] - model_metadata["DATA_" + color][bad] log.info(" discard {} stars with dcolor= {}".format( np.sum(bad), list(vals))) if args.min_color is not None: log.info("apply cut DATA_{}>{}".format(color, args.min_color)) good = (model_metadata["DATA_{}".format(color)] > args.min_color) bad = ok & (~good) ok &= good if np.any(bad): vals = model_metadata["DATA_{}".format(color)][bad] log.info(" discard {} stars with {}= {}".format( np.sum(bad), color, list(vals))) if args.chi2cut_nsig > 0: # automatically reject stars that ar chi2 outliers mchi2 = np.median(model_metadata["CHI2DOF"]) rmschi2 = np.std(model_metadata["CHI2DOF"]) maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2 log.info("apply cut CHI2DOF<{} based on chi2cut_nsig={}".format( maxchi2, args.chi2cut_nsig)) good = (model_metadata["CHI2DOF"] <= maxchi2) bad = ok & (~good) ok &= good if np.any(bad): log.info(" discard {} stars with CHI2DOF={}".format( np.sum(bad), list(model_metadata["CHI2DOF"][bad]))) ok = np.where(ok)[0] if ok.size == 0: log.error("selection cuts discarded all stars") sys.exit(12) nstars = model_flux.shape[0] nbad = nstars - ok.size if nbad > 0: log.warning("discarding %d star(s) out of %d because of cuts" % (nbad, nstars)) model_flux = model_flux[ok] model_fibers = model_fibers[ok] model_metadata = model_metadata[:][ok] # check that the model_fibers are actually standard stars fibermap = frame.fibermap ## check whether star fibers from args.models are consistent with fibers from fibermap ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit fibermap_std_indices = np.where(isStdStar(fibermap))[0] if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)): target_colnames, target_masks, survey = main_cmx_or_sv(fibermap) colname = target_colnames[0] for i in model_fibers % 500: log.error( "inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap" .format(i, fibermap["OBJTYPE"][i], colname, fibermap[colname][i])) sys.exit(12) # Make sure the fibers of interest aren't entirely masked. if np.sum( np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) == frame.nwave) == len(model_fibers): log.warning('All standard-star spectra are masked!') return fluxcalib = compute_flux_calibration( frame, model_wave, model_flux, model_fibers % 500, highest_throughput_nstars=args.highest_throughput, exposure_seeing_fwhm=args.seeing_fwhm) # QA if (args.qafile is not None): from desispec.io import write_qa_frame from desispec.io.qa import load_qa_frame from desispec.qa import qa_plots log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame_meta=frame.meta, flavor=frame.meta['FLAVOR']) # Run #import pdb; pdb.set_trace() qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib) # record inputs frame.meta['IN_FRAME'] = shorten_filename(args.infile) frame.meta['IN_SKY'] = shorten_filename(args.sky) frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat) frame.meta['STDMODEL'] = shorten_filename(args.models) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s" % args.outfile)
def main(args): log = get_logger() log.info("read frame") # read frame frame = read_frame(args.infile) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel = read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux, model_wave, model_fibers = read_stdstar_models(args.models) model_tuple = model_flux, model_wave, model_fibers # check that the model_fibers are actually standard stars fibermap = frame.fibermap model_fibers = model_fibers % 500 if np.any(fibermap['OBJTYPE'][model_fibers] != 'STD'): for i in model_fibers: log.error( "inconsistency with spectrum %d, OBJTYPE='%s' in fibermap" % (i, fibermap["OBJTYPE"][i])) sys.exit(12) #fluxcalib, indiv_stars = compute_flux_calibration(frame, model_wave, model_flux) fluxcalib = compute_flux_calibration(frame, model_wave, model_flux) # QA if (args.qafile is not None): log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run qaframe.run_qa('FLUXCALIB', (frame, fluxcalib, model_tuple)) #, indiv_stars)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib, model_tuple) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s" % args.outfile)
def run_pa(self, input_frame, skymodel): from desispec.sky import subtract_sky subtract_sky(input_frame, skymodel) return input_frame
def main(args=None, comm=None): if args is None: args = parse() # elif isinstance(args, (list, tuple)): # args = parse(args) log = get_logger() start_mpi_connect = time.time() if comm is not None: #- Use the provided comm to determine rank and size rank = comm.rank size = comm.size else: #- Check MPI flags and determine the comm, rank, and size given the arguments comm, rank, size = assign_mpi(do_mpi=args.mpi, do_batch=args.batch, log=log) stop_mpi_connect = time.time() #- Start timer; only print log messages from rank 0 (others are silent) timer = desiutil.timer.Timer(silent=(rank > 0)) #- Fill in timing information for steps before we had the timer created if args.starttime is not None: timer.start('startup', starttime=args.starttime) timer.stop('startup', stoptime=start_imports) timer.start('imports', starttime=start_imports) timer.stop('imports', stoptime=stop_imports) timer.start('mpi_connect', starttime=start_mpi_connect) timer.stop('mpi_connect', stoptime=stop_mpi_connect) #- Freeze IERS after parsing args so that it doesn't bother if only --help timer.start('freeze_iers') desiutil.iers.freeze_iers() timer.stop('freeze_iers') #- Preflight checks timer.start('preflight') if rank > 0: #- Let rank 0 fetch these, and then broadcast args, hdr, camhdr = None, None, None else: args, hdr, camhdr = update_args_with_headers(args) ## Make sure badamps is formatted properly if comm is not None and rank == 0 and args.badamps is not None: args.badamps = validate_badamps(args.badamps) if comm is not None: args = comm.bcast(args, root=0) hdr = comm.bcast(hdr, root=0) camhdr = comm.bcast(camhdr, root=0) known_obstype = [ 'SCIENCE', 'ARC', 'FLAT', 'ZERO', 'DARK', 'TESTARC', 'TESTFLAT', 'PIXFLAT', 'SKY', 'TWILIGHT', 'OTHER' ] if args.obstype not in known_obstype: raise RuntimeError('obstype {} not in {}'.format( args.obstype, known_obstype)) timer.stop('preflight') #------------------------------------------------------------------------- #- Create and submit a batch job if requested if args.batch: #exp_str = '{:08d}'.format(args.expid) jobdesc = args.obstype.lower() if args.obstype == 'SCIENCE': # if not doing pre-stdstar fitting or stdstar fitting and if there is # no flag stopping flux calibration, set job to poststdstar if args.noprestdstarfit and args.nostdstarfit and ( not args.nofluxcalib): jobdesc = 'poststdstar' # elif told not to do std or post stdstar but the flag for prestdstar isn't set, # then perform prestdstar elif (not args.noprestdstarfit ) and args.nostdstarfit and args.nofluxcalib: jobdesc = 'prestdstar' #elif (not args.noprestdstarfit) and (not args.nostdstarfit) and (not args.nofluxcalib): # jobdesc = 'science' scriptfile = create_desi_proc_batch_script(night=args.night, exp=args.expid, cameras=args.cameras,\ jobdesc=jobdesc, queue=args.queue, runtime=args.runtime,\ batch_opts=args.batch_opts, timingfile=args.timingfile, system_name=args.system_name) err = 0 if not args.nosubmit: err = subprocess.call(['sbatch', scriptfile]) sys.exit(err) #------------------------------------------------------------------------- #- Proceeding with running #- What are we going to do? if rank == 0: log.info('----------') log.info('Input {}'.format(args.input)) log.info('Night {} expid {}'.format(args.night, args.expid)) log.info('Obstype {}'.format(args.obstype)) log.info('Cameras {}'.format(args.cameras)) log.info('Output root {}'.format(desispec.io.specprod_root())) log.info('----------') #- Create output directories if needed if rank == 0: preprocdir = os.path.dirname( findfile('preproc', args.night, args.expid, 'b0')) expdir = os.path.dirname( findfile('frame', args.night, args.expid, 'b0')) os.makedirs(preprocdir, exist_ok=True) os.makedirs(expdir, exist_ok=True) #- Wait for rank 0 to make directories before proceeding if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Preproc #- All obstypes get preprocessed timer.start('fibermap') #- Assemble fibermap for science exposures fibermap = None fibermap_ok = None if rank == 0 and args.obstype == 'SCIENCE': fibermap = findfile('fibermap', args.night, args.expid) if not os.path.exists(fibermap): tmp = findfile('preproc', args.night, args.expid, 'b0') preprocdir = os.path.dirname(tmp) fibermap = os.path.join(preprocdir, os.path.basename(fibermap)) log.info('Creating fibermap {}'.format(fibermap)) cmd = 'assemble_fibermap -n {} -e {} -o {}'.format( args.night, args.expid, fibermap) if args.badamps is not None: cmd += ' --badamps={}'.format(args.badamps) runcmd(cmd, inputs=[], outputs=[fibermap]) fibermap_ok = os.path.exists(fibermap) #- Some commissioning files didn't have coords* files that caused assemble_fibermap to fail #- these are well known failures with no other solution, so for those, just force creation #- of a fibermap with null coordinate information if not fibermap_ok and int(args.night) < 20200310: log.info( "Since night is before 20200310, trying to force fibermap creation without coords file" ) cmd += ' --force' runcmd(cmd, inputs=[], outputs=[fibermap]) fibermap_ok = os.path.exists(fibermap) #- If assemble_fibermap failed and obstype is SCIENCE, exit now if comm is not None: fibermap_ok = comm.bcast(fibermap_ok, root=0) if args.obstype == 'SCIENCE' and not fibermap_ok: sys.stdout.flush() if rank == 0: log.critical( 'assemble_fibermap failed for science exposure; exiting now') sys.exit(13) #- Wait for rank 0 to make fibermap if needed if comm is not None: fibermap = comm.bcast(fibermap, root=0) timer.stop('fibermap') if not (args.obstype in ['SCIENCE'] and args.noprestdstarfit): timer.start('preproc') for i in range(rank, len(args.cameras), size): camera = args.cameras[i] outfile = findfile('preproc', args.night, args.expid, camera) outdir = os.path.dirname(outfile) cmd = "desi_preproc -i {} -o {} --outdir {} --cameras {}".format( args.input, outfile, outdir, camera) if args.scattered_light: cmd += " --scattered-light" if fibermap is not None: cmd += " --fibermap {}".format(fibermap) if not args.obstype in ['ARC']: # never model variance for arcs if not args.no_model_pixel_variance: cmd += " --model-variance" runcmd(cmd, inputs=[args.input], outputs=[outfile]) timer.stop('preproc') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Get input PSFs timer.start('findpsf') input_psf = dict() if rank == 0: for camera in args.cameras: if args.psf is not None: input_psf[camera] = args.psf elif args.calibnight is not None: # look for a psfnight psf for this calib night psfnightfile = findfile('psfnight', args.calibnight, args.expid, camera) if not os.path.isfile(psfnightfile): log.error("no {}".format(psfnightfile)) raise IOError("no {}".format(psfnightfile)) input_psf[camera] = psfnightfile else: # look for a psfnight psf psfnightfile = findfile('psfnight', args.night, args.expid, camera) if os.path.isfile(psfnightfile): input_psf[camera] = psfnightfile elif args.most_recent_calib: nightfile = find_most_recent(args.night, file_type='psfnight') if nightfile is None: input_psf[camera] = findcalibfile( [hdr, camhdr[camera]], 'PSF') else: input_psf[camera] = nightfile else: input_psf[camera] = findcalibfile([hdr, camhdr[camera]], 'PSF') log.info("Will use input PSF : {}".format(input_psf[camera])) if comm is not None: input_psf = comm.bcast(input_psf, root=0) timer.stop('findpsf') #------------------------------------------------------------------------- #- Traceshift if ( args.obstype in ['FLAT', 'TESTFLAT', 'SKY', 'TWILIGHT'] ) or \ ( args.obstype in ['SCIENCE'] and (not args.noprestdstarfit) ): timer.start('traceshift') if rank == 0 and args.traceshift: log.info('Starting traceshift at {}'.format(time.asctime())) for i in range(rank, len(args.cameras), size): camera = args.cameras[i] preprocfile = findfile('preproc', args.night, args.expid, camera) inpsf = input_psf[camera] outpsf = findfile('psf', args.night, args.expid, camera) if not os.path.isfile(outpsf): if args.traceshift: cmd = "desi_compute_trace_shifts" cmd += " -i {}".format(preprocfile) cmd += " --psf {}".format(inpsf) cmd += " --outpsf {}".format(outpsf) cmd += " --degxx 2 --degxy 0" if args.obstype in ['FLAT', 'TESTFLAT', 'TWILIGHT']: cmd += " --continuum" else: cmd += " --degyx 2 --degyy 0" if args.obstype in ['SCIENCE', 'SKY']: cmd += ' --sky' else: cmd = "ln -s {} {}".format(inpsf, outpsf) runcmd(cmd, inputs=[preprocfile, inpsf], outputs=[outpsf]) else: log.info("PSF {} exists".format(outpsf)) timer.stop('traceshift') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- PSF #- MPI parallelize this step if args.obstype in ['ARC', 'TESTARC']: timer.start('arc_traceshift') if rank == 0: log.info('Starting traceshift before specex PSF fit at {}'.format( time.asctime())) for i in range(rank, len(args.cameras), size): camera = args.cameras[i] preprocfile = findfile('preproc', args.night, args.expid, camera) inpsf = input_psf[camera] outpsf = findfile('psf', args.night, args.expid, camera) outpsf = replace_prefix(outpsf, "psf", "shifted-input-psf") if not os.path.isfile(outpsf): cmd = "desi_compute_trace_shifts" cmd += " -i {}".format(preprocfile) cmd += " --psf {}".format(inpsf) cmd += " --outpsf {}".format(outpsf) cmd += " --degxx 0 --degxy 0 --degyx 0 --degyy 0" cmd += ' --arc-lamps' runcmd(cmd, inputs=[preprocfile, inpsf], outputs=[outpsf]) else: log.info("PSF {} exists".format(outpsf)) timer.stop('arc_traceshift') if comm is not None: comm.barrier() timer.start('psf') if rank == 0: log.info('Starting specex PSF fitting at {}'.format( time.asctime())) if rank > 0: cmds = inputs = outputs = None else: cmds = dict() inputs = dict() outputs = dict() for camera in args.cameras: preprocfile = findfile('preproc', args.night, args.expid, camera) tmpname = findfile('psf', args.night, args.expid, camera) inpsf = replace_prefix(tmpname, "psf", "shifted-input-psf") outpsf = replace_prefix(tmpname, "psf", "fit-psf") log.info("now run specex psf fit") cmd = 'desi_compute_psf' cmd += ' --input-image {}'.format(preprocfile) cmd += ' --input-psf {}'.format(inpsf) cmd += ' --output-psf {}'.format(outpsf) # look for fiber blacklist cfinder = CalibFinder([hdr, camhdr[camera]]) blacklistkey = "FIBERBLACKLIST" if not cfinder.haskey(blacklistkey) and cfinder.haskey( "BROKENFIBERS"): log.warning( "BROKENFIBERS yaml keyword deprecated, please use FIBERBLACKLIST" ) blacklistkey = "BROKENFIBERS" if cfinder.haskey(blacklistkey): blacklist = cfinder.value(blacklistkey) cmd += ' --broken-fibers {}'.format(blacklist) if rank == 0: log.warning('broken fibers: {}'.format(blacklist)) if not os.path.exists(outpsf): cmds[camera] = cmd inputs[camera] = [preprocfile, inpsf] outputs[camera] = [ outpsf, ] if comm is not None: cmds = comm.bcast(cmds, root=0) inputs = comm.bcast(inputs, root=0) outputs = comm.bcast(outputs, root=0) #- split communicator by 20 (number of bundles) group_size = 20 if (rank == 0) and (size % group_size != 0): log.warning( 'MPI size={} should be evenly divisible by {}'.format( size, group_size)) group = rank // group_size num_groups = (size + group_size - 1) // group_size comm_group = comm.Split(color=group) if rank == 0: log.info( f'Fitting PSFs with {num_groups} sub-communicators of size {group_size}' ) for i in range(group, len(args.cameras), num_groups): camera = args.cameras[i] if camera in cmds: cmdargs = cmds[camera].split()[1:] cmdargs = desispec.scripts.specex.parse(cmdargs) if comm_group.rank == 0: print('RUNNING: {}'.format(cmds[camera])) t0 = time.time() timestamp = time.asctime() log.info( f'MPI group {group} ranks {rank}-{rank+group_size-1} fitting PSF for {camera} at {timestamp}' ) try: desispec.scripts.specex.main(cmdargs, comm=comm_group) except Exception as e: if comm_group.rank == 0: log.error( f'FAILED: MPI group {group} ranks {rank}-{rank+group_size-1} camera {camera}' ) log.error('FAILED: {}'.format(cmds[camera])) log.error(e) if comm_group.rank == 0: specex_time = time.time() - t0 log.info( f'specex fit for {camera} took {specex_time:.1f} seconds' ) comm.barrier() else: log.warning( 'fitting PSFs without MPI parallelism; this will be SLOW') for camera in args.cameras: if camera in cmds: runcmd(cmds[camera], inputs=inputs[camera], outputs=outputs[camera]) if comm is not None: comm.barrier() # loop on all cameras and interpolate bad fibers for camera in args.cameras[rank::size]: t0 = time.time() log.info(f'Rank {rank} interpolating {camera} PSF over bad fibers') # look for fiber blacklist cfinder = CalibFinder([hdr, camhdr[camera]]) blacklistkey = "FIBERBLACKLIST" if not cfinder.haskey(blacklistkey) and cfinder.haskey( "BROKENFIBERS"): log.warning( "BROKENFIBERS yaml keyword deprecated, please use FIBERBLACKLIST" ) blacklistkey = "BROKENFIBERS" if cfinder.haskey(blacklistkey): fiberblacklist = cfinder.value(blacklistkey) tmpname = findfile('psf', args.night, args.expid, camera) inpsf = replace_prefix(tmpname, "psf", "fit-psf") outpsf = replace_prefix(tmpname, "psf", "fit-psf-fixed-blacklisted") if os.path.isfile(inpsf) and not os.path.isfile(outpsf): cmd = 'desi_interpolate_fiber_psf' cmd += ' --infile {}'.format(inpsf) cmd += ' --outfile {}'.format(outpsf) cmd += ' --fibers {}'.format(fiberblacklist) log.info( 'For camera {} interpolating PSF for broken fibers: {}' .format(camera, fiberblacklist)) runcmd(cmd, inputs=[inpsf], outputs=[outpsf]) if os.path.isfile(outpsf): os.rename( inpsf, inpsf.replace("fit-psf", "fit-psf-before-blacklisted-fix")) subprocess.call('cp {} {}'.format(outpsf, inpsf), shell=True) dt = time.time() - t0 log.info( f'Rank {rank} {camera} PSF interpolation took {dt:.1f} sec') timer.stop('psf') #------------------------------------------------------------------------- #- Merge PSF of night if applicable #if args.obstype in ['ARC']: if False: if rank == 0: for camera in args.cameras: psfnightfile = findfile('psfnight', args.night, args.expid, camera) if not os.path.isfile( psfnightfile ): # we still don't have a psf night, see if we can compute it ... psfs = glob.glob( findfile('psf', args.night, args.expid, camera).replace("psf", "fit-psf").replace( str(args.expid), "*")) log.info( "Number of PSF for night={} camera={} = {}".format( args.night, camera, len(psfs))) if len(psfs) > 4: # lets do it! log.info("Computing psfnight ...") dirname = os.path.dirname(psfnightfile) if not os.path.isdir(dirname): os.makedirs(dirname) desispec.scripts.specex.mean_psf(psfs, psfnightfile) if os.path.isfile(psfnightfile): # now use this one input_psf[camera] = psfnightfile #------------------------------------------------------------------------- #- Extract #- This is MPI parallel so handle a bit differently # maybe add ARC and TESTARC too if ( args.obstype in ['FLAT', 'TESTFLAT', 'SKY', 'TWILIGHT'] ) or \ ( args.obstype in ['SCIENCE'] and (not args.noprestdstarfit) ): timer.start('extract') if rank == 0: log.info('Starting extractions at {}'.format(time.asctime())) if rank > 0: cmds = inputs = outputs = None else: cmds = dict() inputs = dict() outputs = dict() for camera in args.cameras: cmd = 'desi_extract_spectra' #- Based on data from SM1-SM8, looking at central and edge fibers #- with in mind overlapping arc lamps lines if camera.startswith('b'): cmd += ' -w 3600.0,5800.0,0.8' elif camera.startswith('r'): cmd += ' -w 5760.0,7620.0,0.8' elif camera.startswith('z'): cmd += ' -w 7520.0,9824.0,0.8' preprocfile = findfile('preproc', args.night, args.expid, camera) psffile = findfile('psf', args.night, args.expid, camera) framefile = findfile('frame', args.night, args.expid, camera) cmd += ' -i {}'.format(preprocfile) cmd += ' -p {}'.format(psffile) cmd += ' -o {}'.format(framefile) cmd += ' --psferr 0.1' if args.obstype == 'SCIENCE' or args.obstype == 'SKY': if rank == 0: log.info('Include barycentric correction') cmd += ' --barycentric-correction' if not os.path.exists(framefile): cmds[camera] = cmd inputs[camera] = [preprocfile, psffile] outputs[camera] = [ framefile, ] #- TODO: refactor/combine this with PSF comm splitting logic if comm is not None: cmds = comm.bcast(cmds, root=0) inputs = comm.bcast(inputs, root=0) outputs = comm.bcast(outputs, root=0) #- split communicator by 20 (number of bundles) extract_size = 20 if (rank == 0) and (size % extract_size != 0): log.warning( 'MPI size={} should be evenly divisible by {}'.format( size, extract_size)) extract_group = rank // extract_size num_extract_groups = (size + extract_size - 1) // extract_size comm_extract = comm.Split(color=extract_group) for i in range(extract_group, len(args.cameras), num_extract_groups): camera = args.cameras[i] if camera in cmds: cmdargs = cmds[camera].split()[1:] extract_args = desispec.scripts.extract.parse(cmdargs) if comm_extract.rank == 0: print('RUNNING: {}'.format(cmds[camera])) desispec.scripts.extract.main_mpi(extract_args, comm=comm_extract) comm.barrier() else: log.warning( 'running extractions without MPI parallelism; this will be SLOW' ) for camera in args.cameras: if camera in cmds: runcmd(cmds[camera], inputs=inputs[camera], outputs=outputs[camera]) timer.stop('extract') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Fiberflat if args.obstype in ['FLAT', 'TESTFLAT']: timer.start('fiberflat') if rank == 0: log.info('Starting fiberflats at {}'.format(time.asctime())) for i in range(rank, len(args.cameras), size): camera = args.cameras[i] framefile = findfile('frame', args.night, args.expid, camera) fiberflatfile = findfile('fiberflat', args.night, args.expid, camera) cmd = "desi_compute_fiberflat" cmd += " -i {}".format(framefile) cmd += " -o {}".format(fiberflatfile) runcmd(cmd, inputs=[ framefile, ], outputs=[ fiberflatfile, ]) timer.stop('fiberflat') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Average and auto-calib fiberflats of night if applicable #if args.obstype in ['FLAT']: if False: if rank == 0: fiberflatnightfile = findfile('fiberflatnight', args.night, args.expid, args.cameras[0]) fiberflatdirname = os.path.dirname(fiberflatnightfile) if not os.path.isfile(fiberflatnightfile) and len( args.cameras ) >= 6: # we still don't have them, see if we can compute them, but need at least 2 spectros ... flats = glob.glob( findfile('fiberflat', args.night, args.expid, "b0").replace(str(args.expid), "*").replace("b0", "*")) log.info("Number of fiberflat for night {} = {}".format( args.night, len(flats))) if len(flats) >= 3 * 4 * len( args.cameras ): # lets do it! (3 exposures x 4 lamps x N cameras) log.info( "Computing fiberflatnight per lamp and camera ...") tmpdir = os.path.join(fiberflatdirname, "tmp") if not os.path.isdir(tmpdir): os.makedirs(tmpdir) log.info( "First average measurements per camera and per lamp") average_flats = dict() for camera in args.cameras: # list of flats for this camera flats_for_this_camera = [] for flat in flats: if flat.find(camera) >= 0: flats_for_this_camera.append(flat) #log.info("For camera {} , flats = {}".format(camera,flats_for_this_camera)) #sys.exit(12) # average per lamp (and camera) average_flats[camera] = list() for lampbox in range(4): ofile = os.path.join( tmpdir, "fiberflatnight-camera-{}-lamp-{}.fits".format( camera, lampbox)) if not os.path.isfile(ofile): log.info( "Average flat for camera {} and lamp box #{}" .format(camera, lampbox)) pg = "CALIB DESI-CALIB-0{} LEDs only".format( lampbox) cmd = "desi_average_fiberflat --program '{}' --outfile {} -i ".format( pg, ofile) for flat in flats_for_this_camera: cmd += " {} ".format(flat) runcmd(cmd, inputs=flats_for_this_camera, outputs=[ ofile, ]) if os.path.isfile(ofile): average_flats[camera].append(ofile) else: log.info("Will use existing {}".format(ofile)) average_flats[camera].append(ofile) log.info( "Auto-calibration across lamps and spectro per camera arm (b,r,z)" ) for camera_arm in ["b", "r", "z"]: cameras_for_this_arm = [] flats_for_this_arm = [] for camera in args.cameras: if camera[0].lower() == camera_arm: cameras_for_this_arm.append(camera) if camera in average_flats: for flat in average_flats[camera]: flats_for_this_arm.append(flat) cmd = "desi_autocalib_fiberflat --night {} --arm {} -i ".format( args.night, camera_arm) for flat in flats_for_this_arm: cmd += " {} ".format(flat) runcmd(cmd, inputs=flats_for_this_arm, outputs=[]) log.info("Done with fiber flats per night") if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Get input fiberflat if args.obstype in ['SCIENCE', 'SKY'] and (not args.nofiberflat): timer.start('find_fiberflat') input_fiberflat = dict() if rank == 0: for camera in args.cameras: if args.fiberflat is not None: input_fiberflat[camera] = args.fiberflat elif args.calibnight is not None: # look for a fiberflatnight for this calib night fiberflatnightfile = findfile('fiberflatnight', args.calibnight, args.expid, camera) if not os.path.isfile(fiberflatnightfile): log.error("no {}".format(fiberflatnightfile)) raise IOError("no {}".format(fiberflatnightfile)) input_fiberflat[camera] = fiberflatnightfile else: # look for a fiberflatnight fiberflat fiberflatnightfile = findfile('fiberflatnight', args.night, args.expid, camera) if os.path.isfile(fiberflatnightfile): input_fiberflat[camera] = fiberflatnightfile elif args.most_recent_calib: nightfile = find_most_recent( args.night, file_type='fiberflatnight') if nightfile is None: input_fiberflat[camera] = findcalibfile( [hdr, camhdr[camera]], 'FIBERFLAT') else: input_fiberflat[camera] = nightfile else: input_fiberflat[camera] = findcalibfile( [hdr, camhdr[camera]], 'FIBERFLAT') log.info("Will use input FIBERFLAT: {}".format( input_fiberflat[camera])) if comm is not None: input_fiberflat = comm.bcast(input_fiberflat, root=0) timer.stop('find_fiberflat') #------------------------------------------------------------------------- #- Apply fiberflat and write fframe file if args.obstype in ['SCIENCE', 'SKY'] and args.fframe and \ ( not args.nofiberflat ) and (not args.noprestdstarfit): timer.start('apply_fiberflat') if rank == 0: log.info('Applying fiberflat at {}'.format(time.asctime())) for i in range(rank, len(args.cameras), size): camera = args.cameras[i] fframefile = findfile('fframe', args.night, args.expid, camera) if not os.path.exists(fframefile): framefile = findfile('frame', args.night, args.expid, camera) fr = desispec.io.read_frame(framefile) flatfilename = input_fiberflat[camera] if flatfilename is not None: ff = desispec.io.read_fiberflat(flatfilename) fr.meta['FIBERFLT'] = desispec.io.shorten_filename( flatfilename) apply_fiberflat(fr, ff) fframefile = findfile('fframe', args.night, args.expid, camera) desispec.io.write_frame(fframefile, fr) else: log.warning( "Missing fiberflat for camera {}".format(camera)) timer.stop('apply_fiberflat') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Select random sky fibers (inplace update of frame file) #- TODO: move this to a function somewhere #- TODO: this assigns different sky fibers to each frame of same spectrograph if (args.obstype in [ 'SKY', 'SCIENCE' ]) and (not args.noskysub) and (not args.noprestdstarfit): timer.start('picksky') if rank == 0: log.info('Picking sky fibers at {}'.format(time.asctime())) for i in range(rank, len(args.cameras), size): camera = args.cameras[i] framefile = findfile('frame', args.night, args.expid, camera) orig_frame = desispec.io.read_frame(framefile) #- Make a copy so that we can apply fiberflat fr = deepcopy(orig_frame) if np.any(fr.fibermap['OBJTYPE'] == 'SKY'): log.info('{} sky fibers already set; skipping'.format( os.path.basename(framefile))) continue #- Apply fiberflat then select random fibers below a flux cut flatfilename = input_fiberflat[camera] if flatfilename is None: log.error("No fiberflat for {}".format(camera)) continue ff = desispec.io.read_fiberflat(flatfilename) apply_fiberflat(fr, ff) sumflux = np.sum(fr.flux, axis=1) fluxcut = np.percentile(sumflux, 30) iisky = np.where(sumflux < fluxcut)[0] iisky = np.random.choice(iisky, size=100, replace=False) #- Update fibermap or original frame and write out orig_frame.fibermap['OBJTYPE'][iisky] = 'SKY' orig_frame.fibermap['DESI_TARGET'][iisky] |= desi_mask.SKY desispec.io.write_frame(framefile, orig_frame) timer.stop('picksky') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Sky subtraction if args.obstype in [ 'SCIENCE', 'SKY' ] and (not args.noskysub) and (not args.noprestdstarfit): timer.start('skysub') if rank == 0: log.info('Starting sky subtraction at {}'.format(time.asctime())) for i in range(rank, len(args.cameras), size): camera = args.cameras[i] framefile = findfile('frame', args.night, args.expid, camera) hdr = fitsio.read_header(framefile, 'FLUX') fiberflatfile = input_fiberflat[camera] if fiberflatfile is None: log.error("No fiberflat for {}".format(camera)) continue skyfile = findfile('sky', args.night, args.expid, camera) cmd = "desi_compute_sky" cmd += " -i {}".format(framefile) cmd += " --fiberflat {}".format(fiberflatfile) cmd += " --o {}".format(skyfile) if args.no_extra_variance: cmd += " --no-extra-variance" if not args.no_sky_wavelength_adjustment: cmd += " --adjust-wavelength" if not args.no_sky_lsf_adjustment: cmd += " --adjust-lsf" runcmd(cmd, inputs=[framefile, fiberflatfile], outputs=[ skyfile, ]) #- sframe = flatfielded sky-subtracted but not flux calibrated frame #- Note: this re-reads and re-does steps previously done for picking #- sky fibers; desi_proc is about human efficiency, #- not I/O or CPU efficiency... sframefile = desispec.io.findfile('sframe', args.night, args.expid, camera) if not os.path.exists(sframefile): frame = desispec.io.read_frame(framefile) fiberflat = desispec.io.read_fiberflat(fiberflatfile) sky = desispec.io.read_sky(skyfile) apply_fiberflat(frame, fiberflat) subtract_sky(frame, sky, apply_throughput_correction=True) frame.meta['IN_SKY'] = shorten_filename(skyfile) frame.meta['FIBERFLT'] = shorten_filename(fiberflatfile) desispec.io.write_frame(sframefile, frame) timer.stop('skysub') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Standard Star Fitting if args.obstype in ['SCIENCE',] and \ (not args.noskysub ) and \ (not args.nostdstarfit) : timer.start('stdstarfit') if rank == 0: log.info('Starting flux calibration at {}'.format(time.asctime())) #- Group inputs by spectrograph framefiles = dict() skyfiles = dict() fiberflatfiles = dict() night, expid = args.night, args.expid #- shorter for camera in args.cameras: sp = int(camera[1]) if sp not in framefiles: framefiles[sp] = list() skyfiles[sp] = list() fiberflatfiles[sp] = list() framefiles[sp].append(findfile('frame', night, expid, camera)) skyfiles[sp].append(findfile('sky', night, expid, camera)) fiberflatfiles[sp].append(input_fiberflat[camera]) #- Hardcoded stdstar model version starmodels = os.path.join(os.getenv('DESI_BASIS_TEMPLATES'), 'stdstar_templates_v2.2.fits') #- Fit stdstars per spectrograph (not per-camera) spectro_nums = sorted(framefiles.keys()) ## for sp in spectro_nums[rank::size]: for i in range(rank, len(spectro_nums), size): sp = spectro_nums[i] stdfile = findfile('stdstars', night, expid, spectrograph=sp) cmd = "desi_fit_stdstars" cmd += " --frames {}".format(' '.join(framefiles[sp])) cmd += " --skymodels {}".format(' '.join(skyfiles[sp])) cmd += " --fiberflats {}".format(' '.join(fiberflatfiles[sp])) cmd += " --starmodels {}".format(starmodels) cmd += " --outfile {}".format(stdfile) cmd += " --delta-color 0.1" if args.maxstdstars is not None: cmd += " --maxstdstars {}".format(args.maxstdstars) inputs = framefiles[sp] + skyfiles[sp] + fiberflatfiles[sp] runcmd(cmd, inputs=inputs, outputs=[stdfile]) timer.stop('stdstarfit') if comm is not None: comm.barrier() # ------------------------------------------------------------------------- # - Flux calibration if args.obstype in ['SCIENCE'] and \ (not args.noskysub) and \ (not args.nofluxcalib): timer.start('fluxcalib') night, expid = args.night, args.expid #- shorter #- Compute flux calibration vectors per camera for camera in args.cameras[rank::size]: framefile = findfile('frame', night, expid, camera) skyfile = findfile('sky', night, expid, camera) spectrograph = int(camera[1]) stdfile = findfile('stdstars', night, expid, spectrograph=spectrograph) calibfile = findfile('fluxcalib', night, expid, camera) fiberflatfile = input_fiberflat[camera] cmd = "desi_compute_fluxcalibration" cmd += " --infile {}".format(framefile) cmd += " --sky {}".format(skyfile) cmd += " --fiberflat {}".format(fiberflatfile) cmd += " --models {}".format(stdfile) cmd += " --outfile {}".format(calibfile) cmd += " --delta-color-cut 0.1" inputs = [framefile, skyfile, fiberflatfile, stdfile] runcmd(cmd, inputs=inputs, outputs=[ calibfile, ]) timer.stop('fluxcalib') if comm is not None: comm.barrier() #------------------------------------------------------------------------- #- Applying flux calibration if args.obstype in [ 'SCIENCE', ] and (not args.noskysub) and (not args.nofluxcalib): night, expid = args.night, args.expid #- shorter timer.start('applycalib') if rank == 0: log.info('Starting cframe file creation at {}'.format( time.asctime())) for camera in args.cameras[rank::size]: framefile = findfile('frame', night, expid, camera) skyfile = findfile('sky', night, expid, camera) spectrograph = int(camera[1]) stdfile = findfile('stdstars', night, expid, spectrograph=spectrograph) calibfile = findfile('fluxcalib', night, expid, camera) cframefile = findfile('cframe', night, expid, camera) fiberflatfile = input_fiberflat[camera] cmd = "desi_process_exposure" cmd += " --infile {}".format(framefile) cmd += " --fiberflat {}".format(fiberflatfile) cmd += " --sky {}".format(skyfile) cmd += " --calib {}".format(calibfile) cmd += " --outfile {}".format(cframefile) cmd += " --cosmics-nsig 6" if args.no_xtalk: cmd += " --no-xtalk" inputs = [framefile, fiberflatfile, skyfile, calibfile] runcmd(cmd, inputs=inputs, outputs=[ cframefile, ]) if comm is not None: comm.barrier() timer.stop('applycalib') #------------------------------------------------------------------------- #- Wrap up # if rank == 0: # report = timer.report() # log.info('Rank 0 timing report:\n' + report) if comm is not None: timers = comm.gather(timer, root=0) else: timers = [ timer, ] if rank == 0: stats = desiutil.timer.compute_stats(timers) log.info('Timing summary statistics:\n' + json.dumps(stats, indent=2)) if args.timingfile: if os.path.exists(args.timingfile): with open(args.timingfile) as fx: previous_stats = json.load(fx) #- augment previous_stats with new entries, but don't overwrite old for name in stats: if name not in previous_stats: previous_stats[name] = stats[name] stats = previous_stats tmpfile = args.timingfile + '.tmp' with open(tmpfile, 'w') as fx: json.dump(stats, fx, indent=2) os.rename(tmpfile, args.timingfile) if rank == 0: log.info('All done at {}'.format(time.asctime()))
def main(args): log = get_logger() log.info("read frame") # read frame frame = read_frame(args.infile) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel = read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models( args.models) if args.chi2cut > 0: ok = np.where(model_metadata["CHI2DOF"] < args.chi2cut)[0] if ok.size == 0: log.error("chi2cut has discarded all stars") sys.exit(12) nstars = model_flux.shape[0] nbad = nstars - ok.size if nbad > 0: log.warning("discarding %d star(s) out of %d because of chi2cut" % (nbad, nstars)) model_flux = model_flux[ok] model_fibers = model_fibers[ok] model_metadata = model_metadata[:][ok] if args.delta_color_cut > 0: ok = np.where( np.abs(model_metadata["MODEL_G-R"] - model_metadata["DATA_G-R"]) < args.delta_color_cut)[0] nstars = model_flux.shape[0] nbad = nstars - ok.size if nbad > 0: log.warning( "discarding %d star(s) out of %d because |delta_color|>%f" % (nbad, nstars, args.delta_color_cut)) model_flux = model_flux[ok] model_fibers = model_fibers[ok] model_metadata = model_metadata[:][ok] # automatically reject stars that ar chi2 outliers if args.chi2cut_nsig > 0: mchi2 = np.median(model_metadata["CHI2DOF"]) rmschi2 = np.std(model_metadata["CHI2DOF"]) maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2 ok = np.where(model_metadata["CHI2DOF"] <= maxchi2)[0] nstars = model_flux.shape[0] nbad = nstars - ok.size if nbad > 0: log.warning( "discarding %d star(s) out of %d because reduced chi2 outliers (at %d sigma, giving rchi2<%f )" % (nbad, nstars, args.chi2cut_nsig, maxchi2)) model_flux = model_flux[ok] model_fibers = model_fibers[ok] model_metadata = model_metadata[:][ok] # check that the model_fibers are actually standard stars fibermap = frame.fibermap ## check whether star fibers from args.models are consistent with fibers from fibermap ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit w = np.where(fibermap["OBJTYPE"][model_fibers % 500] != 'STD')[0] if len(w) > 0: for i in model_fibers % 500: log.error( "inconsistency with spectrum %d, OBJTYPE='%s' in fibermap" % (i, fibermap["OBJTYPE"][i])) sys.exit(12) fluxcalib = compute_flux_calibration(frame, model_wave, model_flux, model_fibers % 500) # QA if (args.qafile is not None): log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run #import pdb; pdb.set_trace() qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s" % args.outfile)
def main(args): log = get_logger() if (args.fiberflat is None) and (args.sky is None) and (args.calib is None): log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?') sys.exit(12) frame = read_frame(args.infile) #- Raw scores already added in extraction, but just in case they weren't #- it is harmless to rerun to make sure we have them. compute_and_append_frame_scores(frame,suffix="RAW") if args.cosmics_nsig>0 and args.sky==None : # Reject cosmics (otherwise do it after sky subtraction) log.info("cosmics ray 1D rejection") reject_cosmic_rays_1d(frame,args.cosmics_nsig) if args.fiberflat!=None : log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat to all fibers apply_fiberflat(frame, fiberflat) compute_and_append_frame_scores(frame,suffix="FFLAT") if args.sky!=None : # read sky skymodel=read_sky(args.sky) if args.cosmics_nsig>0 : # use a copy the frame (not elegant but robust) copied_frame = copy.deepcopy(frame) # first subtract sky without throughput correction subtract_sky(copied_frame, skymodel, apply_throughput_correction = False) # then find cosmics log.info("cosmics ray 1D rejection after sky subtraction") reject_cosmic_rays_1d(copied_frame,args.cosmics_nsig) # copy mask frame.mask = copied_frame.mask # and (re-)subtract sky, but just the correction term subtract_sky(frame, skymodel, apply_throughput_correction = (not args.no_sky_throughput_correction) ) else : # subtract sky subtract_sky(frame, skymodel, apply_throughput_correction = (not args.no_sky_throughput_correction) ) compute_and_append_frame_scores(frame,suffix="SKYSUB") if args.calib!=None : log.info("calibrate") # read calibration fluxcalib=read_flux_calibration(args.calib) # apply calibration apply_flux_calibration(frame, fluxcalib) # Ensure that ivars are set to 0 for all values if any designated # fibermask bit is set. Also flips a bits for each frame.mask value using specmask.BADFIBER frame = get_fiberbitmasked_frame(frame,bitmask="flux",ivar_framemask=True) compute_and_append_frame_scores(frame,suffix="CALIB") # save output write_frame(args.outfile, frame, units='10**-17 erg/(s cm2 Angstrom)') log.info("successfully wrote %s"%args.outfile)
def get_skyres(cframes, sub_sky=False, flatten=True): """ Args: cframes: str or list Single cframe or a list of them sub_sky: bool, optional Subtract the sky? This should probably not be done flatten: bool, optional Return a flat, 1D array for each variable combine: bool, optional combine the individual sky fibers? Median 'smash' Returns: wave : ndarray flux : ndarray res : ndarray ivar : ndarray """ from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.sky import subtract_sky if isinstance(cframes, list): all_wave, all_flux, all_res, all_ivar = [], [], [], [] for cframe_file in cframes: wave, flux, res, ivar = get_skyres(cframe_file, flatten=flatten) # Save all_wave.append(wave) all_flux.append(flux) all_res.append(res) all_ivar.append(ivar) # Concatenate -- Shape is preserved (nfibers, npix) twave = np.concatenate(all_wave) tflux = np.concatenate(all_flux) tres = np.concatenate(all_res) tivar = np.concatenate(all_ivar) # Return return twave, tflux, tres, tivar cframe = read_frame(cframes, skip_resolution=True) if cframe.meta['FLAVOR'] in ['flat', 'arc']: raise ValueError("Bad flavor for exposure: {:s}".format(cframes)) # Sky sky_file = cframes.replace('cframe', 'sky') skymodel = read_sky(sky_file) if sub_sky: subtract_sky(cframe, skymodel) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] # Flux calibrated ivar = cframe.ivar[skyfibers] # Flux calibrated flux = skymodel.flux[skyfibers] # Residuals; not flux calibrated! wave = np.outer(np.ones(flux.shape[0]), cframe.wave) # Combine? ''' if combine: res = np.median(res, axis=0) ivar = np.median(ivar, axis=0) flux = np.median(flux, axis=0) wave = np.median(wave, axis=0) ''' # Return if flatten: return wave.flatten(), flux.flatten(), res.flatten(), ivar.flatten() else: return wave, flux, res, ivar
def run_pa(self,input_frame,skymodel): from desispec.sky import subtract_sky subtract_sky(input_frame,skymodel) return input_frame
def main(args) : log=get_logger() cmd = ['desi_compute_fluxcalibration',] for key, value in args.__dict__.items(): if value is not None: cmd += ['--'+key, str(value)] cmd = ' '.join(cmd) log.info(cmd) log.info("read frame") # read frame frame = read_frame(args.infile) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel=read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux,model_wave,model_fibers,model_metadata=read_stdstar_models(args.models) if args.chi2cut > 0 : ok = np.where(model_metadata["CHI2DOF"]<args.chi2cut)[0] if ok.size == 0 : log.error("chi2cut has discarded all stars") sys.exit(12) nstars=model_flux.shape[0] nbad=nstars-ok.size if nbad>0 : log.warning("discarding %d star(s) out of %d because of chi2cut"%(nbad,nstars)) model_flux=model_flux[ok] model_fibers=model_fibers[ok] model_metadata=model_metadata[:][ok] if args.delta_color_cut > 0 : ok = np.where(np.abs(model_metadata["MODEL_G-R"]-model_metadata["DATA_G-R"])<args.delta_color_cut)[0] nstars=model_flux.shape[0] nbad=nstars-ok.size if nbad>0 : log.warning("discarding %d star(s) out of %d because |delta_color|>%f"%(nbad,nstars,args.delta_color_cut)) model_flux=model_flux[ok] model_fibers=model_fibers[ok] model_metadata=model_metadata[:][ok] # automatically reject stars that ar chi2 outliers if args.chi2cut_nsig > 0 : mchi2=np.median(model_metadata["CHI2DOF"]) rmschi2=np.std(model_metadata["CHI2DOF"]) maxchi2=mchi2+args.chi2cut_nsig*rmschi2 ok=np.where(model_metadata["CHI2DOF"]<=maxchi2)[0] nstars=model_flux.shape[0] nbad=nstars-ok.size if nbad>0 : log.warning("discarding %d star(s) out of %d because reduced chi2 outliers (at %d sigma, giving rchi2<%f )"%(nbad,nstars,args.chi2cut_nsig,maxchi2)) model_flux=model_flux[ok] model_fibers=model_fibers[ok] model_metadata=model_metadata[:][ok] # check that the model_fibers are actually standard stars fibermap = frame.fibermap ## check whether star fibers from args.models are consistent with fibers from fibermap ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit fibermap_std_indices = np.where(isStdStar(fibermap['DESI_TARGET']))[0] if np.any(~np.in1d(model_fibers%500, fibermap_std_indices)): for i in model_fibers%500: log.error("inconsistency with spectrum {}, OBJTYPE='{}', DESI_TARGET={} in fibermap".format( (i, fibermap["OBJTYPE"][i], fibermap["DESI_TARGET"][i]))) sys.exit(12) fluxcalib = compute_flux_calibration(frame, model_wave, model_flux, model_fibers%500) # QA if (args.qafile is not None): log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run #import pdb; pdb.set_trace() qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s"%args.outfile)
def main(args): log = get_logger() if (args.fiberflat is None) and (args.sky is None) and (args.calib is None): log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?') sys.exit(12) frame = read_frame(args.infile) #- Raw scores already added in extraction, but just in case they weren't #- it is harmless to rerun to make sure we have them. compute_and_append_frame_scores(frame, suffix="RAW") if args.cosmics_nsig > 0 and args.sky == None: # Reject cosmics (otherwise do it after sky subtraction) log.info("cosmics ray 1D rejection") reject_cosmic_rays_1d(frame, args.cosmics_nsig) if args.fiberflat != None: log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat to all fibers apply_fiberflat(frame, fiberflat) compute_and_append_frame_scores(frame, suffix="FFLAT") if args.sky != None: # read sky skymodel = read_sky(args.sky) if args.cosmics_nsig > 0: # first subtract sky without throughput correction subtract_sky(frame, skymodel, throughput_correction=False) # then find cosmics log.info("cosmics ray 1D rejection after sky subtraction") reject_cosmic_rays_1d(frame, args.cosmics_nsig) if args.sky_throughput_correction: # and (re-)subtract sky, but just the correction term subtract_sky(frame, skymodel, throughput_correction=True, default_throughput_correction=0.) else: # subtract sky subtract_sky(frame, skymodel, throughput_correction=args.sky_throughput_correction) compute_and_append_frame_scores(frame, suffix="SKYSUB") if args.calib != None: log.info("calibrate") # read calibration fluxcalib = read_flux_calibration(args.calib) # apply calibration apply_flux_calibration(frame, fluxcalib) compute_and_append_frame_scores(frame, suffix="CALIB") # save output write_frame(args.outfile, frame, units='1e-17 erg/(s cm2 Angstrom)') log.info("successfully wrote %s" % args.outfile)
def frame_skyres(outfil, frame, skymodel, qaframe, quick_look=False): """ Generate QA plots and files for sky residuals of a given frame Parameters ---------- outfil: str Name of output file frame: Frame object skymodel: SkyModel object qaframe: QAFrame object """ from desispec.sky import subtract_sky # Access metrics ''' wavg_ivar = np.sum(res_ivar,0) chi2_wavg = np.sum(wavg_res**2 * wavg_ivar) dof_wavg = np.sum(wavg_ivar > 0.) pchi2_wavg = scipy.stats.distributions.chi2.sf(chi2_wavg, dof_wavg) chi2_med = np.sum(med_res**2 * wavg_ivar) pchi2_med = scipy.stats.distributions.chi2.sf(chi2_med, dof_wavg) ''' skyfibers = np.array(qaframe.qa_data['SKYSUB']["METRICS"]["SKYFIBERID"]) subtract_sky(frame, skymodel) res=frame.flux[skyfibers] res_ivar=frame.ivar[skyfibers] if quick_look: med_res = qaframe.qa_data['SKYSUB']["METRICS"]["MED_RESID_WAVE"] wavg_res = qaframe.qa_data['SKYSUB']["METRICS"]["WAVG_RES_WAVE"] else: med_res = np.median(res,axis=0) wavg_res = np.sum(res*res_ivar,0) / np.sum(res_ivar,0) # Plot if quick_look: fig = plt.figure(figsize=(8, 10.0)) gs = gridspec.GridSpec(4,2) else: fig = plt.figure(figsize=(8, 6.0)) gs = gridspec.GridSpec(2,2) xmin,xmax = np.min(frame.wave), np.max(frame.wave) # Simple residual plot ax0 = plt.subplot(gs[0,:]) ax0.plot(frame.wave, med_res, label='Median Res') ax0.plot(frame.wave, signal.medfilt(med_res,51), color='black', label='Median**2 Res') ax0.plot(frame.wave, signal.medfilt(wavg_res,51), color='red', label='Med WAvgRes') # ax0.plot([xmin,xmax], [0., 0], '--', color='gray') ax0.plot([xmin,xmax], [0., 0], '--', color='gray') ax0.set_xlabel('Wavelength') ax0.set_ylabel('Sky Residuals (Counts)') ax0.set_xlim(xmin,xmax) ax0.set_xlabel('Wavelength') ax0.set_ylabel('Sky Residuals (Counts)') ax0.set_xlim(xmin,xmax) med0 = np.maximum(np.abs(np.median(med_res)), 1.) ax0.set_ylim(-5.*med0, 5.*med0) #ax0.text(0.5, 0.85, 'Sky Meanspec', # transform=ax_flux.transAxes, ha='center') # Legend legend = ax0.legend(loc='upper right', borderpad=0.3, handletextpad=0.3, fontsize='small') # Histogram of all residuals ax1 = plt.subplot(gs[1,0]) xmin,xmax = -5., 5. # Histogram binsz = qaframe.qa_data['SKYSUB']["PARAMS"]["BIN_SZ"] if 'DEVS_1D' in qaframe.qa_data['SKYSUB']["METRICS"].keys(): # Online hist = np.asarray(qaframe.qa_data['SKYSUB']["METRICS"]["DEVS_1D"]) edges = np.asarray(qaframe.qa_data['SKYSUB']["METRICS"]["DEVS_EDGES"]) else: # Generate for offline gd_res = res_ivar > 0. devs = res[gd_res] * np.sqrt(res_ivar[gd_res]) i0, i1 = int( np.min(devs) / binsz) - 1, int( np.max(devs) / binsz) + 1 rng = tuple( binsz*np.array([i0,i1]) ) nbin = i1-i0 hist, edges = np.histogram(devs, range=rng, bins=nbin) xhist = (edges[1:] + edges[:-1])/2. ax1.hist(xhist, color='blue', bins=edges, weights=hist)#, histtype='step') # PDF for Gaussian area = binsz * np.sum(hist) xppf = np.linspace(scipy.stats.norm.ppf(0.0001), scipy.stats.norm.ppf(0.9999), 100) ax1.plot(xppf, area*scipy.stats.norm.pdf(xppf), 'r-', alpha=1.0) ax1.set_xlabel(r'Res/$\sigma$') ax1.set_ylabel('N') ax1.set_xlim(xmin,xmax) # Meta text #- limit the dictionary to residuals only for meta qaresid=copy.deepcopy(qaframe) resid_keys=['NREJ','NSKY_FIB','NBAD_PCHI','MED_RESID','RESID_PER'] qaresid.qa_data['SKYSUB']['METRICS']={key:value for key,value in qaframe.qa_data['SKYSUB'] ['METRICS'].items() if key in resid_keys} ax2 = plt.subplot(gs[1,1]) ax2.set_axis_off() show_meta(ax2, qaresid, 'SKYSUB', outfil) if quick_look: #- SNR Plot elg_snr_mag = qaframe.qa_data['SKYSUB']["METRICS"]["ELG_SNR_MAG"] lrg_snr_mag = qaframe.qa_data['SKYSUB']["METRICS"]["LRG_SNR_MAG"] qso_snr_mag = qaframe.qa_data['SKYSUB']["METRICS"]["QSO_SNR_MAG"] star_snr_mag = qaframe.qa_data['SKYSUB']["METRICS"]["STAR_SNR_MAG"] ax3 = plt.subplot(gs[2,0]) ax4 = plt.subplot(gs[2,1]) ax5 = plt.subplot(gs[3,0]) ax6 = plt.subplot(gs[3,1]) ax3.set_ylabel(r'Median S/N') ax3.set_xlabel('') ax3.set_title(r'ELG') if len(elg_snr_mag[1]) > 0: #- at least 1 elg fiber? select=np.where((elg_snr_mag[1] != np.array(None)) & (~np.isnan(elg_snr_mag[1])) & (np.abs(elg_snr_mag[1])!=np.inf))[0] #- Remove None, nan and inf values in mag if select.shape[0]>0: xmin=np.min(elg_snr_mag[1][select])-0.1 xmax=np.max(elg_snr_mag[1][select])+0.1 ax3.set_xlim(xmin,xmax) ax3.set_ylim(np.min(elg_snr_mag[0][select])-0.1,np.max(elg_snr_mag[0][select])+0.1) ax3.xaxis.set_ticks(np.arange(int(np.min(elg_snr_mag[1][select])),int(np.max(elg_snr_mag[1][select]))+1,0.5)) ax3.tick_params(axis='x',labelsize=10,labelbottom='on') ax3.tick_params(axis='y',labelsize=10,labelleft='on') ax3.plot(elg_snr_mag[1][select],elg_snr_mag[0][select],'b.') ax4.set_ylabel('') ax4.set_xlabel('') ax4.set_title(r'LRG') if len(lrg_snr_mag[1]) > 0: #- at least 1 lrg fiber? select=np.where((lrg_snr_mag[1] != np.array(None)) & (~np.isnan(lrg_snr_mag[1])) & (np.abs(lrg_snr_mag[1])!=np.inf))[0] if select.shape[0]>0: xmin=np.min(lrg_snr_mag[1][select])-0.1 xmax=np.max(lrg_snr_mag[1][select])+0.1 ax4.set_xlim(xmin,xmax) ax4.set_ylim(np.min(lrg_snr_mag[0][select])-0.1,np.max(lrg_snr_mag[0][select])+0.1) ax4.xaxis.set_ticks(np.arange(int(np.min(lrg_snr_mag[1][select])),int(np.max(lrg_snr_mag[1][select]))+1,0.5)) ax4.tick_params(axis='x',labelsize=10,labelbottom='on') ax4.tick_params(axis='y',labelsize=10,labelleft='on') ax4.plot(lrg_snr_mag[1][select],lrg_snr_mag[0][select],'r.') ax5.set_ylabel(r'Median S/N') ax5.set_xlabel(r'Mag. (DECAM_R)') ax5.set_title(r'QSO') if len(qso_snr_mag[1]) > 0: #- at least 1 qso fiber? select=np.where((qso_snr_mag[1] != np.array(None)) & (~np.isnan(qso_snr_mag[1])) & (np.abs(qso_snr_mag[1])!=np.inf))[0] #- Remove None, nan and inf values if select.shape[0]>0: xmin=np.min(qso_snr_mag[1][select])-0.1 xmax=np.max(qso_snr_mag[1][select])+0.1 ax5.set_xlim(xmin,xmax) ax5.set_ylim(np.min(qso_snr_mag[0][select])-0.1,np.max(qso_snr_mag[0][select])+0.1) ax5.xaxis.set_ticks(np.arange(int(np.min(qso_snr_mag[1][select])),int(np.max(qso_snr_mag[1][select]))+1,1.0)) ax5.tick_params(axis='x',labelsize=10,labelbottom='on') ax5.tick_params(axis='y',labelsize=10,labelleft='on') ax5.plot(qso_snr_mag[1][select],qso_snr_mag[0][select],'g.') ax6.set_ylabel('') ax6.set_xlabel('Mag. (DECAM_R)') ax6.set_title(r'STD') if len(star_snr_mag[1]) > 0: #- at least 1 std fiber? select=np.where((star_snr_mag[1] != np.array(None)) & (~np.isnan(star_snr_mag[1])) & (np.abs(star_snr_mag[1])!=np.inf))[0] if select.shape[0]>0: xmin=np.min(star_snr_mag[1][select])-0.1 xmax=np.max(star_snr_mag[1][select])+0.1 ax6.set_xlim(xmin,xmax) ax6.set_ylim(np.min(star_snr_mag[0][select])-0.1,np.max(star_snr_mag[0][select])+0.1) ax6.xaxis.set_ticks(np.arange(int(np.min(star_snr_mag[1][select])),int(np.max(star_snr_mag[1][select]))+1,0.5)) ax6.tick_params(axis='x',labelsize=10,labelbottom='on') ax6.tick_params(axis='y',labelsize=10,labelleft='on') ax6.plot(star_snr_mag[1][select],star_snr_mag[0][select],'k.') """ # Meta xlbl = 0.1 ylbl = 0.85 i0 = outfil.rfind('/') ax2.text(xlbl, ylbl, outfil[i0+1:], color='black', transform=ax2.transAxes, ha='left') yoff=0.15 for key in sorted(qaframe.data['SKYSUB']['METRICS'].keys()): if key in ['QA_FIG']: continue # Show ylbl -= yoff ax2.text(xlbl+0.1, ylbl, key+': '+str(qaframe.data['SKYSUB']['METRICS'][key]), transform=ax2.transAxes, ha='left', fontsize='small') """ ''' # Residuals scatt_sz = 0.5 ax_res = plt.subplot(gs[1]) ax_res.get_xaxis().set_ticks([]) # Suppress labeling res = (sky_model - (true_flux*scl))/(true_flux*scl) rms = np.sqrt(np.sum(res**2)/len(res)) #ax_res.set_ylim(-3.*rms, 3.*rms) ax_res.set_ylim(-2, 2) ax_res.set_ylabel('Frac Res') # Error #ax_res.plot(true_wave, 2.*ms_sig/sky_model, color='red') ax_res.scatter(wave,res, marker='o',s=scatt_sz) ax_res.plot([xmin,xmax], [0.,0], 'g-') ax_res.set_xlim(xmin,xmax) # Relative to error ax_sig = plt.subplot(gs[2]) ax_sig.set_xlabel('Wavelength') sig_res = (sky_model - (true_flux*scl))/sky_sig ax_sig.scatter(wave, sig_res, marker='o',s=scatt_sz) ax_sig.set_ylabel(r'Res $\delta/\sigma$') ax_sig.set_ylim(-5., 5.) ax_sig.plot([xmin,xmax], [0.,0], 'g-') ax_sig.set_xlim(xmin,xmax) ''' # Finish plt.tight_layout(pad=0.1,h_pad=0.0,w_pad=0.0) outfile = makepath(outfil) plt.savefig(outfil) plt.close() print('Wrote QA SkyRes file: {:s}'.format(outfil))
def run_pa(self,input_frame,skymodel): from desispec.quicklook.quicksky import subtract_sky sframe=subtract_sky(input_frame,skymodel) return sframe
def main(args): log = get_logger() cmd = [ 'desi_compute_fluxcalibration', ] for key, value in args.__dict__.items(): if value is not None: cmd += ['--' + key, str(value)] cmd = ' '.join(cmd) log.info(cmd) log.info("read frame") # read frame frame = read_frame(args.infile) # Set fibermask flagged spectra to have 0 flux and variance frame = get_fiberbitmasked_frame(frame, bitmask='flux', ivar_framemask=True) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel = read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models( args.models) ok = np.ones(len(model_metadata), dtype=bool) if args.chi2cut > 0: log.info("Apply cut CHI2DOF<{}".format(args.chi2cut)) ok &= (model_metadata["CHI2DOF"] < args.chi2cut) if args.delta_color_cut > 0: log.info("Apply cut |delta color|<{}".format(args.delta_color_cut)) ok &= (np.abs(model_metadata["MODEL_G-R"] - model_metadata["DATA_G-R"]) < args.delta_color_cut) if args.min_color is not None: log.info("Apply cut DATA_G-R>{}".format(args.min_color)) ok &= (model_metadata["DATA_G-R"] > args.min_color) if args.chi2cut_nsig > 0: # automatically reject stars that ar chi2 outliers mchi2 = np.median(model_metadata["CHI2DOF"]) rmschi2 = np.std(model_metadata["CHI2DOF"]) maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2 log.info("Apply cut CHI2DOF<{} based on chi2cut_nsig={}".format( maxchi2, args.chi2cut_nsig)) ok &= (model_metadata["CHI2DOF"] <= maxchi2) ok = np.where(ok)[0] if ok.size == 0: log.error("cuts discarded all stars") sys.exit(12) nstars = model_flux.shape[0] nbad = nstars - ok.size if nbad > 0: log.warning("discarding %d star(s) out of %d because of cuts" % (nbad, nstars)) model_flux = model_flux[ok] model_fibers = model_fibers[ok] model_metadata = model_metadata[:][ok] # check that the model_fibers are actually standard stars fibermap = frame.fibermap ## check whether star fibers from args.models are consistent with fibers from fibermap ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit fibermap_std_indices = np.where(isStdStar(fibermap))[0] if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)): target_colnames, target_masks, survey = main_cmx_or_sv(fibermap) colname = target_colnames[0] for i in model_fibers % 500: log.error( "inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap" .format(i, fibermap["OBJTYPE"][i], colname, fibermap[colname][i])) sys.exit(12) # Make sure the fibers of interest aren't entirely masked. if np.sum( np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) == frame.nwave) == len(model_fibers): log.warning('All standard-star spectra are masked!') return fluxcalib = compute_flux_calibration( frame, model_wave, model_flux, model_fibers % 500, highest_throughput_nstars=args.highest_throughput) # QA if (args.qafile is not None): log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame_meta=frame.meta, flavor=frame.meta['FLAVOR']) # Run #import pdb; pdb.set_trace() qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s" % args.outfile)
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--infile', type=str, default=None, required=True, help='path of DESI exposure frame fits file') parser.add_argument('--fiberflat', type=str, default=None, help='path of DESI fiberflat fits file') parser.add_argument('--sky', type=str, default=None, help='path of DESI sky fits file') parser.add_argument('--calib', type=str, default=None, help='path of DESI calibration fits file') parser.add_argument('--outfile', type=str, default=None, required=True, help='path of DESI sky fits file') # add calibration here when exists args = parser.parse_args() log = get_logger() if (args.fiberflat is None) and (args.sky is None) and (args.calib is None): log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?') sys.exit(12) frame = read_frame(args.infile) if args.fiberflat != None: log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat to sky fibers apply_fiberflat(frame, fiberflat) if args.sky != None: log.info("subtract sky") # read sky skymodel = read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) if args.calib != None: log.info("calibrate") # read calibration fluxcalib = read_flux_calibration(args.calib) # apply calibration apply_flux_calibration(frame, fluxcalib) # save output write_frame(args.outfile, frame) log.info("successfully wrote %s" % args.outfile)
def main(args): log = get_logger() if (args.fiberflat is None) and (args.sky is None) and (args.calib is None): log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?') sys.exit(12) if (not args.no_tsnr) and (args.calib is None): log.critical( 'need --fiberflat --sky and --calib to compute template SNR') sys.exit(12) frame = read_frame(args.infile) if not args.no_tsnr: # tsnr alpha calc. requires uncalibrated + no substraction rame. uncalibrated_frame = copy.deepcopy(frame) #- Raw scores already added in extraction, but just in case they weren't #- it is harmless to rerun to make sure we have them. compute_and_append_frame_scores(frame, suffix="RAW") if args.cosmics_nsig > 0 and args.sky == None: # Reject cosmics (otherwise do it after sky subtraction) log.info("cosmics ray 1D rejection") reject_cosmic_rays_1d(frame, args.cosmics_nsig) if args.fiberflat != None: log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat to all fibers apply_fiberflat(frame, fiberflat) compute_and_append_frame_scores(frame, suffix="FFLAT") else: fiberflat = None if args.no_xtalk: zero_ivar = (not args.no_zero_ivar) else: zero_ivar = False if args.sky != None: # read sky skymodel = read_sky(args.sky) if args.cosmics_nsig > 0: # use a copy the frame (not elegant but robust) copied_frame = copy.deepcopy(frame) # first subtract sky without throughput correction subtract_sky(copied_frame, skymodel, apply_throughput_correction=False, zero_ivar=zero_ivar) # then find cosmics log.info("cosmics ray 1D rejection after sky subtraction") reject_cosmic_rays_1d(copied_frame, args.cosmics_nsig) # copy mask frame.mask = copied_frame.mask # and (re-)subtract sky, but just the correction term subtract_sky(frame, skymodel, apply_throughput_correction=( not args.no_sky_throughput_correction), zero_ivar=zero_ivar) else: # subtract sky subtract_sky(frame, skymodel, apply_throughput_correction=( not args.no_sky_throughput_correction), zero_ivar=zero_ivar) compute_and_append_frame_scores(frame, suffix="SKYSUB") if not args.no_xtalk: log.info("fiber crosstalk correction") correct_fiber_crosstalk(frame, fiberflat) if not args.no_zero_ivar: frame.ivar *= (frame.mask == 0) if args.calib != None: log.info("calibrate") # read calibration fluxcalib = read_flux_calibration(args.calib) # apply calibration apply_flux_calibration(frame, fluxcalib) # Ensure that ivars are set to 0 for all values if any designated # fibermask bit is set. Also flips a bits for each frame.mask value using specmask.BADFIBER frame = get_fiberbitmasked_frame( frame, bitmask="flux", ivar_framemask=(not args.no_zero_ivar)) compute_and_append_frame_scores(frame, suffix="CALIB") if not args.no_tsnr: log.info("calculating tsnr") results, alpha = calc_tsnr2(uncalibrated_frame, fiberflat=fiberflat, skymodel=skymodel, fluxcalib=fluxcalib, alpha_only=args.alpha_only) frame.meta['TSNRALPH'] = alpha comments = {k: "from calc_frame_tsnr" for k in results.keys()} append_frame_scores(frame, results, comments, overwrite=True) # record inputs frame.meta['IN_FRAME'] = shorten_filename(args.infile) frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat) frame.meta['IN_SKY'] = shorten_filename(args.sky) frame.meta['IN_CALIB'] = shorten_filename(args.calib) # save output write_frame(args.outfile, frame, units='10**-17 erg/(s cm2 Angstrom)') log.info("successfully wrote %s" % args.outfile)