def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '--infile', type=str, default=None, required=True, help= 'path of DESI frame fits file corresponding to a continuum lamp exposure' ) parser.add_argument('--outfile', type=str, default=None, required=True, help='path of DESI fiberflat fits file') args = parser.parse_args() log = get_logger() log.info("starting") frame = read_frame(args.infile) fiberflat = compute_fiberflat(frame) write_fiberflat(args.outfile, fiberflat, frame.header) log.info("successfully wrote %s" % args.outfile)
def main(args): log = get_logger() log.info("starting at {}".format(time.asctime())) # Process frame = read_frame(args.infile) fiberflat = compute_fiberflat(frame, nsig_clipping=args.nsig, accuracy=args.acc, smoothing_res=args.smoothing_resolution) # QA if (args.qafile is not None): log.info("performing fiberflat QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run qaframe.run_qa('FIBERFLAT', (frame, fiberflat)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fiberflat(args.qafig, qaframe, frame, fiberflat) # Write write_fiberflat(args.outfile, fiberflat, frame.meta) log.info("successfully wrote %s" % args.outfile) log.info("done at {}".format(time.asctime()))
def main(args) : log=get_logger() log.info("starting") # Process frame = read_frame(args.infile) fiberflat = compute_fiberflat(frame) # QA if (args.qafile is not None): log.info("performing fiberflat QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run qaframe.run_qa('FIBERFLAT', (frame, fiberflat)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fiberflat(args.qafig, qaframe, frame, fiberflat) # Write write_fiberflat(args.outfile, fiberflat, frame.meta) log.info("successfully wrote %s"%args.outfile)
def get_skyres(cframes, sub_sky=False): from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.sky import subtract_sky if isinstance(cframes, list): all_wave, all_flux, all_res, all_ivar = [], [], [], [] for cframe_file in cframes: wave, flux, res, ivar = get_skyres(cframe_file) # Concatenate and return all_wave.append(wave) all_flux.append(flux) all_res.append(res) all_ivar.append(ivar) return np.concatenate(all_wave), np.concatenate(all_flux), \ np.concatenate(all_res), np.concatenate(all_ivar) cframe = read_frame(cframes) if cframe.meta['FLAVOR'] in ['flat', 'arc']: raise ValueError("Bad flavor for exposure: {:s}".format(cframes)) # Sky sky_file = cframes.replace('cframe', 'sky') skymodel = read_sky(sky_file) if sub_sky: subtract_sky(cframe, skymodel) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] ivar = cframe.ivar[skyfibers] flux = skymodel.flux[skyfibers] # Residuals wave = np.outer(np.ones(flux.shape[0]), cframe.wave) # Return return wave.flatten(), flux.flatten(), res.flatten(), ivar.flatten()
def main(args): log = get_logger() log.info("starting") # Process frame = read_frame(args.infile) fiberflat = compute_fiberflat(frame) # QA if (args.qafile is not None): log.info("performing fiberflat QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run qaframe.run_qa('FIBERFLAT', (frame, fiberflat)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fiberflat(args.qafig, qaframe, frame, fiberflat) # Write write_fiberflat(args.outfile, fiberflat, frame.meta) log.info("successfully wrote %s" % args.outfile)
def main(args): log = get_logger() for filename in args.infile: log.info("reading %s" % filename) frame = io.read_frame(filename) flux_per_angstrom = None if args.flux_per_angstrom: flux_per_angstrom = True elif args.flux_per_pixel: flux_per_angstrom = False else: flux_per_angstrom = None scores, comments = compute_and_append_frame_scores( frame, suffix=args.suffix, flux_per_angstrom=flux_per_angstrom, overwrite=args.overwrite) log.info("Adding or replacing SCORES extention with {} in {}".format( scores.keys(), filename)) write_bintable(filename, data=scores, comments=comments, extname="SCORES", clobber=True)
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--infile', type=str, default=None, required=True, help='path of DESI exposure frame fits file') parser.add_argument('--fibermap', type=str, default=None, required=True, help='path of DESI exposure frame fits file') parser.add_argument('--fiberflat', type=str, default=None, required=True, help='path of DESI fiberflat fits file') parser.add_argument('--outfile', type=str, default=None, required=True, help='path of DESI sky fits file') args = parser.parse_args() log = get_logger() log.info("starting") # read exposure to load data and get range of spectra frame = read_frame(args.infile) specmin = frame.header["SPECMIN"] specmax = frame.header["SPECMAX"] # read fibermap to locate sky fibers fibermap = read_fibermap(args.fibermap) selection = np.where((fibermap["OBJTYPE"] == "SKY") & (fibermap["FIBER"] >= specmin) & (fibermap["FIBER"] <= specmax))[0] if selection.size == 0: log.error("no sky fiber in fibermap %s" % args.fibermap) sys.exit(12) # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat to sky fibers apply_fiberflat(frame, fiberflat) # compute sky model skymodel = compute_sky(frame, fibermap) # write result write_sky(args.outfile, skymodel, frame.header) log.info("successfully wrote %s" % args.outfile)
def load_s2n_values(objtype, nights, channel, sub_exposures=None): fdict = dict(waves=[], s2n=[], fluxes=[], exptime=[], OII=[]) for night in nights: if sub_exposures is not None: exposures = sub_exposures else: exposures = get_exposures(night) #, raw=True) for exposure in exposures: fibermap_path = findfile(filetype='fibermap', night=night, expid=exposure) fibermap_data = read_fibermap(fibermap_path) flavor = fibermap_data.meta['FLAVOR'] if flavor.lower() in ('arc', 'flat', 'bias'): log.debug('Skipping calibration {} exposure {:08d}'.format( flavor, exposure)) continue # Load simspec simspec_file = fibermap_path.replace('fibermap', 'simspec') sps_hdu = fits.open(simspec_file) sps_tab = Table(sps_hdu['TRUTH'].data, masked=True) sps_hdu.close() objs = sps_tab['TEMPLATETYPE'] == objtype if np.sum(objs) == 0: continue # Load spectra (flux or not fluxed; should not matter) for ii in range(10): camera = channel + str(ii) cframe_path = findfile(filetype='cframe', night=night, expid=exposure, camera=camera) try: cframe = read_frame(cframe_path) except: log.warn("Cannot find file: {:s}".format(cframe_path)) continue # Calculate S/N per Ang dwave = cframe.wave - np.roll(cframe.wave, 1) dwave[0] = dwave[1] # iobjs = objs[cframe.fibers] if np.sum(iobjs) == 0: continue s2n = cframe.flux[iobjs, :] * np.sqrt( cframe.ivar[iobjs, :]) / np.sqrt(dwave) # Save fdict['waves'].append(cframe.wave) fdict['s2n'].append(s2n) fdict['fluxes'].append(sps_tab['MAG'][cframe.fibers[iobjs]]) if objtype == 'ELG': fdict['OII'].append( sps_tab['OIIFLUX'][cframe.fibers[iobjs]]) fdict['exptime'].append(cframe.meta['EXPTIME']) # Return return fdict
def main(args) : log=get_logger() log.info("read frame") # read frame frame = read_frame(args.infile) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel=read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux,model_wave,model_fibers=read_stdstar_models(args.models) # check that the model_fibers are actually standard stars fibermap = frame.fibermap model_fibers = model_fibers%500 if np.any(fibermap['OBJTYPE'][model_fibers] != 'STD'): for i in model_fibers: log.error("inconsistency with spectrum %d, OBJTYPE='%s' in fibermap"%(i,fibermap["OBJTYPE"][i])) sys.exit(12) fluxcalib = compute_flux_calibration(frame, model_wave, model_flux) # QA if (args.qafile is not None): log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run #import pdb; pdb.set_trace() qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s"%args.outfile)
def main() : parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--infile', type = str, default = None, required=True, help = 'path of DESI exposure frame fits file') parser.add_argument('--fiberflat', type = str, default = None, help = 'path of DESI fiberflat fits file') parser.add_argument('--sky', type = str, default = None, help = 'path of DESI sky fits file') parser.add_argument('--calib', type = str, default = None, help = 'path of DESI calibration fits file') parser.add_argument('--outfile', type = str, default = None, required=True, help = 'path of DESI sky fits file') # add calibration here when exists args = parser.parse_args() log = get_logger() if (args.fiberflat is None) and (args.sky is None) and (args.calib is None): log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?') sys.exit(12) frame = read_frame(args.infile) if args.fiberflat!=None : log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat to sky fibers apply_fiberflat(frame, fiberflat) if args.sky!=None : log.info("subtract sky") # read sky skymodel=read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) if args.calib!=None : log.info("calibrate") # read calibration fluxcalib=read_flux_calibration(args.calib) # apply calibration apply_flux_calibration(frame, fluxcalib) # save output write_frame(args.outfile, frame) log.info("successfully wrote %s"%args.outfile)
def main(args): log = get_logger() frame=read_frame(args.infile, skip_resolution=True) fibermap=read_fibermap(args.infile) fiberflat=read_fiberflat(args.fiberflat) skymodel=read_sky(args.sky) fluxcalib=read_flux_calibration(args.calib) cam=args.infile.split('/')[-1].split('-')[1] band=cam[0] bands=[band] # Indices of sky fibers. sky_indx = np.where(fibermap['OBJTYPE'] == 'SKY')[0] rd_var, sky_var = calc_var(bands, args.nea, args.psf, frame, fluxcalib, fiberflat, skymodel, components=True) var = calc_var(bands, args.nea, args.psf, frame, fluxcalib, fiberflat, skymodel, components=False) nsky = 4 fig, axes = plt.subplots(1, nsky, figsize=(5 * nsky, 5)) for i in range(nsky): def calc_alphavar(alpha): return alpha * rd_var[sky_indx,:] + sky_var[sky_indx,:] def alpha_fit(alpha): _var = calc_alphavar(alpha) ivar = 1. / _var X2 = (frame.ivar[sky_indx,:] - ivar)**2. return np.sum(X2) res = minimize(alpha_fit, x0=[1.]) alpha = res.x[0] indx = sky_indx[i] axes[i].plot(skymodel.wave, median_filter(frame.ivar[indx,:], 10), lw=0.4, label='Sky frame IVAR', alpha=0.4) axes[i].plot(skymodel.wave, 1./rd_var[indx,:], lw=0.4, label='Model rd. IVAR', alpha=0.4) # axes[i].plot(skymodel.wave, 1./sky_var[indx,:], lw=0.4, label='Model Sky IVAR', alpha=0.4) # axes[i].plot(skymodel.wave, 1./var[indx,:], lw=0.4, label=r'Model IVAR', alpha=0.4) axes[i].plot(skymodel.wave, median_filter(1./calc_alphavar(alpha)[i,:], 10), lw=0.4, label=r'$\alpha$ Model IVAR', alpha=0.4) axes[i].set_title(r'Fiber {:d} ($\alpha$ = {:.6f})'.format(indx, alpha)) axes[i].set_xlabel(r'Wavelength [$AA$]') axes[i].set_yscale('log') axes[i].set_ylim(bottom=5.e-4, top=3.e-2) axes[i].legend(frameon=False, loc=2) axes[0].set_ylabel('e/A') pl.show()
def slurp(self, make_frameqa=False, remove=True, **kwargs): """ Slurp all the individual QA files into one master QA file Args: make_frameqa: bool, optional Regenerate the individual QA files (at the frame level first) remove: bool, optional Remove Returns: """ from desispec.io import meta from desispec.qa import QA_Exposure from desispec.io import write_qa_prod import pdb log = get_logger() # Remake? if make_frameqa: self.make_frameqa(**kwargs) # Loop on nights path_nights = glob.glob(self.specprod_dir + '/exposures/*') nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights] # Reset log.info("Resetting qa_exps in qa_prod") self.qa_exps = [] # Loop for night in nights: # Loop on exposures for exposure in get_exposures(night, specprod_dir=self.specprod_dir): frames_dict = get_files(filetype=str('frame'), night=night, expid=exposure, specprod_dir=self.specprod_dir) if len(frames_dict) == 0: continue # Load any frame (for the type) key = list(frames_dict.keys())[0] frame_fil = frames_dict[key] frame = read_frame(frame_fil) qa_exp = QA_Exposure(exposure, night, frame.meta['FLAVOR'], specprod_dir=self.specprod_dir, remove=remove) # Append self.qa_exps.append(qa_exp) # Write outroot = self.specprod_dir + '/' + self.prod_name + '_qa' write_qa_prod(outroot, self)
def main(args) : log=get_logger() log.info("starting") # read exposure to load data and get range of spectra frame = read_frame(args.infile) specmin, specmax = np.min(frame.fibers), np.max(frame.fibers) if args.cosmics_nsig>0 : # Reject cosmics reject_cosmic_rays_1d(frame,args.cosmics_nsig) # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat to sky fibers apply_fiberflat(frame, fiberflat) # compute sky model skymodel = compute_sky(frame,add_variance=(not args.no_extra_variance),\ angular_variation_deg=args.angular_variation_deg,\ chromatic_variation_deg=args.chromatic_variation_deg,\ adjust_wavelength=args.adjust_wavelength,\ adjust_lsf=args.adjust_lsf) # QA if (args.qafile is not None) or (args.qafig is not None): log.info("performing skysub QA") # Load qaframe = load_qa_frame(args.qafile, frame_meta=frame.meta, flavor=frame.meta['FLAVOR']) # Run qaframe.run_qa('SKYSUB', (frame, skymodel)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_skyres(args.qafig, frame, skymodel, qaframe) # record inputs frame.meta['IN_FRAME'] = shorten_filename(args.infile) frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat) # write result write_sky(args.outfile, skymodel, frame.meta) log.info("successfully wrote %s"%args.outfile)
def main() : parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--infile', type = str, default = None, required=True, help = 'path of DESI exposure frame fits file') parser.add_argument('--fibermap', type = str, default = None, required=True, help = 'path of DESI exposure frame fits file') parser.add_argument('--fiberflat', type = str, default = None, required=True, help = 'path of DESI fiberflat fits file') parser.add_argument('--outfile', type = str, default = None, required=True, help = 'path of DESI sky fits file') args = parser.parse_args() log=get_logger() log.info("starting") # read exposure to load data and get range of spectra frame = read_frame(args.infile) specmin=frame.header["SPECMIN"] specmax=frame.header["SPECMAX"] # read fibermap to locate sky fibers fibermap = read_fibermap(args.fibermap) selection=np.where((fibermap["OBJTYPE"]=="SKY")&(fibermap["FIBER"]>=specmin)&(fibermap["FIBER"]<=specmax))[0] if selection.size == 0 : log.error("no sky fiber in fibermap %s"%args.fibermap) sys.exit(12) # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat to sky fibers apply_fiberflat(frame, fiberflat) # compute sky model skymodel = compute_sky(frame, fibermap) # write result write_sky(args.outfile, skymodel, frame.header) log.info("successfully wrote %s"%args.outfile)
def slurp(self, make_frameqa=False, remove=True, **kwargs): """ Slurp all the individual QA files into one master QA file Args: make_frameqa: bool, optional Regenerate the individual QA files (at the frame level first) remove: bool, optional Remove Returns: """ from desispec.io import meta from desispec.qa import QA_Exposure from desispec.io import write_qa_prod import pdb # Remake? if make_frameqa: self.make_frameqa(**kwargs) # Loop on nights path_nights = glob.glob(self.specprod_dir+'/exposures/*') nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights] # Reset log.info("Resetting qa_exps in qa_prod") self.qa_exps = [] # Loop for night in nights: # Loop on exposures for exposure in get_exposures(night, specprod_dir = self.specprod_dir): frames_dict = get_files(filetype = str('frame'), night = night, expid = exposure, specprod_dir = self.specprod_dir) if len(frames_dict.keys()) == 0: continue # Load any frame (for the type) key = frames_dict.keys()[0] frame_fil = frames_dict[key] frame = read_frame(frame_fil) qa_exp = QA_Exposure(exposure, night, frame.meta['FLAVOR'], specprod_dir=self.specprod_dir, remove=remove) # Append self.qa_exps.append(qa_exp) # Write outroot = self.specprod_dir+'/'+self.prod_name+'_qa' write_qa_prod(outroot, self)
def main(args) : log = get_logger() for filename in args.infile : log.info("reading %s"%filename) frame=io.read_frame(filename) flux_per_angstrom=None if args.flux_per_angstrom : flux_per_angstrom=True elif args.flux_per_pixel : flux_per_angstrom=False else : flux_per_angstrom=None scores,comments=compute_and_append_frame_scores(frame,suffix=args.suffix,flux_per_angstrom=flux_per_angstrom,overwrite=args.overwrite) log.info("Adding or replacing SCORES extention with {} in {}".format(scores.keys(),filename)) write_bintable(filename,data=scores,comments=comments,extname="SCORES",clobber=True)
def main() : parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--infile', type = str, default = None, required=True, help = 'path of DESI frame fits file corresponding to a continuum lamp exposure') parser.add_argument('--outfile', type = str, default = None, required=True, help = 'path of DESI fiberflat fits file') args = parser.parse_args() log=get_logger() log.info("starting") frame = read_frame(args.infile) fiberflat = compute_fiberflat(frame) write_fiberflat(args.outfile, fiberflat, frame.header) log.info("successfully wrote %s"%args.outfile)
def main(args): log = get_logger() log.info("starting") # read exposure to load data and get range of spectra frame = read_frame(args.infile) specmin, specmax = np.min(frame.fibers), np.max(frame.fibers) # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat to sky fibers apply_fiberflat(frame, fiberflat) # compute sky model skymodel = compute_sky(frame) # QA if (args.qafile is not None) or (args.qafig is not None): log.info("performing skysub QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run qaframe.run_qa('SKYSUB', (frame, skymodel)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_skyres(args.qafig, frame, skymodel, qaframe) # write result write_sky(args.outfile, skymodel, frame.meta) log.info("successfully wrote %s" % args.outfile)
def main(args) : log=get_logger() log.info("starting") # read exposure to load data and get range of spectra frame = read_frame(args.infile) specmin, specmax = np.min(frame.fibers), np.max(frame.fibers) # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat to sky fibers apply_fiberflat(frame, fiberflat) # compute sky model skymodel = compute_sky(frame) # QA if (args.qafile is not None) or (args.qafig is not None): log.info("performing skysub QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run qaframe.run_qa('SKYSUB', (frame, skymodel)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_skyres(args.qafig, frame, skymodel, qaframe) # write result write_sky(args.outfile, skymodel, frame.meta) log.info("successfully wrote %s"%args.outfile)
def main(args): log = get_logger() if (args.fiberflat is None) and (args.sky is None) and (args.calib is None): log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?') sys.exit(12) frame = read_frame(args.infile) if args.fiberflat!=None : log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat to sky fibers apply_fiberflat(frame, fiberflat) if args.sky!=None : log.info("subtract sky") # read sky skymodel=read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) if args.calib!=None : log.info("calibrate") # read calibration fluxcalib=read_flux_calibration(args.calib) # apply calibration apply_flux_calibration(frame, fluxcalib) # save output write_frame(args.outfile, frame, units='1e-17 erg/(s cm2 A)') log.info("successfully wrote %s"%args.outfile)
def get_sky(night, expid, exptime, ftype="model", redux="daily", smoothing=100.0, specsim_darksky=False, nightly_darksky=False): # AR ftype = "data" or "model" # AR redux = "daily" or "blanc" # AR if ftype = "data" : read the sky fibers from frame*fits + apply flat-field # AR if ftype = "model": read the sky model from sky*.fits for the first fiber of each petal (see DJS email from 29Dec2020) # AR those are in electron / angstrom # AR to integrate over the decam-r-band, we need cameras b and r if ftype not in ["data", "model"]: sys.exit("ftype should be 'data' or 'model'") sky = np.zeros(len(fullwave)) reduxdir = dailydir.replace("daily", redux) # AR see AK email [desi-data 5218] if redux == "blanc": specs = ["0", "1", "3", "4", "5", "7", "8", "9"] else: specs = np.arange(10, dtype=int).astype(str) for camera in ["b", "r", "z"]: norm_cam = np.zeros(len(fullwave[cslice[camera]])) sky_cam = np.zeros(len(fullwave[cslice[camera]])) for spec in specs: # AR data if ftype == "data": frfn = os.path.join( reduxdir, "exposures", "{}".format(night), expid, "frame-{}{}-{}.fits".format(camera, spec, expid), ) if not os.path.isfile(frfn): print("Skipping non-existent {}".format(frfn)) continue fr = read_frame(frfn, skip_resolution=True) flfn = os.environ['DESI_SPECTRO_CALIB'] + '/' + CalibFinder( [fr.meta]).data['FIBERFLAT'] # calib_flux [1e-17 erg/s/cm2/Angstrom] = uncalib_flux [electron/Angstrom] / (calibration_model * exptime [s]) # fl = read_fiberflat(flfn) # No exptime correction. # apply_fiberflat(fr, fl) # AR cutting on sky fibers with at least one valid pixel. ii = (fr.fibermap["OBJTYPE"] == "SKY") & (fr.ivar.sum(axis=1) > 0) # AR frame*fits are in e- / angstrom ; adding the N sky fibers # sky_cam += fr.flux[ii, :].sum(axis=0) # nspec += ii.sum() # Ignores fiberflat corrected (fl), as includes e.g. fiberloss. sky_cam += (fr.flux[ii, :] * fr.ivar[ii, :]).sum(axis=0) norm_cam += fr.ivar[ii, :].sum(axis=0) # AR model if ftype == "model": fn = os.path.join( reduxdir, "exposures", "{}".format(night), expid, "sky-{}{}-{}.fits".format(camera, spec, expid), ) if not os.path.isfile(fn): print("Skipping non-existent {}".format(fn)) else: print("Solving for {}".format(fn)) fd = fitsio.FITS(fn) assert np.allclose(fullwave[cslice[camera]], fd["WAVELENGTH"].read()) fd = fitsio.FITS(fn) with fd as hdus: exptime = hdus[0].read_header()['EXPTIME'] flux = hdus['SKY'].read() ivar = hdus['IVAR'].read() mask = hdus['MASK'].read() # Verify that we have the expected wavelengths. # assert np.allclose(detected[camera].wave, hdus['WAVELENGTH'].read()) # Verify that ivar is purely statistical. # assert np.array_equal(ivar, hdus['STATIVAR'].read()) # Verify that the model has no masked pixels. # assert np.all((mask == 0) & (ivar > 0)) # Verify that the sky model is constant. # assert np.array_equal(np.max(ivar, axis=0), np.min(ivar, axis=0)) # assert np.allclose(np.max(flux, axis=0), np.min(flux, axis=0)) # There are actually small variations in flux! # TODO: figure out where these variations come from. # For now, take the median over fibers. if fd["IVAR"][0, :][0].max() > 0: sky_cam += fd["SKY"][0, :][ 0] # AR reading the first fiber only norm_cam += np.ones(len(fullwave[cslice[camera]])) # sky_cam += np.median(flux, axis=0) # norm_cam += np.ones(len(fullwave[cslice[camera]])) fd.close() # AR sky model flux in incident photon / angstrom / s # if nspec > 0: keep = norm_cam > 0 if keep.sum() > 0: # [e/A/s] / throughput. sky[cslice[camera]][keep] = (sky_cam[keep] / norm_cam[keep] / exptime / spec_thru[camera][keep]) else: print("{}-{}-{}: no spectra for {}".format(night, expid, camera, ftype)) # AR sky model flux in erg / angstrom / s (using the photon energy in erg). e_phot_erg = (constants.h.to(units.erg * units.s) * constants.c.to(units.angstrom / units.s) / (fullwave * units.angstrom)) sky *= e_phot_erg.value # AR sky model flux in [erg / angstrom / s / cm**2 / arcsec**2]. sky /= (telap_cm2 * fiber_area_arcsec2) if smoothing > 0.0: sky = scipy.ndimage.gaussian_filter1d(sky, 100.) # AR integrate over the DECam r-band vfilter = filters.load_filters('bessell-V') rfilter = filters.load_filters('decam2014-r') # AR zero-padding spectrum so that it covers the DECam r-band range sky_pad, fullwave_pad = vfilter.pad_spectrum(sky, fullwave, method="zero") vmag = vfilter.get_ab_magnitudes( sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom), fullwave_pad * units.angstrom).as_array()[0][0] # AR zero-padding spectrum so that it covers the DECam r-band range sky_pad, fullwave_pad = rfilter.pad_spectrum(sky, fullwave, method="zero") rmag = rfilter.get_ab_magnitudes( sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom), fullwave_pad * units.angstrom).as_array()[0][0] if specsim_darksky: fd = fitsio.FITS(fn) # Dark Sky at given airmass (cnst. across spectrograph / camera). simulator.atmosphere.airmass = fd['SKY'].read_header()['AIRMASS'] # Force below the horizon: dark sky. simulator.atmosphere.moon.moon_zenith = 120. * u.deg simulator.simulate() # [1e-17 erg / (Angstrom arcsec2 cm2 s)]. sim_darksky = simulator.atmosphere.surface_brightness sim_darksky *= 1.e-17 dsky_pad, dskywave_pad = rfilter.pad_spectrum(sim_darksky.value, config.wavelength.value, method="zero") dsky_rmag = rfilter.get_ab_magnitudes( dsky_pad * units.erg / (units.cm**2 * units.s * units.angstrom), dskywave_pad * units.angstrom).as_array()[0][0] sim_darksky = resample_flux(fullwave, config.wavelength.value, sim_darksky.value) sim_darksky *= u.erg / (u.cm**2 * u.s * u.angstrom * u.arcsec**2) sky = np.clip(sky - sim_darksky.value, a_min=0.0, a_max=None) # AR zero-padding spectrum so that it covers the DECam r-band range sky_pad, fullwave_pad = vfilter.pad_spectrum(sky, fullwave, method="zero") vmag_nodark = vfilter.get_ab_magnitudes( sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom), fullwave_pad * units.angstrom).as_array()[0][0] # AR zero-padding spectrum so that it covers the DECam r-band range sky_pad, fullwave_pad = rfilter.pad_spectrum(sky, fullwave, method="zero") rmag_nodark = rfilter.get_ab_magnitudes( sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom), fullwave_pad * units.angstrom).as_array()[0][0] return fullwave, sky, vmag, rmag, vmag_nodark, rmag_nodark elif nightly_darksky: from pkg_resources import resource_filename if night in nightly_dsky_cache.keys(): return nightly_dsky_cache[night] gfa_info = resource_filename('bgs-cmxsv', 'dat/sv1-exposures.fits') gfa_info = Table.read(gfa_info) bright_cut = 20.50 good_conds = (gfa_info['GFA_TRANSPARENCY_MED'] > 0.95) & (gfa_info['GFA_SKY_MAG_AB_MED'] >= bright_cut) good_conds = gfa_info[good_conds] expids = np.array([ np.int(x.split('/')[-1]) for x in glob.glob( os.path.join(reduxdir, "exposures", "{}/00*".format(night))) ]) isgood = np.isin(good_conds['EXPID'], expids) if np.any(isgood): good_conds = good_conds[isgood] best = good_conds['GFA_SKY_MAG_AB_MED'] == good_conds[ 'GFA_SKY_MAG_AB_MED'].max() print('Nightly Dark GFA r-mag for {}: {}'.format( night, good_conds['GFA_SKY_MAG_AB_MED'].max())) best_expid = good_conds[best]['EXPID'][0] best_expid = '{:08d}'.format(best_expid) darkwave, darksky, dark_vmag, dark_rmag = get_sky( night, best_expid, exptime, ftype="model", redux="daily", smoothing=0.0, specsim_darksky=False, nightly_darksky=False) sky = np.clip(sky - darksky, a_min=0.0, a_max=None) # AR zero-padding spectrum so that it covers the DECam r-band range sky_pad, fullwave_pad = vfilter.pad_spectrum(sky, fullwave, method="zero") vmag_nodark = vfilter.get_ab_magnitudes( sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom), fullwave_pad * units.angstrom).as_array()[0][0] # AR zero-padding spectrum so that it covers the DECam r-band range sky_pad, fullwave_pad = rfilter.pad_spectrum(sky, fullwave, method="zero") rmag_nodark = rfilter.get_ab_magnitudes( sky_pad * units.erg / (units.cm**2 * units.s * units.angstrom), fullwave_pad * units.angstrom).as_array()[0][0] nightly_dsky_cache[ night] = fullwave, sky, vmag, rmag, vmag_nodark, rmag_nodark else: print( 'No nightly darksky available for night: {}. Defaulting to specsim.' .format(night)) nightly_dsky_cache[night] = get_sky(night, expid, exptime, ftype="model", redux="daily", smoothing=0.0, specsim_darksky=True, nightly_darksky=False) return nightly_dsky_cache[night] else: pass return fullwave, sky, vmag, rmag
def main() : parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--infile', type = str, default = None, required=True, help = 'path of DESI exposure frame fits file') parser.add_argument('--fibermap', type = str, default = None, required=True, help = 'path of DESI exposure frame fits file') parser.add_argument('--fiberflat', type = str, default = None, required=True, help = 'path of DESI fiberflat fits file') parser.add_argument('--sky', type = str, default = None, required=True, help = 'path of DESI sky fits file') parser.add_argument('--models', type = str, default = None, required=True, help = 'path of spetro-photometric stellar spectra fits file') parser.add_argument('--outfile', type = str, default = None, required=True, help = 'path of DESI flux calbration fits file') args = parser.parse_args() log=get_logger() log.info("read frame") # read frame frame = read_frame(args.infile) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel=read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux,model_wave,model_fibers=read_stdstar_models(args.models) # select fibers SPECMIN=frame.header["SPECMIN"] SPECMAX=frame.header["SPECMAX"] selec=np.where((model_fibers>=SPECMIN)&(model_fibers<=SPECMAX))[0] if selec.size == 0 : log.error("no stellar models for this spectro") sys.exit(12) fibers=model_fibers[selec]-frame.header["SPECMIN"] log.info("star fibers= %s"%str(fibers)) table = read_fibermap(args.fibermap) bad=np.where(table["OBJTYPE"][fibers]!="STD")[0] if bad.size > 0 : for fiber in fibers[bad] : log.error("inconsistency with fiber %d, OBJTYPE='%s' in fibermap"%(fiber,table["OBJTYPE"][fiber])) sys.exit(12) fluxcalib = compute_flux_calibration(frame, fibers, model_wave, model_flux) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.header) log.info("successfully wrote %s"%args.outfile)
def main(args): """ finds the best models of all standard stars in the frame and normlize the model flux. Output is written to a file and will be called for calibration. """ log = get_logger() log.info("mag delta %s = %f (for the pre-selection of stellar models)" % (args.color, args.delta_color)) frames = {} flats = {} skies = {} spectrograph = None starfibers = None starindices = None fibermap = None # READ DATA ############################################ for filename in args.frames: log.info("reading %s" % filename) frame = io.read_frame(filename) header = fits.getheader(filename, 0) frame_fibermap = frame.fibermap frame_starindices = np.where(isStdStar(frame_fibermap))[0] #- Confirm that all fluxes have entries but trust targeting bits #- to get basic magnitude range correct keep = np.ones(len(frame_starindices), dtype=bool) for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']: #- and W1 and W2? keep &= frame_fibermap[colname][frame_starindices] > 10**( (22.5 - 30) / 2.5) keep &= frame_fibermap[colname][frame_starindices] < 10**( (22.5 - 0) / 2.5) frame_starindices = frame_starindices[keep] camera = safe_read_key(header, "CAMERA").strip().lower() if spectrograph is None: spectrograph = frame.spectrograph fibermap = frame_fibermap starindices = frame_starindices starfibers = fibermap["FIBER"][starindices] elif spectrograph != frame.spectrograph: log.error("incompatible spectrographs %d != %d" % (spectrograph, frame.spectrograph)) raise ValueError("incompatible spectrographs %d != %d" % (spectrograph, frame.spectrograph)) elif starindices.size != frame_starindices.size or np.sum( starindices != frame_starindices) > 0: log.error("incompatible fibermap") raise ValueError("incompatible fibermap") if not camera in frames: frames[camera] = [] frames[camera].append(frame) for filename in args.skymodels: log.info("reading %s" % filename) sky = io.read_sky(filename) header = fits.getheader(filename, 0) camera = safe_read_key(header, "CAMERA").strip().lower() if not camera in skies: skies[camera] = [] skies[camera].append(sky) for filename in args.fiberflats: log.info("reading %s" % filename) header = fits.getheader(filename, 0) flat = io.read_fiberflat(filename) camera = safe_read_key(header, "CAMERA").strip().lower() # NEED TO ADD MORE CHECKS if camera in flats: log.warning( "cannot handle several flats of same camera (%s), will use only the first one" % camera) #raise ValueError("cannot handle several flats of same camera (%s)"%camera) else: flats[camera] = flat if starindices.size == 0: log.error("no STD star found in fibermap") raise ValueError("no STD star found in fibermap") log.info("found %d STD stars" % starindices.size) log.warning("Not using flux errors for Standard Star fits!") # DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA ############################################ for cam in frames: if not cam in skies: log.warning("Missing sky for %s" % cam) frames.pop(cam) continue if not cam in flats: log.warning("Missing flat for %s" % cam) frames.pop(cam) continue flat = flats[cam] for frame, sky in zip(frames[cam], skies[cam]): frame.flux = frame.flux[starindices] frame.ivar = frame.ivar[starindices] frame.ivar *= (frame.mask[starindices] == 0) frame.ivar *= (sky.ivar[starindices] != 0) frame.ivar *= (sky.mask[starindices] == 0) frame.ivar *= (flat.ivar[starindices] != 0) frame.ivar *= (flat.mask[starindices] == 0) frame.flux *= (frame.ivar > 0) # just for clean plots for star in range(frame.flux.shape[0]): ok = np.where((frame.ivar[star] > 0) & (flat.fiberflat[star] != 0))[0] if ok.size > 0: frame.flux[star] = frame.flux[star] / flat.fiberflat[ star] - sky.flux[star] frame.resolution_data = frame.resolution_data[starindices] nstars = starindices.size fibermap = Table(fibermap[starindices]) # READ MODELS ############################################ log.info("reading star models in %s" % args.starmodels) stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates( args.starmodels) # COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG ############################################ #- Support older fibermaps if 'PHOTSYS' not in fibermap.colnames: log.warning('Old fibermap format; using defaults for missing columns') log.warning(" PHOTSYS = 'S'") log.warning(" MW_TRANSMISSION_G/R/Z = 1.0") log.warning(" EBV = 0.0") fibermap['PHOTSYS'] = 'S' fibermap['MW_TRANSMISSION_G'] = 1.0 fibermap['MW_TRANSMISSION_R'] = 1.0 fibermap['MW_TRANSMISSION_Z'] = 1.0 fibermap['EBV'] = 0.0 model_filters = dict() if 'S' in fibermap['PHOTSYS']: for filter_name in ['DECAM_G', 'DECAM_R', 'DECAM_Z']: model_filters[filter_name] = load_filter(filter_name) if 'N' in fibermap['PHOTSYS']: for filter_name in ['BASS_G', 'BASS_R', 'MZLS_Z']: model_filters[filter_name] = load_filter(filter_name) if len(model_filters) == 0: raise ValueError("No filters loaded; neither 'N' nor 'S' in PHOTSYS?") log.info("computing model mags for %s" % sorted(model_filters.keys())) model_mags = dict() fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom for filter_name, filter_response in model_filters.items(): model_mags[filter_name] = filter_response.get_ab_magnitude( stdflux * fluxunits, stdwave) log.info("done computing model mags") # LOOP ON STARS TO FIND BEST MODEL ############################################ linear_coefficients = np.zeros((nstars, stdflux.shape[0])) chi2dof = np.zeros((nstars)) redshift = np.zeros((nstars)) normflux = [] star_mags = dict() star_unextincted_mags = dict() for band in ['G', 'R', 'Z']: star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_' + band]) star_unextincted_mags[band] = 22.5 - 2.5 * np.log10( fibermap['FLUX_' + band] / fibermap['MW_TRANSMISSION_' + band]) star_colors = dict() star_colors['G-R'] = star_mags['G'] - star_mags['R'] star_colors['R-Z'] = star_mags['R'] - star_mags['Z'] star_unextincted_colors = dict() star_unextincted_colors[ 'G-R'] = star_unextincted_mags['G'] - star_unextincted_mags['R'] star_unextincted_colors[ 'R-Z'] = star_unextincted_mags['R'] - star_unextincted_mags['Z'] fitted_model_colors = np.zeros(nstars) for star in range(nstars): log.info("finding best model for observed star #%d" % star) # np.array of wave,flux,ivar,resol wave = {} flux = {} ivar = {} resolution_data = {} for camera in frames: for i, frame in enumerate(frames[camera]): identifier = "%s-%d" % (camera, i) wave[identifier] = frame.wave flux[identifier] = frame.flux[star] ivar[identifier] = frame.ivar[star] resolution_data[identifier] = frame.resolution_data[star] # preselect models based on magnitudes if fibermap['PHOTSYS'][star] == 'N': if args.color == 'G-R': model_colors = model_mags['BASS_G'] - model_mags['BASS_R'] elif args.color == 'R-Z': model_colors = model_mags['BASS_R'] - model_mags['MZLS_Z'] else: raise ValueError('Unknown color {}'.format(args.color)) else: if args.color == 'G-R': model_colors = model_mags['DECAM_G'] - model_mags['DECAM_R'] elif args.color == 'R-Z': model_colors = model_mags['DECAM_R'] - model_mags['DECAM_Z'] else: raise ValueError('Unknown color {}'.format(args.color)) color_diff = model_colors - star_unextincted_colors[args.color][star] selection = np.abs(color_diff) < args.delta_color # smallest cube in parameter space including this selection (needed for interpolation) new_selection = (teff >= np.min(teff[selection])) & (teff <= np.max( teff[selection])) new_selection &= (logg >= np.min(logg[selection])) & (logg <= np.max( logg[selection])) new_selection &= (feh >= np.min(feh[selection])) & (feh <= np.max( feh[selection])) selection = np.where(new_selection)[0] log.info( "star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d" % (star, starfibers[star], args.color, star_unextincted_colors[args.color][star], selection.size, stdflux.shape[0])) # Match unextincted standard stars to data coefficients, redshift[star], chi2dof[star] = match_templates( wave, flux, ivar, resolution_data, stdwave, stdflux[selection], teff[selection], logg[selection], feh[selection], ncpu=args.ncpu, z_max=args.z_max, z_res=args.z_res, template_error=args.template_error) linear_coefficients[star, selection] = coefficients log.info( 'Star Fiber: {0}; TEFF: {1}; LOGG: {2}; FEH: {3}; Redshift: {4}; Chisq/dof: {5}' .format(starfibers[star], np.inner(teff, linear_coefficients[star]), np.inner(logg, linear_coefficients[star]), np.inner(feh, linear_coefficients[star]), redshift[star], chi2dof[star])) # Apply redshift to original spectrum at full resolution model = np.zeros(stdwave.size) redshifted_stdwave = stdwave * (1 + redshift[star]) for i, c in enumerate(linear_coefficients[star]): if c != 0: model += c * np.interp(stdwave, redshifted_stdwave, stdflux[i]) # Apply dust extinction to the model model *= dust_transmission(stdwave, fibermap['EBV'][star]) # Compute final color of dust-extincted model if fibermap['PHOTSYS'][star] == 'N': if args.color == 'G-R': model_mag1 = model_filters['BASS_G'].get_ab_magnitude( model * fluxunits, stdwave) model_mag2 = model_filters['BASS_R'].get_ab_magnitude( model * fluxunits, stdwave) model_magr = model_mag2 elif args.color == 'R-Z': model_mag1 = model_filters['BASS_R'].get_ab_magnitude( model * fluxunits, stdwave) model_mag2 = model_filters['MZLS_Z'].get_ab_magnitude( model * fluxunits, stdwave) model_magr = model_mag1 else: raise ValueError('Unknown color {}'.format(args.color)) else: if args.color == 'G-R': model_mag1 = model_filters['DECAM_G'].get_ab_magnitude( model * fluxunits, stdwave) model_mag2 = model_filters['DECAM_R'].get_ab_magnitude( model * fluxunits, stdwave) model_magr = model_mag2 elif args.color == 'R-Z': model_mag1 = model_filters['DECAM_R'].get_ab_magnitude( model * fluxunits, stdwave) model_mag2 = model_filters['DECAM_Z'].get_ab_magnitude( model * fluxunits, stdwave) model_magr = model_mag1 else: raise ValueError('Unknown color {}'.format(args.color)) fitted_model_colors[star] = model_mag1 - model_mag2 #- TODO: move this back into normalize_templates, at the cost of #- recalculating a model magnitude? # Normalize the best model using reported magnitude scalefac = 10**((model_magr - star_mags['R'][star]) / 2.5) log.info('scaling R mag {} to {} using scale {}'.format( model_magr, star_mags['R'][star], scalefac)) normflux.append(model * scalefac) # Now write the normalized flux for all best models to a file normflux = np.array(normflux) data = {} data['LOGG'] = linear_coefficients.dot(logg) data['TEFF'] = linear_coefficients.dot(teff) data['FEH'] = linear_coefficients.dot(feh) data['CHI2DOF'] = chi2dof data['REDSHIFT'] = redshift data['COEFF'] = linear_coefficients data['DATA_%s' % args.color] = star_colors[args.color] data['MODEL_%s' % args.color] = fitted_model_colors io.write_stdstar_models(args.outfile, normflux, stdwave, starfibers, data)
def main(args): log = get_logger() cmd = [ 'desi_compute_fluxcalibration', ] for key, value in args.__dict__.items(): if value is not None: cmd += ['--' + key, str(value)] cmd = ' '.join(cmd) log.info(cmd) log.info("read frame") # read frame frame = read_frame(args.infile) # Set fibermask flagged spectra to have 0 flux and variance frame = get_fiberbitmasked_frame(frame, bitmask='flux', ivar_framemask=True) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel = read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models( args.models) ok = np.ones(len(model_metadata), dtype=bool) if args.chi2cut > 0: log.info("Apply cut CHI2DOF<{}".format(args.chi2cut)) ok &= (model_metadata["CHI2DOF"] < args.chi2cut) if args.delta_color_cut > 0: log.info("Apply cut |delta color|<{}".format(args.delta_color_cut)) ok &= (np.abs(model_metadata["MODEL_G-R"] - model_metadata["DATA_G-R"]) < args.delta_color_cut) if args.min_color is not None: log.info("Apply cut DATA_G-R>{}".format(args.min_color)) ok &= (model_metadata["DATA_G-R"] > args.min_color) if args.chi2cut_nsig > 0: # automatically reject stars that ar chi2 outliers mchi2 = np.median(model_metadata["CHI2DOF"]) rmschi2 = np.std(model_metadata["CHI2DOF"]) maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2 log.info("Apply cut CHI2DOF<{} based on chi2cut_nsig={}".format( maxchi2, args.chi2cut_nsig)) ok &= (model_metadata["CHI2DOF"] <= maxchi2) ok = np.where(ok)[0] if ok.size == 0: log.error("cuts discarded all stars") sys.exit(12) nstars = model_flux.shape[0] nbad = nstars - ok.size if nbad > 0: log.warning("discarding %d star(s) out of %d because of cuts" % (nbad, nstars)) model_flux = model_flux[ok] model_fibers = model_fibers[ok] model_metadata = model_metadata[:][ok] # check that the model_fibers are actually standard stars fibermap = frame.fibermap ## check whether star fibers from args.models are consistent with fibers from fibermap ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit fibermap_std_indices = np.where(isStdStar(fibermap))[0] if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)): target_colnames, target_masks, survey = main_cmx_or_sv(fibermap) colname = target_colnames[0] for i in model_fibers % 500: log.error( "inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap" .format(i, fibermap["OBJTYPE"][i], colname, fibermap[colname][i])) sys.exit(12) # Make sure the fibers of interest aren't entirely masked. if np.sum( np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) == frame.nwave) == len(model_fibers): log.warning('All standard-star spectra are masked!') return fluxcalib = compute_flux_calibration( frame, model_wave, model_flux, model_fibers % 500, highest_throughput_nstars=args.highest_throughput) # QA if (args.qafile is not None): log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame_meta=frame.meta, flavor=frame.meta['FLAVOR']) # Run #import pdb; pdb.set_trace() qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s" % args.outfile)
def main(args): """ finds the best models of all standard stars in the frame and normlize the model flux. Output is written to a file and will be called for calibration. """ log = get_logger() log.info("mag delta %s = %f (for the pre-selection of stellar models)" % (args.color, args.delta_color)) frames = {} flats = {} skies = {} spectrograph = None starfibers = None starindices = None fibermap = None # READ DATA ############################################ for filename in args.frames: log.info("reading %s" % filename) frame = io.read_frame(filename) header = fits.getheader(filename, 0) frame_fibermap = frame.fibermap frame_starindices = np.where(frame_fibermap["OBJTYPE"] == "STD")[0] # check magnitude are well defined or discard stars tmp = [] for i in frame_starindices: mags = frame_fibermap["MAG"][i] ok = np.sum((mags > 0) & (mags < 30)) if np.sum((mags > 0) & (mags < 30)) == mags.size: tmp.append(i) frame_starindices = np.array(tmp).astype(int) camera = safe_read_key(header, "CAMERA").strip().lower() if spectrograph is None: spectrograph = frame.spectrograph fibermap = frame_fibermap starindices = frame_starindices starfibers = fibermap["FIBER"][starindices] elif spectrograph != frame.spectrograph: log.error("incompatible spectrographs %d != %d" % (spectrograph, frame.spectrograph)) raise ValueError("incompatible spectrographs %d != %d" % (spectrograph, frame.spectrograph)) elif starindices.size != frame_starindices.size or np.sum( starindices != frame_starindices) > 0: log.error("incompatible fibermap") raise ValueError("incompatible fibermap") if not camera in frames: frames[camera] = [] frames[camera].append(frame) for filename in args.skymodels: log.info("reading %s" % filename) sky = io.read_sky(filename) header = fits.getheader(filename, 0) camera = safe_read_key(header, "CAMERA").strip().lower() if not camera in skies: skies[camera] = [] skies[camera].append(sky) for filename in args.fiberflats: log.info("reading %s" % filename) header = fits.getheader(filename, 0) flat = io.read_fiberflat(filename) camera = safe_read_key(header, "CAMERA").strip().lower() # NEED TO ADD MORE CHECKS if camera in flats: log.warning( "cannot handle several flats of same camera (%s), will use only the first one" % camera) #raise ValueError("cannot handle several flats of same camera (%s)"%camera) else: flats[camera] = flat if starindices.size == 0: log.error("no STD star found in fibermap") raise ValueError("no STD star found in fibermap") log.info("found %d STD stars" % starindices.size) imaging_filters = fibermap["FILTER"][starindices] imaging_mags = fibermap["MAG"][starindices] log.warning( "NO MAG ERRORS IN FIBERMAP, I AM IGNORING MEASUREMENT ERRORS !!") ebv = np.zeros(starindices.size) if "SFD_EBV" in fibermap.columns.names: log.info("Using 'SFD_EBV' from fibermap") ebv = fibermap["SFD_EBV"][starindices] else: log.warning("NO EXTINCTION VALUES IN FIBERMAP!!") # DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA ############################################ for cam in frames: if not cam in skies: log.warning("Missing sky for %s" % cam) frames.pop(cam) continue if not cam in flats: log.warning("Missing flat for %s" % cam) frames.pop(cam) continue flat = flats[cam] for frame, sky in zip(frames[cam], skies[cam]): frame.flux = frame.flux[starindices] frame.ivar = frame.ivar[starindices] frame.ivar *= (frame.mask[starindices] == 0) frame.ivar *= (sky.ivar[starindices] != 0) frame.ivar *= (sky.mask[starindices] == 0) frame.ivar *= (flat.ivar[starindices] != 0) frame.ivar *= (flat.mask[starindices] == 0) frame.flux *= (frame.ivar > 0) # just for clean plots for star in range(frame.flux.shape[0]): ok = np.where((frame.ivar[star] > 0) & (flat.fiberflat[star] != 0))[0] if ok.size > 0: frame.flux[star] = frame.flux[star] / flat.fiberflat[ star] - sky.flux[star] frame.resolution_data = frame.resolution_data[starindices] nstars = starindices.size starindices = None # we don't need this anymore # READ MODELS ############################################ log.info("reading star models in %s" % args.starmodels) stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates( args.starmodels) # COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG ############################################ model_filters = [] for tmp in np.unique(imaging_filters): if len(tmp) > 0: # can be one empty entry model_filters.append(tmp) log.info("computing model mags %s" % model_filters) model_mags = np.zeros((stdflux.shape[0], len(model_filters))) fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom for index in range(len(model_filters)): if model_filters[index].startswith('WISE'): log.warning('not computing stdstar {} mags'.format( model_filters[index])) continue filter_response = load_filter(model_filters[index]) for m in range(stdflux.shape[0]): model_mags[m, index] = filter_response.get_ab_magnitude( stdflux[m] * fluxunits, stdwave) log.info("done computing model mags") mean_extinction_delta_mags = None mean_ebv = np.mean(ebv) if mean_ebv > 0: log.info( "Compute a mean delta_color from average E(B-V) = %3.2f based on canonial model star" % mean_ebv) # compute a mean delta_color from mean_ebv based on canonial model star ####################################################################### # will then use this color offset in the model pre-selection # find canonical f-type model: Teff=6000, logg=4, Fe/H=-1.5 canonical_model = np.argmin((teff - 6000.0)**2 + (logg - 4.0)**2 + (feh + 1.5)**2) canonical_model_mags_without_extinction = model_mags[canonical_model] canonical_model_mags_with_extinction = np.zeros( canonical_model_mags_without_extinction.shape) canonical_model_reddened_flux = stdflux[ canonical_model] * dust_transmission(stdwave, mean_ebv) for index in range(len(model_filters)): if model_filters[index].startswith('WISE'): log.warning('not computing stdstar {} mags'.format( model_filters[index])) continue filter_response = load_filter(model_filters[index]) canonical_model_mags_with_extinction[ index] = filter_response.get_ab_magnitude( canonical_model_reddened_flux * fluxunits, stdwave) mean_extinction_delta_mags = canonical_model_mags_with_extinction - canonical_model_mags_without_extinction # LOOP ON STARS TO FIND BEST MODEL ############################################ linear_coefficients = np.zeros((nstars, stdflux.shape[0])) chi2dof = np.zeros((nstars)) redshift = np.zeros((nstars)) normflux = [] star_colors_array = np.zeros((nstars)) model_colors_array = np.zeros((nstars)) for star in range(nstars): log.info("finding best model for observed star #%d" % star) # np.array of wave,flux,ivar,resol wave = {} flux = {} ivar = {} resolution_data = {} for camera in frames: for i, frame in enumerate(frames[camera]): identifier = "%s-%d" % (camera, i) wave[identifier] = frame.wave flux[identifier] = frame.flux[star] ivar[identifier] = frame.ivar[star] resolution_data[identifier] = frame.resolution_data[star] # preselec models based on magnitudes # compute star color index1, index2 = get_color_filter_indices(imaging_filters[star], args.color) if index1 < 0 or index2 < 0: log.error("cannot compute '%s' color from %s" % (color_name, filters)) filter1 = imaging_filters[star][index1] filter2 = imaging_filters[star][index2] star_color = imaging_mags[star][index1] - imaging_mags[star][index2] star_colors_array[star] = star_color # compute models color model_index1 = -1 model_index2 = -1 for i, fname in enumerate(model_filters): if fname == filter1: model_index1 = i elif fname == filter2: model_index2 = i if model_index1 < 0 or model_index2 < 0: log.error("cannot compute '%s' model color from %s" % (color_name, filters)) model_colors = model_mags[:, model_index1] - model_mags[:, model_index2] # apply extinction here # use the colors derived from the cannonical model with the mean ebv of the stars # and simply apply a scaling factor based on the ebv of this star # this is sufficiently precise for the broad model pre-selection we are doing here # the exact reddening of the star to each pre-selected model is # apply afterwards if mean_extinction_delta_mags is not None and mean_ebv != 0: delta_color = (mean_extinction_delta_mags[model_index1] - mean_extinction_delta_mags[model_index2] ) * ebv[star] / mean_ebv model_colors += delta_color log.info( "Apply a %s-%s color offset = %4.3f to the models for star with E(B-V)=%4.3f" % (model_filters[model_index1], model_filters[model_index2], delta_color, ebv[star])) # selection selection = np.abs(model_colors - star_color) < args.delta_color # smallest cube in parameter space including this selection (needed for interpolation) new_selection = (teff >= np.min(teff[selection])) & (teff <= np.max( teff[selection])) new_selection &= (logg >= np.min(logg[selection])) & (logg <= np.max( logg[selection])) new_selection &= (feh >= np.min(feh[selection])) & (feh <= np.max( feh[selection])) selection = np.where(new_selection)[0] log.info( "star#%d fiber #%d, %s = %s-%s = %f, number of pre-selected models = %d/%d" % (star, starfibers[star], args.color, filter1, filter2, star_color, selection.size, stdflux.shape[0])) # apply extinction to selected_models dust_transmission_of_this_star = dust_transmission(stdwave, ebv[star]) selected_reddened_stdflux = stdflux[ selection] * dust_transmission_of_this_star coefficients, redshift[star], chi2dof[star] = match_templates( wave, flux, ivar, resolution_data, stdwave, selected_reddened_stdflux, teff[selection], logg[selection], feh[selection], ncpu=args.ncpu, z_max=args.z_max, z_res=args.z_res, template_error=args.template_error) linear_coefficients[star, selection] = coefficients log.info( 'Star Fiber: {0}; TEFF: {1}; LOGG: {2}; FEH: {3}; Redshift: {4}; Chisq/dof: {5}' .format(starfibers[star], np.inner(teff, linear_coefficients[star]), np.inner(logg, linear_coefficients[star]), np.inner(feh, linear_coefficients[star]), redshift[star], chi2dof[star])) # Apply redshift to original spectrum at full resolution model = np.zeros(stdwave.size) for i, c in enumerate(linear_coefficients[star]): if c != 0: model += c * np.interp(stdwave, stdwave * (1 + redshift[star]), stdflux[i]) # Apply dust extinction model *= dust_transmission_of_this_star # Compute final model color mag1 = load_filter(model_filters[model_index1]).get_ab_magnitude( model * fluxunits, stdwave) mag2 = load_filter(model_filters[model_index2]).get_ab_magnitude( model * fluxunits, stdwave) model_colors_array[star] = mag1 - mag2 # Normalize the best model using reported magnitude normalizedflux = normalize_templates(stdwave, model, imaging_mags[star], imaging_filters[star]) normflux.append(normalizedflux) # Now write the normalized flux for all best models to a file normflux = np.array(normflux) data = {} data['LOGG'] = linear_coefficients.dot(logg) data['TEFF'] = linear_coefficients.dot(teff) data['FEH'] = linear_coefficients.dot(feh) data['CHI2DOF'] = chi2dof data['REDSHIFT'] = redshift data['COEFF'] = linear_coefficients data['DATA_%s' % args.color] = star_colors_array data['MODEL_%s' % args.color] = model_colors_array norm_model_file = args.outfile io.write_stdstar_models(args.outfile, normflux, stdwave, starfibers, data)
def make_frameqa(self, make_plots=False, clobber=True): """ Work through the Production and make QA for all frames Parameters: make_plots: bool, optional Remake the plots too? clobber: bool, optional Returns: """ # imports from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.fiberflat import read_fiberflat from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots from desispec.io.fluxcalibration import read_stdstar_models # Loop on nights path_nights = glob.glob(self.specprod_dir+'/exposures/*') nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights] for night in nights: for exposure in get_exposures(night, specprod_dir = self.specprod_dir): # Object only?? frames_dict = get_files(filetype = str('frame'), night = night, expid = exposure, specprod_dir = self.specprod_dir) for camera,frame_fil in frames_dict.items(): # Load frame frame = read_frame(frame_fil) spectro = int(frame.meta['CAMERA'][-1]) if frame.meta['FLAVOR'] in ['flat','arc']: qatype = 'qa_calib' else: qatype = 'qa_data' qafile = meta.findfile(qatype, night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) if (not clobber) & os.path.isfile(qafile): log.info("qafile={:s} exists. Not over-writing. Consider clobber=True".format(qafile)) continue # Load qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame.meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fiberflat = read_fiberflat(fiberflat_fil) qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber) if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) skymodel = read_sky(sky_fil) qaframe.run_qa('SKYSUB', (frame, skymodel)) if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir, spectrograph=spectro) model_tuple=read_stdstar_models(stdstar_fil) flux_fil = meta.findfile('calib', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fluxcalib = read_flux_calibration(flux_fil) qaframe.run_qa('FLUXCALIB', (frame, fluxcalib, model_tuple))#, indiv_stars)) if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib, model_tuple) # Write write_qa_frame(qafile, qaframe)
ii = np.where((table["PETAL"] == 0) & (table["FIBER"] >= 0))[0] xk = "OFFSET_X" yk = "OFFSET_Y" fiberid = "FIBER" dico = {} for i in ii: dico[table[fiberid][i]] = i expid_arr = ["00033801"] #["00020206","00020207"] flux_b3_all = [] flux_r3_all = [] flux_z3_all = [] for i in range(len(expid_arr)): expid = expid_arr[i] frame_b3 = read_frame("flatfielded-frame-b3-" + expid + "-twilightcorr.fits") flux_b3 = np.median(frame_b3.flux, axis=1) flux_b3 /= np.median(flux_b3) frame_r3 = read_frame("flatfielded-frame-r3-" + expid + "-twilightcorr.fits") flux_r3 = np.median(frame_r3.flux, axis=1) flux_r3 /= np.median(flux_r3) frame_z3 = read_frame("flatfielded-frame-z3-" + expid + "-twilightcorr.fits") flux_z3 = np.median(frame_z3.flux, axis=1) flux_z3 /= np.median(flux_z3) flux_b3_all.append(flux_b3) flux_r3_all.append(flux_r3)
def get_skyres(cframes, sub_sky=False, flatten=True): """ Args: cframes: str or list Single cframe or a list of them sub_sky: bool, optional Subtract the sky? This should probably not be done flatten: bool, optional Return a flat, 1D array for each variable combine: bool, optional combine the individual sky fibers? Median 'smash' Returns: wave : ndarray flux : ndarray res : ndarray ivar : ndarray """ from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.sky import subtract_sky if isinstance(cframes,list): all_wave, all_flux, all_res, all_ivar = [], [], [], [] for cframe_file in cframes: wave, flux, res, ivar = get_skyres(cframe_file, flatten=flatten) # Save all_wave.append(wave) all_flux.append(flux) all_res.append(res) all_ivar.append(ivar) # Concatenate -- Shape is preserved (nfibers, npix) twave = np.concatenate(all_wave) tflux = np.concatenate(all_flux) tres = np.concatenate(all_res) tivar = np.concatenate(all_ivar) # Return return twave, tflux, tres, tivar cframe = read_frame(cframes, skip_resolution=True) if cframe.meta['FLAVOR'] in ['flat','arc']: raise ValueError("Bad flavor for exposure: {:s}".format(cframes)) # Sky sky_file = cframes.replace('cframe', 'sky') skymodel = read_sky(sky_file) if sub_sky: subtract_sky(cframe, skymodel) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] # Flux calibrated ivar = cframe.ivar[skyfibers] # Flux calibrated flux = skymodel.flux[skyfibers] # Residuals; not flux calibrated! wave = np.outer(np.ones(flux.shape[0]), cframe.wave) # Combine? ''' if combine: res = np.median(res, axis=0) ivar = np.median(ivar, axis=0) flux = np.median(flux, axis=0) wave = np.median(wave, axis=0) ''' # Return if flatten: return wave.flatten(), flux.flatten(), res.flatten(), ivar.flatten() else: return wave, flux, res, ivar
def main(args) : """ finds the best models of all standard stars in the frame and normlize the model flux. Output is written to a file and will be called for calibration. """ log = get_logger() log.info("mag delta %s = %f (for the pre-selection of stellar models)"%(args.color,args.delta_color)) frames={} flats={} skies={} spectrograph=None starfibers=None starindices=None fibermap=None # READ DATA ############################################ for filename in args.frames : log.info("reading %s"%filename) frame=io.read_frame(filename) header=fits.getheader(filename, 0) frame_fibermap = frame.fibermap frame_starindices = np.where(isStdStar(frame_fibermap['DESI_TARGET']))[0] #- Confirm that all fluxes have entries but trust targeting bits #- to get basic magnitude range correct keep = np.ones(len(frame_starindices), dtype=bool) for colname in ['FLUX_G', 'FLUX_R', 'FLUX_Z']: #- and W1 and W2? keep &= frame_fibermap[colname][frame_starindices] > 10**((22.5-30)/2.5) keep &= frame_fibermap[colname][frame_starindices] < 10**((22.5-0)/2.5) frame_starindices = frame_starindices[keep] camera=safe_read_key(header,"CAMERA").strip().lower() if spectrograph is None : spectrograph = frame.spectrograph fibermap = frame_fibermap starindices=frame_starindices starfibers=fibermap["FIBER"][starindices] elif spectrograph != frame.spectrograph : log.error("incompatible spectrographs %d != %d"%(spectrograph,frame.spectrograph)) raise ValueError("incompatible spectrographs %d != %d"%(spectrograph,frame.spectrograph)) elif starindices.size != frame_starindices.size or np.sum(starindices!=frame_starindices)>0 : log.error("incompatible fibermap") raise ValueError("incompatible fibermap") if not camera in frames : frames[camera]=[] frames[camera].append(frame) for filename in args.skymodels : log.info("reading %s"%filename) sky=io.read_sky(filename) header=fits.getheader(filename, 0) camera=safe_read_key(header,"CAMERA").strip().lower() if not camera in skies : skies[camera]=[] skies[camera].append(sky) for filename in args.fiberflats : log.info("reading %s"%filename) header=fits.getheader(filename, 0) flat=io.read_fiberflat(filename) camera=safe_read_key(header,"CAMERA").strip().lower() # NEED TO ADD MORE CHECKS if camera in flats: log.warning("cannot handle several flats of same camera (%s), will use only the first one"%camera) #raise ValueError("cannot handle several flats of same camera (%s)"%camera) else : flats[camera]=flat if starindices.size == 0 : log.error("no STD star found in fibermap") raise ValueError("no STD star found in fibermap") log.info("found %d STD stars"%starindices.size) log.warning("Not using flux errors for Standard Star fits!") # DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA ############################################ for cam in frames : if not cam in skies: log.warning("Missing sky for %s"%cam) frames.pop(cam) continue if not cam in flats: log.warning("Missing flat for %s"%cam) frames.pop(cam) continue flat=flats[cam] for frame,sky in zip(frames[cam],skies[cam]) : frame.flux = frame.flux[starindices] frame.ivar = frame.ivar[starindices] frame.ivar *= (frame.mask[starindices] == 0) frame.ivar *= (sky.ivar[starindices] != 0) frame.ivar *= (sky.mask[starindices] == 0) frame.ivar *= (flat.ivar[starindices] != 0) frame.ivar *= (flat.mask[starindices] == 0) frame.flux *= ( frame.ivar > 0) # just for clean plots for star in range(frame.flux.shape[0]) : ok=np.where((frame.ivar[star]>0)&(flat.fiberflat[star]!=0))[0] if ok.size > 0 : frame.flux[star] = frame.flux[star]/flat.fiberflat[star] - sky.flux[star] frame.resolution_data = frame.resolution_data[starindices] nstars = starindices.size fibermap = Table(fibermap[starindices]) # READ MODELS ############################################ log.info("reading star models in %s"%args.starmodels) stdwave,stdflux,templateid,teff,logg,feh=io.read_stdstar_templates(args.starmodels) # COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG ############################################ #- Support older fibermaps if 'PHOTSYS' not in fibermap.colnames: log.warning('Old fibermap format; using defaults for missing columns') log.warning(" PHOTSYS = 'S'") log.warning(" MW_TRANSMISSION_G/R/Z = 1.0") log.warning(" EBV = 0.0") fibermap['PHOTSYS'] = 'S' fibermap['MW_TRANSMISSION_G'] = 1.0 fibermap['MW_TRANSMISSION_R'] = 1.0 fibermap['MW_TRANSMISSION_Z'] = 1.0 fibermap['EBV'] = 0.0 model_filters = dict() if 'S' in fibermap['PHOTSYS']: for filter_name in ['DECAM_G', 'DECAM_R', 'DECAM_Z']: model_filters[filter_name] = load_filter(filter_name) if 'N' in fibermap['PHOTSYS']: for filter_name in ['BASS_G', 'BASS_R', 'MZLS_Z']: model_filters[filter_name] = load_filter(filter_name) if len(model_filters) == 0: raise ValueError("No filters loaded; neither 'N' nor 'S' in PHOTSYS?") log.info("computing model mags for %s"%sorted(model_filters.keys())) model_mags = dict() fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom for filter_name, filter_response in model_filters.items(): model_mags[filter_name] = filter_response.get_ab_magnitude(stdflux*fluxunits,stdwave) log.info("done computing model mags") # LOOP ON STARS TO FIND BEST MODEL ############################################ linear_coefficients=np.zeros((nstars,stdflux.shape[0])) chi2dof=np.zeros((nstars)) redshift=np.zeros((nstars)) normflux=[] star_mags = dict() star_unextincted_mags = dict() for band in ['G', 'R', 'Z']: star_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_'+band]) star_unextincted_mags[band] = 22.5 - 2.5 * np.log10(fibermap['FLUX_'+band] / fibermap['MW_TRANSMISSION_'+band]) star_colors = dict() star_colors['G-R'] = star_mags['G'] - star_mags['R'] star_colors['R-Z'] = star_mags['R'] - star_mags['Z'] star_unextincted_colors = dict() star_unextincted_colors['G-R'] = star_unextincted_mags['G'] - star_unextincted_mags['R'] star_unextincted_colors['R-Z'] = star_unextincted_mags['R'] - star_unextincted_mags['Z'] fitted_model_colors = np.zeros(nstars) for star in range(nstars) : log.info("finding best model for observed star #%d"%star) # np.array of wave,flux,ivar,resol wave = {} flux = {} ivar = {} resolution_data = {} for camera in frames : for i,frame in enumerate(frames[camera]) : identifier="%s-%d"%(camera,i) wave[identifier]=frame.wave flux[identifier]=frame.flux[star] ivar[identifier]=frame.ivar[star] resolution_data[identifier]=frame.resolution_data[star] # preselect models based on magnitudes if fibermap['PHOTSYS'][star] == 'N': if args.color == 'G-R': model_colors = model_mags['BASS_G'] - model_mags['BASS_R'] elif args.color == 'R-Z': model_colors = model_mags['BASS_R'] - model_mags['MZLS_Z'] else: raise ValueError('Unknown color {}'.format(args.color)) else: if args.color == 'G-R': model_colors = model_mags['DECAM_G'] - model_mags['DECAM_R'] elif args.color == 'R-Z': model_colors = model_mags['DECAM_R'] - model_mags['DECAM_Z'] else: raise ValueError('Unknown color {}'.format(args.color)) color_diff = model_colors - star_unextincted_colors[args.color][star] selection = np.abs(color_diff) < args.delta_color # smallest cube in parameter space including this selection (needed for interpolation) new_selection = (teff>=np.min(teff[selection]))&(teff<=np.max(teff[selection])) new_selection &= (logg>=np.min(logg[selection]))&(logg<=np.max(logg[selection])) new_selection &= (feh>=np.min(feh[selection]))&(feh<=np.max(feh[selection])) selection = np.where(new_selection)[0] log.info("star#%d fiber #%d, %s = %f, number of pre-selected models = %d/%d"%( star, starfibers[star], args.color, star_unextincted_colors[args.color][star], selection.size, stdflux.shape[0])) # Match unextincted standard stars to data coefficients, redshift[star], chi2dof[star] = match_templates( wave, flux, ivar, resolution_data, stdwave, stdflux[selection], teff[selection], logg[selection], feh[selection], ncpu=args.ncpu, z_max=args.z_max, z_res=args.z_res, template_error=args.template_error ) linear_coefficients[star,selection] = coefficients log.info('Star Fiber: {0}; TEFF: {1}; LOGG: {2}; FEH: {3}; Redshift: {4}; Chisq/dof: {5}'.format( starfibers[star], np.inner(teff,linear_coefficients[star]), np.inner(logg,linear_coefficients[star]), np.inner(feh,linear_coefficients[star]), redshift[star], chi2dof[star]) ) # Apply redshift to original spectrum at full resolution model=np.zeros(stdwave.size) redshifted_stdwave = stdwave*(1+redshift[star]) for i,c in enumerate(linear_coefficients[star]) : if c != 0 : model += c*np.interp(stdwave,redshifted_stdwave,stdflux[i]) # Apply dust extinction to the model model *= dust_transmission(stdwave, fibermap['EBV'][star]) # Compute final color of dust-extincted model if fibermap['PHOTSYS'][star] == 'N': if args.color == 'G-R': model_mag1 = model_filters['BASS_G'].get_ab_magnitude(model*fluxunits, stdwave) model_mag2 = model_filters['BASS_R'].get_ab_magnitude(model*fluxunits, stdwave) model_magr = model_mag2 elif args.color == 'R-Z': model_mag1 = model_filters['BASS_R'].get_ab_magnitude(model*fluxunits, stdwave) model_mag2 = model_filters['MZLS_Z'].get_ab_magnitude(model*fluxunits, stdwave) model_magr = model_mag1 else: raise ValueError('Unknown color {}'.format(args.color)) else: if args.color == 'G-R': model_mag1 = model_filters['DECAM_G'].get_ab_magnitude(model*fluxunits, stdwave) model_mag2 = model_filters['DECAM_R'].get_ab_magnitude(model*fluxunits, stdwave) model_magr = model_mag2 elif args.color == 'R-Z': model_mag1 = model_filters['DECAM_R'].get_ab_magnitude(model*fluxunits, stdwave) model_mag2 = model_filters['DECAM_Z'].get_ab_magnitude(model*fluxunits, stdwave) model_magr = model_mag1 else: raise ValueError('Unknown color {}'.format(args.color)) fitted_model_colors[star] = model_mag1 - model_mag2 #- TODO: move this back into normalize_templates, at the cost of #- recalculating a model magnitude? # Normalize the best model using reported magnitude scalefac=10**((model_magr - star_mags['R'][star])/2.5) log.info('scaling R mag {} to {} using scale {}'.format(model_magr, star_mags['R'][star], scalefac)) normflux.append(model*scalefac) # Now write the normalized flux for all best models to a file normflux=np.array(normflux) data={} data['LOGG']=linear_coefficients.dot(logg) data['TEFF']= linear_coefficients.dot(teff) data['FEH']= linear_coefficients.dot(feh) data['CHI2DOF']=chi2dof data['REDSHIFT']=redshift data['COEFF']=linear_coefficients data['DATA_%s'%args.color]=star_colors[args.color] data['MODEL_%s'%args.color]=fitted_model_colors io.write_stdstar_models(args.outfile,normflux,stdwave,starfibers,data)
def main(args) : log=get_logger() cmd = ['desi_compute_fluxcalibration',] for key, value in args.__dict__.items(): if value is not None: cmd += ['--'+key, str(value)] cmd = ' '.join(cmd) log.info(cmd) log.info("read frame") # read frame frame = read_frame(args.infile) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel=read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux,model_wave,model_fibers,model_metadata=read_stdstar_models(args.models) if args.chi2cut > 0 : ok = np.where(model_metadata["CHI2DOF"]<args.chi2cut)[0] if ok.size == 0 : log.error("chi2cut has discarded all stars") sys.exit(12) nstars=model_flux.shape[0] nbad=nstars-ok.size if nbad>0 : log.warning("discarding %d star(s) out of %d because of chi2cut"%(nbad,nstars)) model_flux=model_flux[ok] model_fibers=model_fibers[ok] model_metadata=model_metadata[:][ok] if args.delta_color_cut > 0 : ok = np.where(np.abs(model_metadata["MODEL_G-R"]-model_metadata["DATA_G-R"])<args.delta_color_cut)[0] nstars=model_flux.shape[0] nbad=nstars-ok.size if nbad>0 : log.warning("discarding %d star(s) out of %d because |delta_color|>%f"%(nbad,nstars,args.delta_color_cut)) model_flux=model_flux[ok] model_fibers=model_fibers[ok] model_metadata=model_metadata[:][ok] # automatically reject stars that ar chi2 outliers if args.chi2cut_nsig > 0 : mchi2=np.median(model_metadata["CHI2DOF"]) rmschi2=np.std(model_metadata["CHI2DOF"]) maxchi2=mchi2+args.chi2cut_nsig*rmschi2 ok=np.where(model_metadata["CHI2DOF"]<=maxchi2)[0] nstars=model_flux.shape[0] nbad=nstars-ok.size if nbad>0 : log.warning("discarding %d star(s) out of %d because reduced chi2 outliers (at %d sigma, giving rchi2<%f )"%(nbad,nstars,args.chi2cut_nsig,maxchi2)) model_flux=model_flux[ok] model_fibers=model_fibers[ok] model_metadata=model_metadata[:][ok] # check that the model_fibers are actually standard stars fibermap = frame.fibermap ## check whether star fibers from args.models are consistent with fibers from fibermap ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit fibermap_std_indices = np.where(isStdStar(fibermap['DESI_TARGET']))[0] if np.any(~np.in1d(model_fibers%500, fibermap_std_indices)): for i in model_fibers%500: log.error("inconsistency with spectrum {}, OBJTYPE='{}', DESI_TARGET={} in fibermap".format( (i, fibermap["OBJTYPE"][i], fibermap["DESI_TARGET"][i]))) sys.exit(12) fluxcalib = compute_flux_calibration(frame, model_wave, model_flux, model_fibers%500) # QA if (args.qafile is not None): log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run #import pdb; pdb.set_trace() qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s"%args.outfile)
xk = "OFFSET_X" yk = "OFFSET_Y" fiberid = "FIBER" dico = {} for i in ii: dico[table[fiberid][i]] = i frame_file_b3 = '/project/projectdirs/desi/spectro/redux/v0/exposures/20191028/00022301/frame-b3-00022301.fits' frame_file_r3 = '/project/projectdirs/desi/spectro/redux/v0/exposures/20191028/00022301/frame-r3-00022301.fits' frame_file_z3 = '/project/projectdirs/desi/spectro/redux/v0/exposures/20191028/00022301/frame-z3-00022301.fits' sframe_file_b3 = '/project/projectdirs/desi/spectro/redux/v0/exposures/20191028/00022301/sframe-b3-00022301.fits' sframe_file_r3 = '/project/projectdirs/desi/spectro/redux/v0/exposures/20191028/00022301/sframe-r3-00022301.fits' sframe_file_z3 = '/project/projectdirs/desi/spectro/redux/v0/exposures/20191028/00022301/sframe-z3-00022301.fits' frame_b3 = read_frame(frame_file_b3) frame_r3 = read_frame(frame_file_r3) frame_z3 = read_frame(frame_file_z3) sframe_b3 = read_frame(sframe_file_b3) sframe_r3 = read_frame(sframe_file_r3) sframe_z3 = read_frame(sframe_file_z3) flux_b3 = frame_b3.flux flux_r3 = frame_r3.flux flux_z3 = frame_z3.flux sflux_b3 = sframe_b3.flux sflux_r3 = sframe_r3.flux sflux_z3 = sframe_z3.flux fiber = frame_b3.fibermap["FIBER"] - 1500 x = np.zeros(fiber.size)
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None, output_dir=None, clobber=True): """ Generate a qaframe object from an input frame_file name (and night) Write QA to disk Will also make plots if directed Args: frame_file: str specprod_dir: str, optional qa_dir: str, optional -- Location of QA make_plots: bool, optional output_dir: str, optional Returns: """ import glob import os from desispec.io import read_frame from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.qa import qafile_from_framefile from desispec.io.frame import search_for_framefile from desispec.io.fiberflat import read_fiberflat from desispec.fiberflat import apply_fiberflat from desispec.qa import qa_plots from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration if '/' in frame_file: # If present, assume full path is used here pass else: # Find the frame file in the desispec hierarchy? frame_file = search_for_framefile(frame_file) # Load frame frame = read_frame(frame_file) frame_meta = frame.meta night = frame_meta['NIGHT'].strip() camera = frame_meta['CAMERA'].strip() expid = frame_meta['EXPID'] spectro = int(frame_meta['CAMERA'][-1]) # Filename qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir) qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame_meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: # Backwards compatibility fiberflat = read_fiberflat(fiberflat_fil) except FileNotFoundError: fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d') path, basen = os.path.split(fiberflat_fil) path, _ = os.path.split(path) fiberflat_fil = os.path.join(path, basen) fiberflat = read_fiberflat(fiberflat_fil) qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber) if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera) if not os.path.exists(fiberflat_fil): # Backwards compatibility (for now) dummy_fiberflat_fil = meta.findfile( 'fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) # This is dummy path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil)) fiberflat_files = glob.glob( os.path.join(path, '*', 'fiberflat-' + camera + '*.fits')) if len(fiberflat_files) == 0: path = path.replace('exposures', 'calib2d') path, _ = os.path.split(path) # Remove night fiberflat_files = glob.glob( os.path.join(path, 'fiberflat-' + camera + '*.fits')) # Sort and take the first (same as old pipeline) fiberflat_files.sort() fiberflat_fil = fiberflat_files[0] fiberflat = read_fiberflat(fiberflat_fil) apply_fiberflat(frame, fiberflat) # Load sky model and run try: skymodel = read_sky(sky_fil) except FileNotFoundError: warnings.warn( "Sky file {:s} not found. Skipping..".format(sky_fil)) else: qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber) if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) #qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, spectrograph=spectro) # try: # model_tuple=read_stdstar_models(stdstar_fil) # except FileNotFoundError: # warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil)) # else: flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: fluxcalib = read_flux_calibration(flux_fil) except FileNotFoundError: warnings.warn( "Flux file {:s} not found. Skipping..".format(flux_fil)) else: qaframe.run_qa( 'FLUXCALIB', (frame, fluxcalib)) # , model_tuple))#, indiv_stars)) if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir) qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple) # Write write_qa_frame(qafile, qaframe, verbose=True) return qaframe
def main(args): # imports import glob from desispec.io import findfile from desispec.io import get_exposures from desispec.io import get_files from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.qa.qa_plots import skysub_resid import copy import pdb # Log log = get_logger() log.info("starting") # Exposures? if args.expids is not None: expids = [int(iarg) for iarg in args.expids.split(',')] else: expids = 'all' # Nights? if args.nights is not None: gdnights = [iarg for iarg in args.nights.split(',')] else: gdnights = 'all' # Channels? if args.channels is not None: gdchannels = [iarg for iarg in args.channels.split(',')] else: gdchannels = 'all' # Sky dict sky_dict = dict(wave=[], skyflux=[], res=[], count=0) channel_dict = dict( b=copy.deepcopy(sky_dict), r=copy.deepcopy(sky_dict), z=copy.deepcopy(sky_dict), ) # Loop on nights path_nights = glob.glob(args.specprod_dir + '/exposures/*') nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights] for night in nights: if gdnights == 'all': pass else: if night not in gdnights: continue # Get em for exposure in get_exposures(night, specprod_dir=args.specprod_dir): # Check against input expids if expids == 'all': pass else: if exposure not in expids: continue # Get em frames_dict = get_files(filetype=str('cframe'), night=night, expid=exposure, specprod_dir=args.specprod_dir) for camera, cframe_fil in frames_dict.items(): channel = camera[0] # Check against input if gdchannels == 'all': pass else: if channel not in gdchannels: continue # Load frame log.info('Loading {:s}'.format(cframe_fil)) cframe = read_frame(cframe_fil) if cframe.meta['FLAVOR'] in ['flat', 'arc']: # Probably can't happen continue # Sky sky_file = findfile(str('sky'), night=night, camera=camera, expid=exposure, specprod_dir=args.specprod_dir) skymodel = read_sky(sky_file) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] flux = skymodel.flux[skyfibers] # Residuals tmp = np.outer(np.ones(flux.shape[0]), cframe.wave) # Append #from xastropy.xutils import xdebug as xdb #xdb.set_trace() channel_dict[channel]['wave'].append(tmp.flatten()) channel_dict[channel]['skyflux'].append( np.log10(np.maximum(flux.flatten(), 1e-1))) channel_dict[channel]['res'].append(res.flatten()) channel_dict[channel]['count'] += 1 # Figure for channel in ['b', 'r', 'z']: if channel_dict[channel]['count'] > 0: sky_wave = np.concatenate(channel_dict[channel]['wave']) sky_flux = np.concatenate(channel_dict[channel]['skyflux']) sky_res = np.concatenate(channel_dict[channel]['res']) # Plot skysub_resid(sky_wave, sky_flux, sky_res, outfile='tmp{:s}.png'.format(channel))
def main(args): log = get_logger() log.info("read frame") # read frame frame = read_frame(args.infile) log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat apply_fiberflat(frame, fiberflat) log.info("subtract sky") # read sky skymodel = read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) log.info("compute flux calibration") # read models model_flux, model_wave, model_fibers = read_stdstar_models(args.models) model_tuple = model_flux, model_wave, model_fibers # check that the model_fibers are actually standard stars fibermap = frame.fibermap model_fibers = model_fibers % 500 if np.any(fibermap['OBJTYPE'][model_fibers] != 'STD'): for i in model_fibers: log.error( "inconsistency with spectrum %d, OBJTYPE='%s' in fibermap" % (i, fibermap["OBJTYPE"][i])) sys.exit(12) #fluxcalib, indiv_stars = compute_flux_calibration(frame, model_wave, model_flux) fluxcalib = compute_flux_calibration(frame, model_wave, model_flux) # QA if (args.qafile is not None): log.info("performing fluxcalib QA") # Load qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR']) # Run qaframe.run_qa('FLUXCALIB', (frame, fluxcalib, model_tuple)) #, indiv_stars)) # Write if args.qafile is not None: write_qa_frame(args.qafile, qaframe) log.info("successfully wrote {:s}".format(args.qafile)) # Figure(s) if args.qafig is not None: qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib, model_tuple) # write result write_flux_calibration(args.outfile, fluxcalib, header=frame.meta) log.info("successfully wrote %s" % args.outfile)
def main(args=None): log = get_logger() if args is None: args = parse() if args.lin_step is not None and args.log10_step is not None: log.critical( "cannot have both linear and logarthmic bins :-), choose either --lin-step or --log10-step" ) return 12 if args.coadd_cameras and (args.lin_step is not None or args.log10_step is not None): log.critical( "cannot specify a new wavelength binning along with --coadd-cameras option" ) return 12 if len(args.infile) == 0: log.critical("You must specify input files") return 12 log.info("reading input ...") # inspect headers input_is_frames = False input_is_spectra = False for filename in args.infile: ifile = fitsio.FITS(filename) head = ifile[0].read_header() identified = False if "EXTNAME" in head and head["EXTNAME"] == "FLUX": print(filename, "is a frame") input_is_frames = True identified = True ifile.close() continue for hdu in ifile: head = hdu.read_header() if "EXTNAME" in head and head["EXTNAME"].find("_FLUX") >= 0: print(filename, "is a spectra") input_is_spectra = True identified = True break ifile.close() if not identified: log.error( "{} not identified as frame of spectra file".format(filename)) sys.exit(1) if input_is_frames and input_is_spectra: log.error("cannot combine input spectra and frames") sys.exit(1) if input_is_spectra: spectra = read_spectra(args.infile[0]) for filename in args.infile[1:]: log.info("append {}".format(filename)) spectra.update(read_spectra(filename)) else: # frames frames = dict() cameras = {} for filename in args.infile: frame = read_frame(filename) night = frame.meta['NIGHT'] expid = frame.meta['EXPID'] camera = frame.meta['CAMERA'] frames[(night, expid, camera)] = frame if args.coadd_cameras: cam, spec = camera[0], camera[1] # Keep a list of cameras (b,r,z) for each exposure + spec if (night, expid) not in cameras.keys(): cameras[(night, expid)] = {spec: [cam]} elif spec not in cameras[(night, expid)].keys(): cameras[(night, expid)][spec] = [cam] else: cameras[(night, expid)][spec].append(cam) if args.coadd_cameras: # If not all 3 cameras are available, remove the incomplete sets for (night, expid), camdict in cameras.items(): for spec, camlist in camdict.items(): log.info("Found {} for SP{} on NIGHT {} EXP {}".format( camlist, spec, night, expid)) if len(camlist) != 3 or np.any( np.sort(camlist) != np.array(['b', 'r', 'z'])): for cam in camlist: frames.pop((night, expid, cam + spec)) log.warning( "Removing {}{} from Night {} EXP {}".format( cam, spec, night, expid)) #import pdb #pdb.set_trace() spectra = frames2spectra(frames) #- hacks to make SpectraLite like a Spectra spectra.fibermap = Table(spectra.fibermap) del frames #- maybe free some memory if args.coadd_cameras: log.info("coadding cameras ...") spectra = coadd_cameras(spectra, cosmics_nsig=args.nsig) else: log.info("coadding ...") coadd(spectra, cosmics_nsig=args.nsig) if args.lin_step is not None: log.info("resampling ...") spectra = resample_spectra_lin_or_log(spectra, linear_step=args.lin_step, wave_min=args.wave_min, wave_max=args.wave_max, fast=args.fast, nproc=args.nproc) if args.log10_step is not None: log.info("resampling ...") spectra = resample_spectra_lin_or_log(spectra, log10_step=args.log10_step, wave_min=args.wave_min, wave_max=args.wave_max, fast=args.fast, nproc=args.nproc) #- Add input files to header if spectra.meta is None: spectra.meta = dict() for i, filename in enumerate(args.infile): spectra.meta['INFIL{:03d}'.format(i)] = os.path.basename(filename) log.info("writing {} ...".format(args.outfile)) write_spectra(args.outfile, spectra) log.info("done")
def main(args): """ finds the best models of all standard stars in the frame and normlize the model flux. Output is written to a file and will be called for calibration. """ log = get_logger() log.info("mag delta %s = %f (for the pre-selection of stellar models)" % (args.color, args.delta_color)) frames = {} flats = {} skies = {} spectrograph = None starfibers = None starindices = None fibermap = None # READ DATA ############################################ for filename in args.frames: log.info("reading %s" % filename) frame = io.read_frame(filename) header = fits.getheader(filename, 0) frame_fibermap = frame.fibermap frame_starindices = np.where(frame_fibermap["OBJTYPE"] == "STD")[0] camera = safe_read_key(header, "CAMERA").strip().lower() if spectrograph is None: spectrograph = frame.spectrograph fibermap = frame_fibermap starindices = frame_starindices starfibers = fibermap["FIBER"][starindices] elif spectrograph != frame.spectrograph: log.error("incompatible spectrographs %d != %d" % (spectrograph, frame.spectrograph)) raise ValueError("incompatible spectrographs %d != %d" % (spectrograph, frame.spectrograph)) elif starindices.size != frame_starindices.size or np.sum( starindices != frame_starindices) > 0: log.error("incompatible fibermap") raise ValueError("incompatible fibermap") if frames.has_key(camera): log.error( "cannot handle for now several frame of same camera (%s)" % camera) raise ValueError( "cannot handle for now several frame of same camera (%s)" % camera) frames[camera] = frame for filename in args.skymodels: log.info("reading %s" % filename) sky = io.read_sky(filename) header = fits.getheader(filename, 0) camera = safe_read_key(header, "CAMERA").strip().lower() # NEED TO ADD MORE CHECKS if skies.has_key(camera): log.error("cannot handle several skymodels of same camera (%s)" % camera) raise ValueError( "cannot handle several skymodels of same camera (%s)" % camera) skies[camera] = sky for filename in args.fiberflats: log.info("reading %s" % filename) header = fits.getheader(filename, 0) flat = io.read_fiberflat(filename) camera = safe_read_key(header, "CAMERA").strip().lower() # NEED TO ADD MORE CHECKS if flats.has_key(camera): log.error("cannot handle several flats of same camera (%s)" % camera) raise ValueError( "cannot handle several flats of same camera (%s)" % camera) flats[camera] = flat if starindices.size == 0: log.error("no STD star found in fibermap") raise ValueError("no STD star found in fibermap") log.info("found %d STD stars" % starindices.size) imaging_filters = fibermap["FILTER"][starindices] imaging_mags = fibermap["MAG"][starindices] log.warning( "NO MAG ERRORS IN FIBERMAP, I AM IGNORING MEASUREMENT ERRORS !!") log.warning( "NO EXTINCTION VALUES IN FIBERMAP, I AM IGNORING THIS FOR NOW !!") # DIVIDE FLAT AND SUBTRACT SKY , TRIM DATA ############################################ for cam in frames: if not skies.has_key(cam): log.warning("Missing sky for %s" % cam) frames.pop(cam) continue if not flats.has_key(cam): log.warning("Missing flat for %s" % cam) frames.pop(cam) continue frames[cam].flux = frames[cam].flux[starindices] frames[cam].ivar = frames[cam].ivar[starindices] frames[cam].ivar *= (frames[cam].mask[starindices] == 0) frames[cam].ivar *= (skies[cam].ivar[starindices] != 0) frames[cam].ivar *= (skies[cam].mask[starindices] == 0) frames[cam].ivar *= (flats[cam].ivar[starindices] != 0) frames[cam].ivar *= (flats[cam].mask[starindices] == 0) frames[cam].flux *= (frames[cam].ivar > 0) # just for clean plots for star in range(frames[cam].flux.shape[0]): ok = np.where((frames[cam].ivar[star] > 0) & (flats[cam].fiberflat[star] != 0))[0] if ok.size > 0: frames[cam].flux[star] = frames[cam].flux[star] / flats[ cam].fiberflat[star] - skies[cam].flux[star] nstars = starindices.size starindices = None # we don't need this anymore # READ MODELS ############################################ log.info("reading star models in %s" % args.starmodels) stdwave, stdflux, templateid, teff, logg, feh = io.read_stdstar_templates( args.starmodels) # COMPUTE MAGS OF MODELS FOR EACH STD STAR MAG ############################################ model_filters = [] for tmp in np.unique(imaging_filters): if len(tmp) > 0: # can be one empty entry model_filters.append(tmp) log.info("computing model mags %s" % model_filters) model_mags = np.zeros((stdflux.shape[0], len(model_filters))) fluxunits = 1e-17 * units.erg / units.s / units.cm**2 / units.Angstrom for index in range(len(model_filters)): filter_response = load_filter(model_filters[index]) for m in range(stdflux.shape[0]): model_mags[m, index] = filter_response.get_ab_magnitude( stdflux[m] * fluxunits, stdwave) log.info("done computing model mags") # LOOP ON STARS TO FIND BEST MODEL ############################################ bestModelIndex = np.arange(nstars) templateID = np.arange(nstars) chi2dof = np.zeros((nstars)) redshift = np.zeros((nstars)) normflux = [] for star in range(nstars): log.info("finding best model for observed star #%d" % star) # np.array of wave,flux,ivar,resol wave = {} flux = {} ivar = {} resolution_data = {} for camera in frames: band = camera[0] wave[band] = frames[camera].wave flux[band] = frames[camera].flux[star] ivar[band] = frames[camera].ivar[star] resolution_data[band] = frames[camera].resolution_data[star] # preselec models based on magnitudes # compute star color index1, index2 = get_color_filter_indices(imaging_filters[star], args.color) if index1 < 0 or index2 < 0: log.error("cannot compute '%s' color from %s" % (color_name, filters)) filter1 = imaging_filters[star][index1] filter2 = imaging_filters[star][index2] star_color = imaging_mags[star][index1] - imaging_mags[star][index2] # compute models color model_index1 = -1 model_index2 = -1 for i, fname in enumerate(model_filters): if fname == filter1: model_index1 = i elif fname == filter2: model_index2 = i if model_index1 < 0 or model_index2 < 0: log.error("cannot compute '%s' model color from %s" % (color_name, filters)) model_colors = model_mags[:, model_index1] - model_mags[:, model_index2] # selection selection = np.where( np.abs(model_colors - star_color) < args.delta_color)[0] log.info( "star#%d fiber #%d, %s = %s-%s = %f, number of pre-selected models = %d/%d" % (star, starfibers[star], args.color, filter1, filter2, star_color, selection.size, stdflux.shape[0])) index_in_selection, redshift[star], chi2dof[star] = match_templates( wave, flux, ivar, resolution_data, stdwave, stdflux[selection], teff[selection], logg[selection], feh[selection], ncpu=args.ncpu, z_max=args.z_max, z_res=args.z_res) bestModelIndex[star] = selection[index_in_selection] log.info( 'Star Fiber: {0}; TemplateID: {1}; Redshift: {2}; Chisq/dof: {3}'. format(starfibers[star], bestModelIndex[star], redshift[star], chi2dof[star])) # Apply redshift to original spectrum at full resolution tmp = np.interp(stdwave, stdwave / (1 + redshift[star]), stdflux[bestModelIndex[star]]) # Normalize the best model using reported magnitude normalizedflux = normalize_templates(stdwave, tmp, imaging_mags[star], imaging_filters[star]) normflux.append(normalizedflux) # Now write the normalized flux for all best models to a file normflux = np.array(normflux) data = {} data['BESTMODEL'] = bestModelIndex data['TEMPLATEID'] = bestModelIndex # IS THAT IT? data['CHI2DOF'] = chi2dof data['REDSHIFT'] = redshift norm_model_file = args.outfile io.write_stdstar_models(args.outfile, normflux, stdwave, starfibers, data)
def main(args) : # imports import glob from desispec.io import findfile from desispec.io import get_exposures from desispec.io import get_files from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.qa.qa_plots import skysub_resid import copy import pdb # Log log=get_logger() log.info("starting") # Exposures? if args.expids is not None: expids = [int(iarg) for iarg in args.expids.split(',')] else: expids = 'all' # Nights? if args.nights is not None: gdnights = [iarg for iarg in args.nights.split(',')] else: gdnights = 'all' # Channels? if args.channels is not None: gdchannels = [iarg for iarg in args.channels.split(',')] else: gdchannels = 'all' # Sky dict sky_dict = dict(wave=[], skyflux=[], res=[], count=0) channel_dict = dict(b=copy.deepcopy(sky_dict), r=copy.deepcopy(sky_dict), z=copy.deepcopy(sky_dict), ) # Loop on nights path_nights = glob.glob(args.specprod_dir+'/exposures/*') nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights] for night in nights: if gdnights == 'all': pass else: if night not in gdnights: continue # Get em for exposure in get_exposures(night, specprod_dir = args.specprod_dir): # Check against input expids if expids == 'all': pass else: if exposure not in expids: continue # Get em frames_dict = get_files(filetype=str('cframe'), night=night, expid=exposure, specprod_dir=args.specprod_dir) for camera, cframe_fil in frames_dict.items(): channel = camera[0] # Check against input if gdchannels == 'all': pass else: if channel not in gdchannels: continue # Load frame log.info('Loading {:s}'.format(cframe_fil)) cframe = read_frame(cframe_fil) if cframe.meta['FLAVOR'] in ['flat','arc']: # Probably can't happen continue # Sky sky_file = findfile(str('sky'), night=night, camera=camera, expid=exposure, specprod_dir=args.specprod_dir) skymodel = read_sky(sky_file) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] flux = skymodel.flux[skyfibers] # Residuals tmp = np.outer(np.ones(flux.shape[0]), cframe.wave) # Append #from xastropy.xutils import xdebug as xdb #xdb.set_trace() channel_dict[channel]['wave'].append(tmp.flatten()) channel_dict[channel]['skyflux'].append( np.log10(np.maximum(flux.flatten(),1e-1))) channel_dict[channel]['res'].append(res.flatten()) channel_dict[channel]['count'] += 1 # Figure for channel in ['b', 'r', 'z']: if channel_dict[channel]['count'] > 0: sky_wave = np.concatenate(channel_dict[channel]['wave']) sky_flux = np.concatenate(channel_dict[channel]['skyflux']) sky_res = np.concatenate(channel_dict[channel]['res']) # Plot skysub_resid(sky_wave, sky_flux, sky_res, outfile='tmp{:s}.png'.format(channel))
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None, output_dir=None, clobber=True): """ Generate a qaframe object from an input frame_file name (and night) Write QA to disk Will also make plots if directed Args: frame_file: str specprod_dir: str, optional qa_dir: str, optional -- Location of QA make_plots: bool, optional output_dir: str, optional Returns: """ import glob import os from desispec.io import read_frame from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.qa import qafile_from_framefile from desispec.io.frame import search_for_framefile from desispec.io.fiberflat import read_fiberflat from desispec.fiberflat import apply_fiberflat from desispec.qa import qa_plots from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots_ql if '/' in frame_file: # If present, assume full path is used here pass else: # Find the frame file in the desispec hierarchy? frame_file = search_for_framefile(frame_file) # Load frame frame = read_frame(frame_file) frame_meta = frame.meta night = frame_meta['NIGHT'].strip() camera = frame_meta['CAMERA'].strip() expid = frame_meta['EXPID'] spectro = int(frame_meta['CAMERA'][-1]) # Filename qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir) if os.path.isfile(qafile) and (not clobber): write = False else: write = True qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame_meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: # Backwards compatibility fiberflat = read_fiberflat(fiberflat_fil) except FileNotFoundError: fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d') path, basen = os.path.split(fiberflat_fil) path, _ = os.path.split(path) fiberflat_fil = os.path.join(path, basen) fiberflat = read_fiberflat(fiberflat_fil) if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber): write = True if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid, qaprod_dir=qaprod_dir, specprod_dir=specprod_dir, outdir=output_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera) if not os.path.exists(fiberflat_fil): # Backwards compatibility (for now) dummy_fiberflat_fil = meta.findfile( 'fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) # This is dummy path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil)) fiberflat_files = glob.glob( os.path.join(path, '*', 'fiberflat-' + camera + '*.fits')) if len(fiberflat_files) == 0: path = path.replace('exposures', 'calib2d') path, _ = os.path.split(path) # Remove night fiberflat_files = glob.glob( os.path.join(path, 'fiberflat-' + camera + '*.fits')) # Sort and take the first (same as old pipeline) fiberflat_files.sort() fiberflat_fil = fiberflat_files[0] fiberflat = read_fiberflat(fiberflat_fil) apply_fiberflat(frame, fiberflat) # Load sky model and run try: skymodel = read_sky(sky_fil) except FileNotFoundError: warnings.warn( "Sky file {:s} not found. Skipping..".format(sky_fil)) else: if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber): write = True if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) #qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe) # S/N QA on cframe if qatype == 'qa_data': # cframe cframe_file = frame_file.replace('frame-', 'cframe-') cframe = read_frame(cframe_file) if qaframe.run_qa('S2N', (cframe, ), clobber=clobber): write = True # Figure? if make_plots: s2n_dict = copy.deepcopy(qaframe.qa_data['S2N']) qafig = meta.findfile('qa_s2n_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) #badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist() #sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE') coeff = s2n_dict['METRICS']['FITCOEFF_TGT'] #[sci_idx] # Add an item or two for the QL method s2n_dict['CAMERA'] = camera s2n_dict['EXPID'] = expid s2n_dict['PANAME'] = 'SNRFit' s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA'] s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC'] objlist = s2n_dict['METRICS']['OBJLIST'] # Deal with YAML list instead of ndarray s2n_dict['METRICS']['MEDIAN_SNR'] = np.array( s2n_dict['METRICS']['MEDIAN_SNR']) # Generate if (not os.path.isfile(qafig)) or clobber: qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist, [[]] * len(objlist), coeff) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, spectrograph=spectro) # try: # model_tuple=read_stdstar_models(stdstar_fil) # except FileNotFoundError: # warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil)) # else: flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: fluxcalib = read_flux_calibration(flux_fil) except FileNotFoundError: warnings.warn( "Flux file {:s} not found. Skipping..".format(flux_fil)) else: if qaframe.run_qa( 'FLUXCALIB', (frame, fluxcalib), clobber=clobber): # , model_tuple))#, indiv_stars)) write = True if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple) # Write if write: write_qa_frame(qafile, qaframe, verbose=True) return qaframe
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None, output_dir=None, clobber=True): """ Generate a qaframe object from an input frame_file name (and night) Write QA to disk Will also make plots if directed Args: frame_file: str specprod_dir: str, optional qa_dir: str, optional -- Location of QA make_plots: bool, optional output_dir: str, optional Returns: """ import glob import os from desispec.io import read_frame from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.qa import qafile_from_framefile from desispec.io.frame import search_for_framefile from desispec.io.fiberflat import read_fiberflat from desispec.fiberflat import apply_fiberflat from desispec.qa import qa_plots from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots_ql if '/' in frame_file: # If present, assume full path is used here pass else: # Find the frame file in the desispec hierarchy? frame_file = search_for_framefile(frame_file) # Load frame frame = read_frame(frame_file) frame_meta = frame.meta night = frame_meta['NIGHT'].strip() camera = frame_meta['CAMERA'].strip() expid = frame_meta['EXPID'] spectro = int(frame_meta['CAMERA'][-1]) # Filename qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir) if os.path.isfile(qafile) and (not clobber): write = False else: write = True qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame_meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: # Backwards compatibility fiberflat = read_fiberflat(fiberflat_fil) except FileNotFoundError: fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d') path, basen = os.path.split(fiberflat_fil) path,_ = os.path.split(path) fiberflat_fil = os.path.join(path, basen) fiberflat = read_fiberflat(fiberflat_fil) if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber): write = True if make_plots: # Do it qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid, qaprod_dir=qaprod_dir, specprod_dir=specprod_dir, outdir=output_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera) if not os.path.exists(fiberflat_fil): # Backwards compatibility (for now) dummy_fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) # This is dummy path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil)) fiberflat_files = glob.glob(os.path.join(path,'*','fiberflat-'+camera+'*.fits')) if len(fiberflat_files) == 0: path = path.replace('exposures', 'calib2d') path,_ = os.path.split(path) # Remove night fiberflat_files = glob.glob(os.path.join(path,'fiberflat-'+camera+'*.fits')) # Sort and take the first (same as old pipeline) fiberflat_files.sort() fiberflat_fil = fiberflat_files[0] fiberflat = read_fiberflat(fiberflat_fil) apply_fiberflat(frame, fiberflat) # Load sky model and run try: skymodel = read_sky(sky_fil) except FileNotFoundError: warnings.warn("Sky file {:s} not found. Skipping..".format(sky_fil)) else: if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber): write=True if make_plots: qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) #qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe) # S/N QA on cframe if qatype == 'qa_data': # cframe cframe_file = frame_file.replace('frame-', 'cframe-') cframe = read_frame(cframe_file) if qaframe.run_qa('S2N', (cframe,), clobber=clobber): write=True # Figure? if make_plots: s2n_dict = copy.deepcopy(qaframe.qa_data['S2N']) qafig = meta.findfile('qa_s2n_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) #badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist() #sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE') coeff = s2n_dict['METRICS']['FITCOEFF_TGT']#[sci_idx] # Add an item or two for the QL method s2n_dict['CAMERA'] = camera s2n_dict['EXPID'] = expid s2n_dict['PANAME'] = 'SNRFit' s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA'] s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC'] objlist = s2n_dict['METRICS']['OBJLIST'] # Deal with YAML list instead of ndarray s2n_dict['METRICS']['MEDIAN_SNR'] = np.array(s2n_dict['METRICS']['MEDIAN_SNR']) # Generate if (not os.path.isfile(qafig)) or clobber: qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist, [[]]*len(objlist), coeff) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, spectrograph=spectro) # try: # model_tuple=read_stdstar_models(stdstar_fil) # except FileNotFoundError: # warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil)) # else: flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) try: fluxcalib = read_flux_calibration(flux_fil) except FileNotFoundError: warnings.warn("Flux file {:s} not found. Skipping..".format(flux_fil)) else: if qaframe.run_qa('FLUXCALIB', (frame, fluxcalib), clobber=clobber): # , model_tuple))#, indiv_stars)) write = True if make_plots: qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir) if (not os.path.isfile(qafig)) or clobber: qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple) # Write if write: write_qa_frame(qafile, qaframe, verbose=True) return qaframe
def main() : """ finds the best models of all standard stars in the frame and normlize the model flux. Output is written to a file and will be called for calibration. """ parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--fiberflatexpid', type = int, help = 'fiberflat exposure ID') parser.add_argument('--fibermap', type = str, help = 'path of fibermap file') parser.add_argument('--models', type = str, help = 'path of spectro-photometric stellar spectra fits') parser.add_argument('--spectrograph', type = int, default = 0, help = 'spectrograph number, can go 0-9') parser.add_argument('--outfile', type = str, help = 'output file for normalized stdstar model flux') args = parser.parse_args() log = get_logger() # Call necessary environment variables. No need if add argument to give full file path. if 'DESI_SPECTRO_REDUX' not in os.environ: raise RuntimeError('Set environment DESI_SPECTRO_REDUX. It is needed to read the needed datafiles') DESI_SPECTRO_REDUX=os.environ['DESI_SPECTRO_REDUX'] PRODNAME=os.environ['PRODNAME'] if 'DESISIM' not in os.environ: raise RuntimeError('Set environment DESISIM. It will be neede to read the filter transmission files for calibration') DESISIM=os.environ['DESISIM'] # to read the filter transmission files if args.fibermap is None or args.models is None or \ args.spectrograph is None or args.outfile is None or \ args.fiberflatexpid is None: log.critical('Missing a required argument') parser.print_help() sys.exit(12) # read Standard Stars from the fibermap file # returns the Fiber id, filter names and mags for the standard stars fiber_tbdata,fiber_header=io.read_fibermap(args.fibermap, header=True) #- Trim to just fibers on this spectrograph ii = (500*args.spectrograph <= fiber_tbdata["FIBER"]) ii &= (fiber_tbdata["FIBER"] < 500*(args.spectrograph+1)) fiber_tbdata = fiber_tbdata[ii] #- Get info for the standard stars refStarIdx=np.where(fiber_tbdata["OBJTYPE"]=="STD") refFibers=fiber_tbdata["FIBER"][refStarIdx] refFilters=fiber_tbdata["FILTER"][refStarIdx] refMags=fiber_tbdata["MAG"] fibers={"FIBER":refFibers,"FILTER":refFilters,"MAG":refMags} NIGHT=fiber_header['NIGHT'] EXPID=fiber_header['EXPID'] filters=fibers["FILTER"] if 'DESISIM' not in os.environ: raise RuntimeError('Set environment DESISIM. Can not find filter response files') basepath=DESISIM+"/data/" #now load all the skyfiles, framefiles, fiberflatfiles etc # all three channels files are simultaneously treated for model fitting skyfile={} framefile={} fiberflatfile={} for i in ["b","r","z"]: camera = i+str(args.spectrograph) skyfile[i] = io.findfile('sky', NIGHT, EXPID, camera) framefile[i] = io.findfile('frame', NIGHT, EXPID, camera) fiberflatfile[i] = io.findfile('fiberflat', NIGHT, args.fiberflatexpid, camera) #Read Frames, Flats and Sky files frameFlux={} frameIvar={} frameWave={} frameResolution={} framehdr={} fiberFlat={} ivarFlat={} maskFlat={} meanspecFlat={} waveFlat={} headerFlat={} sky={} skyivar={} skymask={} skywave={} skyhdr={} for i in ["b","r","z"]: #arg=(night,expid,'%s%s'%(i,spectrograph)) #- minimal code change for refactored I/O, while not taking advantage of simplified structure frame = io.read_frame(framefile[i]) frameFlux[i] = frame.flux frameIvar[i] = frame.ivar frameWave[i] = frame.wave frameResolution[i] = frame.resolution_data framehdr[i] = frame.header ff = io.read_fiberflat(fiberflatfile[i]) fiberFlat[i] = ff.fiberflat ivarFlat[i] = ff.ivar maskFlat[i] = ff.mask meanspecFlat[i] = ff.meanspec waveFlat[i] = ff.wave headerFlat[i] = ff.header skymodel = io.read_sky(skyfile[i]) sky[i] = skymodel.flux skyivar[i] = skymodel.ivar skymask[i] = skymodel.mask skywave[i] = skymodel.wave skyhdr[i] = skymodel.header # Convolve Sky with Detector Resolution, so as to subtract from data. Convolve for all 500 specs. Subtracting sky this way should be equivalent to sky_subtract convolvedsky={"b":sky["b"], "r":sky["r"], "z":sky["z"]} # Read the standard Star data and divide by flat and subtract sky stars=[] ivars=[] for i in fibers["FIBER"]: #flat and sky should have same wavelength binning as data, otherwise should be rebinned. stars.append((i,{"b":[frameFlux["b"][i]/fiberFlat["b"][i]-convolvedsky["b"][i],frameWave["b"]], "r":[frameFlux["r"][i]/fiberFlat["r"][i]-convolvedsky["r"][i],frameWave["r"]], "z":[frameFlux["z"][i]/fiberFlat["z"][i]-convolvedsky["z"][i],frameWave]},fibers["MAG"][i])) ivars.append((i,{"b":[frameIvar["b"][i]],"r":[frameIvar["r"][i,:]],"z":[frameIvar["z"][i,:]]})) stdwave,stdflux,templateid=io.read_stdstar_templates(args.models) #- Trim standard star wavelengths to just the range we need minwave = min([min(w) for w in frameWave.values()]) maxwave = max([max(w) for w in frameWave.values()]) ii = (minwave-10 < stdwave) & (stdwave < maxwave+10) stdwave = stdwave[ii] stdflux = stdflux[:, ii] log.info('Number of Standard Stars in this frame: {0:d}'.format(len(stars))) if len(stars) == 0: log.critical("No standard stars! Exiting") sys.exit(1) # Now for each star, find the best model and normalize. normflux=[] bestModelIndex=np.arange(len(stars)) templateID=np.arange(len(stars)) chi2dof=np.zeros(len(stars)) #- TODO: don't use 'l' as a variable name. Can look like a '1' for k,l in enumerate(stars): log.info("checking best model for star {0}".format(l[0])) starindex=l[0] mags=l[2] filters=fibers["FILTER"][k] rflux=stars[k][1]["r"][0] bflux=stars[k][1]["b"][0] zflux=stars[k][1]["z"][0] flux={"b":bflux,"r":rflux,"z":zflux} #print ivars rivar=ivars[k][1]["r"][0] bivar=ivars[k][1]["b"][0] zivar=ivars[k][1]["z"][0] ivar={"b":bivar,"r":rivar,"z":zivar} resol_star={"r":frameResolution["r"][l[0]],"b":frameResolution["b"][l[0]],"z":frameResolution["z"][l[0]]} # Now find the best Model bestModelIndex[k],bestmodelWave,bestModelFlux,chi2dof[k]=match_templates(frameWave,flux,ivar,resol_star,stdwave,stdflux) log.info('Star Fiber: {0}; Best Model Fiber: {1}; TemplateID: {2}; Chisq/dof: {3}'.format(l[0],bestModelIndex[k],templateid[bestModelIndex[k]],chi2dof[k])) # Normalize the best model using reported magnitude modelwave,normalizedflux=normalize_templates(stdwave,stdflux[bestModelIndex[k]],mags,filters,basepath) normflux.append(normalizedflux) # Now write the normalized flux for all best models to a file normflux=np.array(normflux) stdfibers=fibers["FIBER"] data={} data['BESTMODEL']=bestModelIndex data['CHI2DOF']=chi2dof data['TEMPLATEID']=templateid[bestModelIndex] norm_model_file=args.outfile io.write_stdstar_model(norm_model_file,normflux,stdwave,stdfibers,data)
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--infile', type=str, default=None, required=True, help='path of DESI exposure frame fits file') parser.add_argument('--fiberflat', type=str, default=None, help='path of DESI fiberflat fits file') parser.add_argument('--sky', type=str, default=None, help='path of DESI sky fits file') parser.add_argument('--calib', type=str, default=None, help='path of DESI calibration fits file') parser.add_argument('--outfile', type=str, default=None, required=True, help='path of DESI sky fits file') # add calibration here when exists args = parser.parse_args() log = get_logger() if (args.fiberflat is None) and (args.sky is None) and (args.calib is None): log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?') sys.exit(12) frame = read_frame(args.infile) if args.fiberflat != None: log.info("apply fiberflat") # read fiberflat fiberflat = read_fiberflat(args.fiberflat) # apply fiberflat to sky fibers apply_fiberflat(frame, fiberflat) if args.sky != None: log.info("subtract sky") # read sky skymodel = read_sky(args.sky) # subtract sky subtract_sky(frame, skymodel) if args.calib != None: log.info("calibrate") # read calibration fluxcalib = read_flux_calibration(args.calib) # apply calibration apply_flux_calibration(frame, fluxcalib) # save output write_frame(args.outfile, frame) log.info("successfully wrote %s" % args.outfile)
def load_all_s2n_values(nights, channel, sub_exposures=None): """ Calculate S/N values for a set of spectra from an input list of nights Args: nights: list channel: str ('b','r','z') sub_exposures: Returns: fdict: dict Contains all the S/N info for all nights in the given channel """ fdict = dict(waves=[], s2n=[], fluxes=[], exptime=[], OII=[], objtype=[]) for night in nights: if sub_exposures is not None: exposures = sub_exposures else: exposures = get_exposures(night)#, raw=True) for exposure in exposures: fibermap_path = findfile(filetype='fibermap', night=night, expid=exposure) fibermap_data = read_fibermap(fibermap_path) flavor = fibermap_data.meta['FLAVOR'] if flavor.lower() in ('arc', 'flat', 'bias'): log.debug('Skipping calibration {} exposure {:08d}'.format(flavor, exposure)) continue # Load simspec simspec_file = fibermap_path.replace('fibermap', 'simspec') log.debug('Getting truth from {}'.format(simspec_file)) sps_hdu = fits.open(simspec_file) sps_tab = Table(sps_hdu['TRUTH'].data,masked=True) #- Get OIIFLUX from separate HDU and join if ('OIIFLUX' not in sps_tab.colnames) and ('TRUTH_ELG' in sps_hdu): elg_truth = Table(sps_hdu['TRUTH_ELG'].data) sps_tab = join(sps_tab, elg_truth['TARGETID', 'OIIFLUX'], keys='TARGETID', join_type='left') else: sps_tab['OIIFLUX'] = 0.0 sps_hdu.close() #objs = sps_tab['TEMPLATETYPE'] == objtype #if np.sum(objs) == 0: # continue # Load spectra (flux or not fluxed; should not matter) for ii in range(10): camera = channel+str(ii) cframe_path = findfile(filetype='cframe', night=night, expid=exposure, camera=camera) try: log.debug('Reading from {}'.format(cframe_path)) cframe = read_frame(cframe_path) except (IOError, OSError): log.warn("Cannot find file: {:s}".format(cframe_path)) continue # Calculate S/N per Ang dwave = cframe.wave - np.roll(cframe.wave,1) dwave[0] = dwave[1] # Calculate s2n = cframe.flux * np.sqrt(cframe.ivar) / np.sqrt(dwave) #s2n = cframe.flux[iobjs,:] * np.sqrt(cframe.ivar[iobjs,:]) / np.sqrt(dwave) # Save fdict['objtype'].append(sps_tab['TEMPLATETYPE'].data[cframe.fibers]) fdict['waves'].append(cframe.wave) fdict['s2n'].append(s2n) fdict['fluxes'].append(sps_tab['MAG'].data[cframe.fibers]) fdict['OII'].append(sps_tab['OIIFLUX'].data[cframe.fibers]) fdict['exptime'].append(cframe.meta['EXPTIME']) # Return return fdict
ff_file_b3='/project/projectdirs/desi/spectro/desi_spectro_calib/trunk/spec/sp3/fiberflat-sm4-b-20191022.fits' ff_file_r3='/project/projectdirs/desi/spectro/desi_spectro_calib/trunk/spec/sp3/fiberflat-sm4-r-20191022.fits' ff_file_z3='/project/projectdirs/desi/spectro/desi_spectro_calib/trunk/spec/sp3/fiberflat-sm4-z-20191022.fits' fff_b3=fitsio.FITS(ff_file_b3) fff_r3=fitsio.FITS(ff_file_r3) fff_z3=fitsio.FITS(ff_file_z3) ff_b3=fff_b3['FIBERFLAT'][:,:] ff_r3=fff_r3['FIBERFLAT'][:,:] ff_z3=fff_z3['FIBERFLAT'][:,:] frame_b3=read_frame('/project/projectdirs/desi/spectro/redux/v0/exposures/20191028/00022301/frame-b3-00022301.fits') fiber=frame_b3.fibermap["FIBER"]-1500 x=np.zeros(fiber.size) y=np.zeros(fiber.size) for j,fib in enumerate(fiber) : x[j]=table[xk][dico[fib]] y[j]=table[yk][dico[fib]] ff_b3_median=np.median(ff_b3,axis=1) ff_r3_median=np.median(ff_r3,axis=1) ff_z3_median=np.median(ff_z3,axis=1) s1=30 contrast=2 plt.figure(0,figsize=(12,10))
def make_frameqa(self, make_plots=False, clobber=True): """ Work through the Production and make QA for all frames Parameters: make_plots: bool, optional Remake the plots too? clobber: bool, optional Returns: """ # imports from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame from desispec.io.fiberflat import read_fiberflat from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration from desispec.qa import qa_plots from desispec.io.fluxcalibration import read_stdstar_models # Loop on nights path_nights = glob.glob(self.specprod_dir + '/exposures/*') nights = [ipathn[ipathn.rfind('/') + 1:] for ipathn in path_nights] for night in nights: for exposure in get_exposures(night, specprod_dir=self.specprod_dir): # Object only?? frames_dict = get_files(filetype=str('frame'), night=night, expid=exposure, specprod_dir=self.specprod_dir) for camera, frame_fil in frames_dict.items(): # Load frame frame = read_frame(frame_fil) spectro = int(frame.meta['CAMERA'][-1]) if frame.meta['FLAVOR'] in ['flat', 'arc']: qatype = 'qa_calib' else: qatype = 'qa_data' qafile = meta.findfile(qatype, night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) if (not clobber) & os.path.isfile(qafile): log.info( "qafile={:s} exists. Not over-writing. Consider clobber=True" .format(qafile)) continue # Load qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) # Flat QA if frame.meta['FLAVOR'] in ['flat']: fiberflat_fil = meta.findfile( 'fiberflat', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fiberflat = read_fiberflat(fiberflat_fil) qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber) if make_plots: # Do it qafig = meta.findfile( 'qa_flat_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) # SkySub QA if qatype == 'qa_data': sky_fil = meta.findfile('sky', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) skymodel = read_sky(sky_fil) qaframe.run_qa('SKYSUB', (frame, skymodel)) if make_plots: qafig = meta.findfile( 'qa_sky_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) # FluxCalib QA if qatype == 'qa_data': # Standard stars stdstar_fil = meta.findfile( 'stdstars', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir, spectrograph=spectro) model_tuple = read_stdstar_models(stdstar_fil) flux_fil = meta.findfile( 'calib', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) fluxcalib = read_flux_calibration(flux_fil) qaframe.run_qa( 'FLUXCALIB', (frame, fluxcalib, model_tuple)) #, indiv_stars)) if make_plots: qafig = meta.findfile( 'qa_flux_fig', night=night, camera=camera, expid=exposure, specprod_dir=self.specprod_dir) qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib, model_tuple) # Write write_qa_frame(qafile, qaframe)
def get_skyres(cframes, sub_sky=False, flatten=True): """ Args: cframes: str or list Single cframe or a list of them sub_sky: bool, optional Subtract the sky? This should probably not be done flatten: bool, optional Return a flat, 1D array for each variable combine: bool, optional combine the individual sky fibers? Median 'smash' Returns: wave : ndarray flux : ndarray res : ndarray ivar : ndarray """ from desispec.io import read_frame from desispec.io.sky import read_sky from desispec.sky import subtract_sky if isinstance(cframes, list): all_wave, all_flux, all_res, all_ivar = [], [], [], [] for cframe_file in cframes: wave, flux, res, ivar = get_skyres(cframe_file, flatten=flatten) # Save all_wave.append(wave) all_flux.append(flux) all_res.append(res) all_ivar.append(ivar) # Concatenate -- Shape is preserved (nfibers, npix) twave = np.concatenate(all_wave) tflux = np.concatenate(all_flux) tres = np.concatenate(all_res) tivar = np.concatenate(all_ivar) # Return return twave, tflux, tres, tivar cframe = read_frame(cframes, skip_resolution=True) if cframe.meta['FLAVOR'] in ['flat', 'arc']: raise ValueError("Bad flavor for exposure: {:s}".format(cframes)) # Sky sky_file = cframes.replace('cframe', 'sky') skymodel = read_sky(sky_file) if sub_sky: subtract_sky(cframe, skymodel) # Resid skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0] res = cframe.flux[skyfibers] # Flux calibrated ivar = cframe.ivar[skyfibers] # Flux calibrated flux = skymodel.flux[skyfibers] # Residuals; not flux calibrated! wave = np.outer(np.ones(flux.shape[0]), cframe.wave) # Combine? ''' if combine: res = np.median(res, axis=0) ivar = np.median(ivar, axis=0) flux = np.median(flux, axis=0) wave = np.median(wave, axis=0) ''' # Return if flatten: return wave.flatten(), flux.flatten(), res.flatten(), ivar.flatten() else: return wave, flux, res, ivar