def test_subtract_sky_with_gradient_using_compute_non_uniform_sky(self):
spectra = self._get_spectra(with_gradient=True)
sky = compute_sky(spectra,angular_variation_deg=1,chromatic_variation_deg=-1,add_variance=self.add_variance)
subtract_sky(spectra, sky)
#- allow some slop in the sky subtraction
self.assertTrue(np.allclose(spectra.flux, 0, rtol=1e-3, atol=1e-3))
def test_subtract_sky_with_gradient_using_compute_polynomial_times_sky(self):
spectra = self._get_spectra(with_gradient=True)
sky = compute_sky(spectra,angular_variation_deg=1,chromatic_variation_deg=1,add_variance=self.add_variance)
subtract_sky(spectra, sky)
#- allow some slop in the sky subtraction
self.assertTrue(np.allclose(spectra.flux, 0, rtol=1e-3, atol=1e-3)) # this is not exact, it's an iterative fit
def test_subtract_sky_with_gradient_using_compute_non_uniform_sky(self):
spectra = self._get_spectra(with_gradient=True)
sky = compute_sky(spectra,angular_variation_deg=1,chromatic_variation_deg=-1,add_variance=self.add_variance)
subtract_sky(spectra, sky)
#- allow some slop in the sky subtraction
self.assertTrue(np.allclose(spectra.flux, 0, rtol=1e-3, atol=1e-3))
def run_sky_subtraction(night, expid, cameras, basedir, nsky_list, reps=None):
'''Runs sky subtraction with new sky models and frame files.
Args:
night: YYYYMMDD (float)
expid: exposure id without padding zeros (float)
cameras: list of cameras corresponding to frame and sky files in given directory, example ['r3', 'z3', 'b3'] (list or array)
basedir: where to look for frame files
nsky_list: list with different numbers of fibers frame files and sky files were generated with.
Options:
rep: number of different frame/sky files for each camera and nsky combination. Default is 5
'''
if reps == None:
reps = 5
for cam in cameras:
framefile = desispec.io.findfile('frame', night, expid, camera=cam)
header = fitsio.read_header(framefile)
fiberflatfile = findcalibfile([
header,
], 'FIBERFLAT')
fiberflat = desispec.io.read_fiberflat(fiberflatfile)
for N in range(reps):
for n in nsky_list:
newframefile = basedir + '/frame-{}-{:08d}-{}-{}.fits'.format(
cam, expid, n, N)
skyfile = basedir + '/sky-{}-{:08d}-{}-{}.fits'.format(
cam, expid, n, N)
sframe = desispec.io.read_frame(newframefile)
sky = desispec.io.read_sky(skyfile)
apply_fiberflat(sframe, fiberflat)
subtract_sky(sframe, sky)
sframefile = basedir + '/sframe-{}-{:08d}-{}-{}.fits'.format(
cam, expid, n, N)
desispec.io.write_frame(sframefile, sframe)
def test_subtract_sky_with_gradient_using_compute_polynomial_times_sky(self):
spectra = self._get_spectra(with_gradient=True)
sky = compute_sky(spectra,angular_variation_deg=1,chromatic_variation_deg=1,add_variance=self.add_variance)
subtract_sky(spectra, sky)
#- allow some slop in the sky subtraction
self.assertTrue(np.allclose(spectra.flux, 0, rtol=1e-3, atol=1.)) # this is not exact, it's an iterative fit
def get_skyres(cframes, sub_sky=False):
from desispec.io import read_frame
from desispec.io.sky import read_sky
from desispec.sky import subtract_sky
if isinstance(cframes, list):
all_wave, all_flux, all_res, all_ivar = [], [], [], []
for cframe_file in cframes:
wave, flux, res, ivar = get_skyres(cframe_file)
# Concatenate and return
all_wave.append(wave)
all_flux.append(flux)
all_res.append(res)
all_ivar.append(ivar)
return np.concatenate(all_wave), np.concatenate(all_flux), \
np.concatenate(all_res), np.concatenate(all_ivar)
cframe = read_frame(cframes)
if cframe.meta['FLAVOR'] in ['flat', 'arc']:
raise ValueError("Bad flavor for exposure: {:s}".format(cframes))
# Sky
sky_file = cframes.replace('cframe', 'sky')
skymodel = read_sky(sky_file)
if sub_sky:
subtract_sky(cframe, skymodel)
# Resid
skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0]
res = cframe.flux[skyfibers]
ivar = cframe.ivar[skyfibers]
flux = skymodel.flux[skyfibers] # Residuals
wave = np.outer(np.ones(flux.shape[0]), cframe.wave)
# Return
return wave.flatten(), flux.flatten(), res.flatten(), ivar.flatten()
def main(args) :
log=get_logger()
log.info("read frame")
# read frame
frame = read_frame(args.infile)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel=read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux,model_wave,model_fibers=read_stdstar_models(args.models)
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
model_fibers = model_fibers%500
if np.any(fibermap['OBJTYPE'][model_fibers] != 'STD'):
for i in model_fibers:
log.error("inconsistency with spectrum %d, OBJTYPE='%s' in fibermap"%(i,fibermap["OBJTYPE"][i]))
sys.exit(12)
fluxcalib = compute_flux_calibration(frame, model_wave, model_flux)
# QA
if (args.qafile is not None):
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR'])
# Run
#import pdb; pdb.set_trace()
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s"%args.outfile)
def main() :
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--infile', type = str, default = None, required=True,
help = 'path of DESI exposure frame fits file')
parser.add_argument('--fiberflat', type = str, default = None,
help = 'path of DESI fiberflat fits file')
parser.add_argument('--sky', type = str, default = None,
help = 'path of DESI sky fits file')
parser.add_argument('--calib', type = str, default = None,
help = 'path of DESI calibration fits file')
parser.add_argument('--outfile', type = str, default = None, required=True,
help = 'path of DESI sky fits file')
# add calibration here when exists
args = parser.parse_args()
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
frame = read_frame(args.infile)
if args.fiberflat!=None :
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
if args.sky!=None :
log.info("subtract sky")
# read sky
skymodel=read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
if args.calib!=None :
log.info("calibrate")
# read calibration
fluxcalib=read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
# save output
write_frame(args.outfile, frame)
log.info("successfully wrote %s"%args.outfile)
def run_pa(self,input_frame,skymodel,dump=False,dumpfile=None):
from desispec.quicklook.quicksky import subtract_sky
sframe=subtract_sky(input_frame,skymodel)
if dump and dumpfile is not None:
from desispec import io
night = sframe.meta['NIGHT']
expid = sframe.meta['EXPID']
io.write_frame(dumpfile, sframe)
log.info("Wrote intermediate file %s after %s"%(dumpfile,self.name))
return sframe
def run_pa(self,input_frame,skymodel,dump=False,dumpfile=None):
from desispec.quicklook.quicksky import subtract_sky
sframe=subtract_sky(input_frame,skymodel)
if dump and dumpfile is not None:
from desispec import io
night = sframe.meta['NIGHT']
expid = sframe.meta['EXPID']
io.write_frame(dumpfile, sframe)
log.info("Wrote intermediate file %s after %s"%(dumpfile,self.name))
return (sframe,skymodel)
def main(args):
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
frame = read_frame(args.infile)
if args.fiberflat!=None :
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
if args.sky!=None :
log.info("subtract sky")
# read sky
skymodel=read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
if args.calib!=None :
log.info("calibrate")
# read calibration
fluxcalib=read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
# save output
write_frame(args.outfile, frame, units='1e-17 erg/(s cm2 A)')
log.info("successfully wrote %s"%args.outfile)
def main() :
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--infile', type = str, default = None, required=True,
help = 'path of DESI exposure frame fits file')
parser.add_argument('--fibermap', type = str, default = None, required=True,
help = 'path of DESI exposure frame fits file')
parser.add_argument('--fiberflat', type = str, default = None, required=True,
help = 'path of DESI fiberflat fits file')
parser.add_argument('--sky', type = str, default = None, required=True,
help = 'path of DESI sky fits file')
parser.add_argument('--models', type = str, default = None, required=True,
help = 'path of spetro-photometric stellar spectra fits file')
parser.add_argument('--outfile', type = str, default = None, required=True,
help = 'path of DESI flux calbration fits file')
args = parser.parse_args()
log=get_logger()
log.info("read frame")
# read frame
frame = read_frame(args.infile)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel=read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux,model_wave,model_fibers=read_stdstar_models(args.models)
# select fibers
SPECMIN=frame.header["SPECMIN"]
SPECMAX=frame.header["SPECMAX"]
selec=np.where((model_fibers>=SPECMIN)&(model_fibers<=SPECMAX))[0]
if selec.size == 0 :
log.error("no stellar models for this spectro")
sys.exit(12)
fibers=model_fibers[selec]-frame.header["SPECMIN"]
log.info("star fibers= %s"%str(fibers))
table = read_fibermap(args.fibermap)
bad=np.where(table["OBJTYPE"][fibers]!="STD")[0]
if bad.size > 0 :
for fiber in fibers[bad] :
log.error("inconsistency with fiber %d, OBJTYPE='%s' in fibermap"%(fiber,table["OBJTYPE"][fiber]))
sys.exit(12)
fluxcalib = compute_flux_calibration(frame, fibers, model_wave, model_flux)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.header)
log.info("successfully wrote %s"%args.outfile)
def get_skyres(cframes, sub_sky=False, flatten=True):
"""
Args:
cframes: str or list
Single cframe or a list of them
sub_sky: bool, optional
Subtract the sky? This should probably not be done
flatten: bool, optional
Return a flat, 1D array for each variable
combine: bool, optional
combine the individual sky fibers? Median 'smash'
Returns:
wave : ndarray
flux : ndarray
res : ndarray
ivar : ndarray
"""
from desispec.io import read_frame
from desispec.io.sky import read_sky
from desispec.sky import subtract_sky
if isinstance(cframes,list):
all_wave, all_flux, all_res, all_ivar = [], [], [], []
for cframe_file in cframes:
wave, flux, res, ivar = get_skyres(cframe_file, flatten=flatten)
# Save
all_wave.append(wave)
all_flux.append(flux)
all_res.append(res)
all_ivar.append(ivar)
# Concatenate -- Shape is preserved (nfibers, npix)
twave = np.concatenate(all_wave)
tflux = np.concatenate(all_flux)
tres = np.concatenate(all_res)
tivar = np.concatenate(all_ivar)
# Return
return twave, tflux, tres, tivar
cframe = read_frame(cframes, skip_resolution=True)
if cframe.meta['FLAVOR'] in ['flat','arc']:
raise ValueError("Bad flavor for exposure: {:s}".format(cframes))
# Sky
sky_file = cframes.replace('cframe', 'sky')
skymodel = read_sky(sky_file)
if sub_sky:
subtract_sky(cframe, skymodel)
# Resid
skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0]
res = cframe.flux[skyfibers] # Flux calibrated
ivar = cframe.ivar[skyfibers] # Flux calibrated
flux = skymodel.flux[skyfibers] # Residuals; not flux calibrated!
wave = np.outer(np.ones(flux.shape[0]), cframe.wave)
# Combine?
'''
if combine:
res = np.median(res, axis=0)
ivar = np.median(ivar, axis=0)
flux = np.median(flux, axis=0)
wave = np.median(wave, axis=0)
'''
# Return
if flatten:
return wave.flatten(), flux.flatten(), res.flatten(), ivar.flatten()
else:
return wave, flux, res, ivar
def test_subtract_sky(self):
spectra = self._get_spectra()
sky = compute_sky(spectra,add_variance=self.add_variance)
subtract_sky(spectra, sky)
#- allow some slop in the sky subtraction
self.assertTrue(np.allclose(spectra.flux, 0, rtol=1e-5, atol=1e-6))
def main(args):
log = get_logger()
cmd = [
'desi_compute_fluxcalibration',
]
for key, value in args.__dict__.items():
if value is not None:
cmd += ['--' + key, str(value)]
cmd = ' '.join(cmd)
log.info(cmd)
log.info("read frame")
# read frame
frame = read_frame(args.infile)
# Set fibermask flagged spectra to have 0 flux and variance
frame = get_fiberbitmasked_frame(frame,
bitmask='flux',
ivar_framemask=True)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel = read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models(
args.models)
ok = np.ones(len(model_metadata), dtype=bool)
if args.chi2cut > 0:
log.info("apply cut CHI2DOF<{}".format(args.chi2cut))
good = (model_metadata["CHI2DOF"] < args.chi2cut)
bad = ~good
ok &= good
if np.any(bad):
log.info(" discard {} stars with CHI2DOF= {}".format(
np.sum(bad), list(model_metadata["CHI2DOF"][bad])))
legacy_filters = ('G-R', 'R-Z')
gaia_filters = ('GAIA-BP-RP', 'GAIA-G-RP')
model_column_list = model_metadata.columns.names
if args.color is None:
if 'MODEL_G-R' in model_column_list:
color = 'G-R'
elif 'MODEL_GAIA-BP-RP' in model_column_list:
log.info('Using Gaia filters')
color = 'GAIA-BP-RP'
else:
log.error(
"Can't find either G-R or BP-RP color in the model file.")
sys.exit(15)
else:
if args.color not in legacy_filters and args.color not in gaia_filters:
log.error(
'Color name {} is not allowed, must be one of {} {}'.format(
args.color, legacy_filters, gaia_filters))
sys.exit(14)
color = args.color
if color not in model_column_list:
# This should't happen
log.error(
'The color {} was not computed in the models'.format(color))
sys.exit(16)
if args.delta_color_cut > 0:
log.info("apply cut |delta color|<{}".format(args.delta_color_cut))
good = (np.abs(model_metadata["MODEL_" + color] -
model_metadata["DATA_" + color]) < args.delta_color_cut)
bad = ok & (~good)
ok &= good
if np.any(bad):
vals = model_metadata["MODEL_" +
color][bad] - model_metadata["DATA_" +
color][bad]
log.info(" discard {} stars with dcolor= {}".format(
np.sum(bad), list(vals)))
if args.min_color is not None:
log.info("apply cut DATA_{}>{}".format(color, args.min_color))
good = (model_metadata["DATA_{}".format(color)] > args.min_color)
bad = ok & (~good)
ok &= good
if np.any(bad):
vals = model_metadata["DATA_{}".format(color)][bad]
log.info(" discard {} stars with {}= {}".format(
np.sum(bad), color, list(vals)))
if args.chi2cut_nsig > 0:
# automatically reject stars that ar chi2 outliers
mchi2 = np.median(model_metadata["CHI2DOF"])
rmschi2 = np.std(model_metadata["CHI2DOF"])
maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2
log.info("apply cut CHI2DOF<{} based on chi2cut_nsig={}".format(
maxchi2, args.chi2cut_nsig))
good = (model_metadata["CHI2DOF"] <= maxchi2)
bad = ok & (~good)
ok &= good
if np.any(bad):
log.info(" discard {} stars with CHI2DOF={}".format(
np.sum(bad), list(model_metadata["CHI2DOF"][bad])))
ok = np.where(ok)[0]
if ok.size == 0:
log.error("selection cuts discarded all stars")
sys.exit(12)
nstars = model_flux.shape[0]
nbad = nstars - ok.size
if nbad > 0:
log.warning("discarding %d star(s) out of %d because of cuts" %
(nbad, nstars))
model_flux = model_flux[ok]
model_fibers = model_fibers[ok]
model_metadata = model_metadata[:][ok]
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
## check whether star fibers from args.models are consistent with fibers from fibermap
## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
fibermap_std_indices = np.where(isStdStar(fibermap))[0]
if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)):
target_colnames, target_masks, survey = main_cmx_or_sv(fibermap)
colname = target_colnames[0]
for i in model_fibers % 500:
log.error(
"inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap"
.format(i, fibermap["OBJTYPE"][i], colname,
fibermap[colname][i]))
sys.exit(12)
# Make sure the fibers of interest aren't entirely masked.
if np.sum(
np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) ==
frame.nwave) == len(model_fibers):
log.warning('All standard-star spectra are masked!')
return
fluxcalib = compute_flux_calibration(
frame,
model_wave,
model_flux,
model_fibers % 500,
highest_throughput_nstars=args.highest_throughput,
exposure_seeing_fwhm=args.seeing_fwhm)
# QA
if (args.qafile is not None):
from desispec.io import write_qa_frame
from desispec.io.qa import load_qa_frame
from desispec.qa import qa_plots
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame_meta=frame.meta,
flavor=frame.meta['FLAVOR'])
# Run
#import pdb; pdb.set_trace()
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)
# record inputs
frame.meta['IN_FRAME'] = shorten_filename(args.infile)
frame.meta['IN_SKY'] = shorten_filename(args.sky)
frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat)
frame.meta['STDMODEL'] = shorten_filename(args.models)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s" % args.outfile)
def main(args):
log = get_logger()
log.info("read frame")
# read frame
frame = read_frame(args.infile)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel = read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux, model_wave, model_fibers = read_stdstar_models(args.models)
model_tuple = model_flux, model_wave, model_fibers
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
model_fibers = model_fibers % 500
if np.any(fibermap['OBJTYPE'][model_fibers] != 'STD'):
for i in model_fibers:
log.error(
"inconsistency with spectrum %d, OBJTYPE='%s' in fibermap" %
(i, fibermap["OBJTYPE"][i]))
sys.exit(12)
#fluxcalib, indiv_stars = compute_flux_calibration(frame, model_wave, model_flux)
fluxcalib = compute_flux_calibration(frame, model_wave, model_flux)
# QA
if (args.qafile is not None):
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame,
flavor=frame.meta['FLAVOR'])
# Run
qaframe.run_qa('FLUXCALIB',
(frame, fluxcalib, model_tuple)) #, indiv_stars))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib,
model_tuple)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s" % args.outfile)
def run_pa(self, input_frame, skymodel):
from desispec.sky import subtract_sky
subtract_sky(input_frame, skymodel)
return input_frame
def main(args=None, comm=None):
if args is None:
args = parse()
# elif isinstance(args, (list, tuple)):
# args = parse(args)
log = get_logger()
start_mpi_connect = time.time()
if comm is not None:
#- Use the provided comm to determine rank and size
rank = comm.rank
size = comm.size
else:
#- Check MPI flags and determine the comm, rank, and size given the arguments
comm, rank, size = assign_mpi(do_mpi=args.mpi,
do_batch=args.batch,
log=log)
stop_mpi_connect = time.time()
#- Start timer; only print log messages from rank 0 (others are silent)
timer = desiutil.timer.Timer(silent=(rank > 0))
#- Fill in timing information for steps before we had the timer created
if args.starttime is not None:
timer.start('startup', starttime=args.starttime)
timer.stop('startup', stoptime=start_imports)
timer.start('imports', starttime=start_imports)
timer.stop('imports', stoptime=stop_imports)
timer.start('mpi_connect', starttime=start_mpi_connect)
timer.stop('mpi_connect', stoptime=stop_mpi_connect)
#- Freeze IERS after parsing args so that it doesn't bother if only --help
timer.start('freeze_iers')
desiutil.iers.freeze_iers()
timer.stop('freeze_iers')
#- Preflight checks
timer.start('preflight')
if rank > 0:
#- Let rank 0 fetch these, and then broadcast
args, hdr, camhdr = None, None, None
else:
args, hdr, camhdr = update_args_with_headers(args)
## Make sure badamps is formatted properly
if comm is not None and rank == 0 and args.badamps is not None:
args.badamps = validate_badamps(args.badamps)
if comm is not None:
args = comm.bcast(args, root=0)
hdr = comm.bcast(hdr, root=0)
camhdr = comm.bcast(camhdr, root=0)
known_obstype = [
'SCIENCE', 'ARC', 'FLAT', 'ZERO', 'DARK', 'TESTARC', 'TESTFLAT',
'PIXFLAT', 'SKY', 'TWILIGHT', 'OTHER'
]
if args.obstype not in known_obstype:
raise RuntimeError('obstype {} not in {}'.format(
args.obstype, known_obstype))
timer.stop('preflight')
#-------------------------------------------------------------------------
#- Create and submit a batch job if requested
if args.batch:
#exp_str = '{:08d}'.format(args.expid)
jobdesc = args.obstype.lower()
if args.obstype == 'SCIENCE':
# if not doing pre-stdstar fitting or stdstar fitting and if there is
# no flag stopping flux calibration, set job to poststdstar
if args.noprestdstarfit and args.nostdstarfit and (
not args.nofluxcalib):
jobdesc = 'poststdstar'
# elif told not to do std or post stdstar but the flag for prestdstar isn't set,
# then perform prestdstar
elif (not args.noprestdstarfit
) and args.nostdstarfit and args.nofluxcalib:
jobdesc = 'prestdstar'
#elif (not args.noprestdstarfit) and (not args.nostdstarfit) and (not args.nofluxcalib):
# jobdesc = 'science'
scriptfile = create_desi_proc_batch_script(night=args.night, exp=args.expid, cameras=args.cameras,\
jobdesc=jobdesc, queue=args.queue, runtime=args.runtime,\
batch_opts=args.batch_opts, timingfile=args.timingfile,
system_name=args.system_name)
err = 0
if not args.nosubmit:
err = subprocess.call(['sbatch', scriptfile])
sys.exit(err)
#-------------------------------------------------------------------------
#- Proceeding with running
#- What are we going to do?
if rank == 0:
log.info('----------')
log.info('Input {}'.format(args.input))
log.info('Night {} expid {}'.format(args.night, args.expid))
log.info('Obstype {}'.format(args.obstype))
log.info('Cameras {}'.format(args.cameras))
log.info('Output root {}'.format(desispec.io.specprod_root()))
log.info('----------')
#- Create output directories if needed
if rank == 0:
preprocdir = os.path.dirname(
findfile('preproc', args.night, args.expid, 'b0'))
expdir = os.path.dirname(
findfile('frame', args.night, args.expid, 'b0'))
os.makedirs(preprocdir, exist_ok=True)
os.makedirs(expdir, exist_ok=True)
#- Wait for rank 0 to make directories before proceeding
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Preproc
#- All obstypes get preprocessed
timer.start('fibermap')
#- Assemble fibermap for science exposures
fibermap = None
fibermap_ok = None
if rank == 0 and args.obstype == 'SCIENCE':
fibermap = findfile('fibermap', args.night, args.expid)
if not os.path.exists(fibermap):
tmp = findfile('preproc', args.night, args.expid, 'b0')
preprocdir = os.path.dirname(tmp)
fibermap = os.path.join(preprocdir, os.path.basename(fibermap))
log.info('Creating fibermap {}'.format(fibermap))
cmd = 'assemble_fibermap -n {} -e {} -o {}'.format(
args.night, args.expid, fibermap)
if args.badamps is not None:
cmd += ' --badamps={}'.format(args.badamps)
runcmd(cmd, inputs=[], outputs=[fibermap])
fibermap_ok = os.path.exists(fibermap)
#- Some commissioning files didn't have coords* files that caused assemble_fibermap to fail
#- these are well known failures with no other solution, so for those, just force creation
#- of a fibermap with null coordinate information
if not fibermap_ok and int(args.night) < 20200310:
log.info(
"Since night is before 20200310, trying to force fibermap creation without coords file"
)
cmd += ' --force'
runcmd(cmd, inputs=[], outputs=[fibermap])
fibermap_ok = os.path.exists(fibermap)
#- If assemble_fibermap failed and obstype is SCIENCE, exit now
if comm is not None:
fibermap_ok = comm.bcast(fibermap_ok, root=0)
if args.obstype == 'SCIENCE' and not fibermap_ok:
sys.stdout.flush()
if rank == 0:
log.critical(
'assemble_fibermap failed for science exposure; exiting now')
sys.exit(13)
#- Wait for rank 0 to make fibermap if needed
if comm is not None:
fibermap = comm.bcast(fibermap, root=0)
timer.stop('fibermap')
if not (args.obstype in ['SCIENCE'] and args.noprestdstarfit):
timer.start('preproc')
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
outfile = findfile('preproc', args.night, args.expid, camera)
outdir = os.path.dirname(outfile)
cmd = "desi_preproc -i {} -o {} --outdir {} --cameras {}".format(
args.input, outfile, outdir, camera)
if args.scattered_light:
cmd += " --scattered-light"
if fibermap is not None:
cmd += " --fibermap {}".format(fibermap)
if not args.obstype in ['ARC']: # never model variance for arcs
if not args.no_model_pixel_variance:
cmd += " --model-variance"
runcmd(cmd, inputs=[args.input], outputs=[outfile])
timer.stop('preproc')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Get input PSFs
timer.start('findpsf')
input_psf = dict()
if rank == 0:
for camera in args.cameras:
if args.psf is not None:
input_psf[camera] = args.psf
elif args.calibnight is not None:
# look for a psfnight psf for this calib night
psfnightfile = findfile('psfnight', args.calibnight,
args.expid, camera)
if not os.path.isfile(psfnightfile):
log.error("no {}".format(psfnightfile))
raise IOError("no {}".format(psfnightfile))
input_psf[camera] = psfnightfile
else:
# look for a psfnight psf
psfnightfile = findfile('psfnight', args.night, args.expid,
camera)
if os.path.isfile(psfnightfile):
input_psf[camera] = psfnightfile
elif args.most_recent_calib:
nightfile = find_most_recent(args.night,
file_type='psfnight')
if nightfile is None:
input_psf[camera] = findcalibfile(
[hdr, camhdr[camera]], 'PSF')
else:
input_psf[camera] = nightfile
else:
input_psf[camera] = findcalibfile([hdr, camhdr[camera]],
'PSF')
log.info("Will use input PSF : {}".format(input_psf[camera]))
if comm is not None:
input_psf = comm.bcast(input_psf, root=0)
timer.stop('findpsf')
#-------------------------------------------------------------------------
#- Traceshift
if ( args.obstype in ['FLAT', 'TESTFLAT', 'SKY', 'TWILIGHT'] ) or \
( args.obstype in ['SCIENCE'] and (not args.noprestdstarfit) ):
timer.start('traceshift')
if rank == 0 and args.traceshift:
log.info('Starting traceshift at {}'.format(time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
preprocfile = findfile('preproc', args.night, args.expid, camera)
inpsf = input_psf[camera]
outpsf = findfile('psf', args.night, args.expid, camera)
if not os.path.isfile(outpsf):
if args.traceshift:
cmd = "desi_compute_trace_shifts"
cmd += " -i {}".format(preprocfile)
cmd += " --psf {}".format(inpsf)
cmd += " --outpsf {}".format(outpsf)
cmd += " --degxx 2 --degxy 0"
if args.obstype in ['FLAT', 'TESTFLAT', 'TWILIGHT']:
cmd += " --continuum"
else:
cmd += " --degyx 2 --degyy 0"
if args.obstype in ['SCIENCE', 'SKY']:
cmd += ' --sky'
else:
cmd = "ln -s {} {}".format(inpsf, outpsf)
runcmd(cmd, inputs=[preprocfile, inpsf], outputs=[outpsf])
else:
log.info("PSF {} exists".format(outpsf))
timer.stop('traceshift')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- PSF
#- MPI parallelize this step
if args.obstype in ['ARC', 'TESTARC']:
timer.start('arc_traceshift')
if rank == 0:
log.info('Starting traceshift before specex PSF fit at {}'.format(
time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
preprocfile = findfile('preproc', args.night, args.expid, camera)
inpsf = input_psf[camera]
outpsf = findfile('psf', args.night, args.expid, camera)
outpsf = replace_prefix(outpsf, "psf", "shifted-input-psf")
if not os.path.isfile(outpsf):
cmd = "desi_compute_trace_shifts"
cmd += " -i {}".format(preprocfile)
cmd += " --psf {}".format(inpsf)
cmd += " --outpsf {}".format(outpsf)
cmd += " --degxx 0 --degxy 0 --degyx 0 --degyy 0"
cmd += ' --arc-lamps'
runcmd(cmd, inputs=[preprocfile, inpsf], outputs=[outpsf])
else:
log.info("PSF {} exists".format(outpsf))
timer.stop('arc_traceshift')
if comm is not None:
comm.barrier()
timer.start('psf')
if rank == 0:
log.info('Starting specex PSF fitting at {}'.format(
time.asctime()))
if rank > 0:
cmds = inputs = outputs = None
else:
cmds = dict()
inputs = dict()
outputs = dict()
for camera in args.cameras:
preprocfile = findfile('preproc', args.night, args.expid,
camera)
tmpname = findfile('psf', args.night, args.expid, camera)
inpsf = replace_prefix(tmpname, "psf", "shifted-input-psf")
outpsf = replace_prefix(tmpname, "psf", "fit-psf")
log.info("now run specex psf fit")
cmd = 'desi_compute_psf'
cmd += ' --input-image {}'.format(preprocfile)
cmd += ' --input-psf {}'.format(inpsf)
cmd += ' --output-psf {}'.format(outpsf)
# look for fiber blacklist
cfinder = CalibFinder([hdr, camhdr[camera]])
blacklistkey = "FIBERBLACKLIST"
if not cfinder.haskey(blacklistkey) and cfinder.haskey(
"BROKENFIBERS"):
log.warning(
"BROKENFIBERS yaml keyword deprecated, please use FIBERBLACKLIST"
)
blacklistkey = "BROKENFIBERS"
if cfinder.haskey(blacklistkey):
blacklist = cfinder.value(blacklistkey)
cmd += ' --broken-fibers {}'.format(blacklist)
if rank == 0:
log.warning('broken fibers: {}'.format(blacklist))
if not os.path.exists(outpsf):
cmds[camera] = cmd
inputs[camera] = [preprocfile, inpsf]
outputs[camera] = [
outpsf,
]
if comm is not None:
cmds = comm.bcast(cmds, root=0)
inputs = comm.bcast(inputs, root=0)
outputs = comm.bcast(outputs, root=0)
#- split communicator by 20 (number of bundles)
group_size = 20
if (rank == 0) and (size % group_size != 0):
log.warning(
'MPI size={} should be evenly divisible by {}'.format(
size, group_size))
group = rank // group_size
num_groups = (size + group_size - 1) // group_size
comm_group = comm.Split(color=group)
if rank == 0:
log.info(
f'Fitting PSFs with {num_groups} sub-communicators of size {group_size}'
)
for i in range(group, len(args.cameras), num_groups):
camera = args.cameras[i]
if camera in cmds:
cmdargs = cmds[camera].split()[1:]
cmdargs = desispec.scripts.specex.parse(cmdargs)
if comm_group.rank == 0:
print('RUNNING: {}'.format(cmds[camera]))
t0 = time.time()
timestamp = time.asctime()
log.info(
f'MPI group {group} ranks {rank}-{rank+group_size-1} fitting PSF for {camera} at {timestamp}'
)
try:
desispec.scripts.specex.main(cmdargs, comm=comm_group)
except Exception as e:
if comm_group.rank == 0:
log.error(
f'FAILED: MPI group {group} ranks {rank}-{rank+group_size-1} camera {camera}'
)
log.error('FAILED: {}'.format(cmds[camera]))
log.error(e)
if comm_group.rank == 0:
specex_time = time.time() - t0
log.info(
f'specex fit for {camera} took {specex_time:.1f} seconds'
)
comm.barrier()
else:
log.warning(
'fitting PSFs without MPI parallelism; this will be SLOW')
for camera in args.cameras:
if camera in cmds:
runcmd(cmds[camera],
inputs=inputs[camera],
outputs=outputs[camera])
if comm is not None:
comm.barrier()
# loop on all cameras and interpolate bad fibers
for camera in args.cameras[rank::size]:
t0 = time.time()
log.info(f'Rank {rank} interpolating {camera} PSF over bad fibers')
# look for fiber blacklist
cfinder = CalibFinder([hdr, camhdr[camera]])
blacklistkey = "FIBERBLACKLIST"
if not cfinder.haskey(blacklistkey) and cfinder.haskey(
"BROKENFIBERS"):
log.warning(
"BROKENFIBERS yaml keyword deprecated, please use FIBERBLACKLIST"
)
blacklistkey = "BROKENFIBERS"
if cfinder.haskey(blacklistkey):
fiberblacklist = cfinder.value(blacklistkey)
tmpname = findfile('psf', args.night, args.expid, camera)
inpsf = replace_prefix(tmpname, "psf", "fit-psf")
outpsf = replace_prefix(tmpname, "psf",
"fit-psf-fixed-blacklisted")
if os.path.isfile(inpsf) and not os.path.isfile(outpsf):
cmd = 'desi_interpolate_fiber_psf'
cmd += ' --infile {}'.format(inpsf)
cmd += ' --outfile {}'.format(outpsf)
cmd += ' --fibers {}'.format(fiberblacklist)
log.info(
'For camera {} interpolating PSF for broken fibers: {}'
.format(camera, fiberblacklist))
runcmd(cmd, inputs=[inpsf], outputs=[outpsf])
if os.path.isfile(outpsf):
os.rename(
inpsf,
inpsf.replace("fit-psf",
"fit-psf-before-blacklisted-fix"))
subprocess.call('cp {} {}'.format(outpsf, inpsf),
shell=True)
dt = time.time() - t0
log.info(
f'Rank {rank} {camera} PSF interpolation took {dt:.1f} sec')
timer.stop('psf')
#-------------------------------------------------------------------------
#- Merge PSF of night if applicable
#if args.obstype in ['ARC']:
if False:
if rank == 0:
for camera in args.cameras:
psfnightfile = findfile('psfnight', args.night, args.expid,
camera)
if not os.path.isfile(
psfnightfile
): # we still don't have a psf night, see if we can compute it ...
psfs = glob.glob(
findfile('psf', args.night, args.expid,
camera).replace("psf", "fit-psf").replace(
str(args.expid), "*"))
log.info(
"Number of PSF for night={} camera={} = {}".format(
args.night, camera, len(psfs)))
if len(psfs) > 4: # lets do it!
log.info("Computing psfnight ...")
dirname = os.path.dirname(psfnightfile)
if not os.path.isdir(dirname):
os.makedirs(dirname)
desispec.scripts.specex.mean_psf(psfs, psfnightfile)
if os.path.isfile(psfnightfile): # now use this one
input_psf[camera] = psfnightfile
#-------------------------------------------------------------------------
#- Extract
#- This is MPI parallel so handle a bit differently
# maybe add ARC and TESTARC too
if ( args.obstype in ['FLAT', 'TESTFLAT', 'SKY', 'TWILIGHT'] ) or \
( args.obstype in ['SCIENCE'] and (not args.noprestdstarfit) ):
timer.start('extract')
if rank == 0:
log.info('Starting extractions at {}'.format(time.asctime()))
if rank > 0:
cmds = inputs = outputs = None
else:
cmds = dict()
inputs = dict()
outputs = dict()
for camera in args.cameras:
cmd = 'desi_extract_spectra'
#- Based on data from SM1-SM8, looking at central and edge fibers
#- with in mind overlapping arc lamps lines
if camera.startswith('b'):
cmd += ' -w 3600.0,5800.0,0.8'
elif camera.startswith('r'):
cmd += ' -w 5760.0,7620.0,0.8'
elif camera.startswith('z'):
cmd += ' -w 7520.0,9824.0,0.8'
preprocfile = findfile('preproc', args.night, args.expid,
camera)
psffile = findfile('psf', args.night, args.expid, camera)
framefile = findfile('frame', args.night, args.expid, camera)
cmd += ' -i {}'.format(preprocfile)
cmd += ' -p {}'.format(psffile)
cmd += ' -o {}'.format(framefile)
cmd += ' --psferr 0.1'
if args.obstype == 'SCIENCE' or args.obstype == 'SKY':
if rank == 0:
log.info('Include barycentric correction')
cmd += ' --barycentric-correction'
if not os.path.exists(framefile):
cmds[camera] = cmd
inputs[camera] = [preprocfile, psffile]
outputs[camera] = [
framefile,
]
#- TODO: refactor/combine this with PSF comm splitting logic
if comm is not None:
cmds = comm.bcast(cmds, root=0)
inputs = comm.bcast(inputs, root=0)
outputs = comm.bcast(outputs, root=0)
#- split communicator by 20 (number of bundles)
extract_size = 20
if (rank == 0) and (size % extract_size != 0):
log.warning(
'MPI size={} should be evenly divisible by {}'.format(
size, extract_size))
extract_group = rank // extract_size
num_extract_groups = (size + extract_size - 1) // extract_size
comm_extract = comm.Split(color=extract_group)
for i in range(extract_group, len(args.cameras),
num_extract_groups):
camera = args.cameras[i]
if camera in cmds:
cmdargs = cmds[camera].split()[1:]
extract_args = desispec.scripts.extract.parse(cmdargs)
if comm_extract.rank == 0:
print('RUNNING: {}'.format(cmds[camera]))
desispec.scripts.extract.main_mpi(extract_args,
comm=comm_extract)
comm.barrier()
else:
log.warning(
'running extractions without MPI parallelism; this will be SLOW'
)
for camera in args.cameras:
if camera in cmds:
runcmd(cmds[camera],
inputs=inputs[camera],
outputs=outputs[camera])
timer.stop('extract')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Fiberflat
if args.obstype in ['FLAT', 'TESTFLAT']:
timer.start('fiberflat')
if rank == 0:
log.info('Starting fiberflats at {}'.format(time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
framefile = findfile('frame', args.night, args.expid, camera)
fiberflatfile = findfile('fiberflat', args.night, args.expid,
camera)
cmd = "desi_compute_fiberflat"
cmd += " -i {}".format(framefile)
cmd += " -o {}".format(fiberflatfile)
runcmd(cmd, inputs=[
framefile,
], outputs=[
fiberflatfile,
])
timer.stop('fiberflat')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Average and auto-calib fiberflats of night if applicable
#if args.obstype in ['FLAT']:
if False:
if rank == 0:
fiberflatnightfile = findfile('fiberflatnight', args.night,
args.expid, args.cameras[0])
fiberflatdirname = os.path.dirname(fiberflatnightfile)
if not os.path.isfile(fiberflatnightfile) and len(
args.cameras
) >= 6: # we still don't have them, see if we can compute them, but need at least 2 spectros ...
flats = glob.glob(
findfile('fiberflat', args.night, args.expid,
"b0").replace(str(args.expid),
"*").replace("b0", "*"))
log.info("Number of fiberflat for night {} = {}".format(
args.night, len(flats)))
if len(flats) >= 3 * 4 * len(
args.cameras
): # lets do it! (3 exposures x 4 lamps x N cameras)
log.info(
"Computing fiberflatnight per lamp and camera ...")
tmpdir = os.path.join(fiberflatdirname, "tmp")
if not os.path.isdir(tmpdir):
os.makedirs(tmpdir)
log.info(
"First average measurements per camera and per lamp")
average_flats = dict()
for camera in args.cameras:
# list of flats for this camera
flats_for_this_camera = []
for flat in flats:
if flat.find(camera) >= 0:
flats_for_this_camera.append(flat)
#log.info("For camera {} , flats = {}".format(camera,flats_for_this_camera))
#sys.exit(12)
# average per lamp (and camera)
average_flats[camera] = list()
for lampbox in range(4):
ofile = os.path.join(
tmpdir,
"fiberflatnight-camera-{}-lamp-{}.fits".format(
camera, lampbox))
if not os.path.isfile(ofile):
log.info(
"Average flat for camera {} and lamp box #{}"
.format(camera, lampbox))
pg = "CALIB DESI-CALIB-0{} LEDs only".format(
lampbox)
cmd = "desi_average_fiberflat --program '{}' --outfile {} -i ".format(
pg, ofile)
for flat in flats_for_this_camera:
cmd += " {} ".format(flat)
runcmd(cmd,
inputs=flats_for_this_camera,
outputs=[
ofile,
])
if os.path.isfile(ofile):
average_flats[camera].append(ofile)
else:
log.info("Will use existing {}".format(ofile))
average_flats[camera].append(ofile)
log.info(
"Auto-calibration across lamps and spectro per camera arm (b,r,z)"
)
for camera_arm in ["b", "r", "z"]:
cameras_for_this_arm = []
flats_for_this_arm = []
for camera in args.cameras:
if camera[0].lower() == camera_arm:
cameras_for_this_arm.append(camera)
if camera in average_flats:
for flat in average_flats[camera]:
flats_for_this_arm.append(flat)
cmd = "desi_autocalib_fiberflat --night {} --arm {} -i ".format(
args.night, camera_arm)
for flat in flats_for_this_arm:
cmd += " {} ".format(flat)
runcmd(cmd, inputs=flats_for_this_arm, outputs=[])
log.info("Done with fiber flats per night")
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Get input fiberflat
if args.obstype in ['SCIENCE', 'SKY'] and (not args.nofiberflat):
timer.start('find_fiberflat')
input_fiberflat = dict()
if rank == 0:
for camera in args.cameras:
if args.fiberflat is not None:
input_fiberflat[camera] = args.fiberflat
elif args.calibnight is not None:
# look for a fiberflatnight for this calib night
fiberflatnightfile = findfile('fiberflatnight',
args.calibnight, args.expid,
camera)
if not os.path.isfile(fiberflatnightfile):
log.error("no {}".format(fiberflatnightfile))
raise IOError("no {}".format(fiberflatnightfile))
input_fiberflat[camera] = fiberflatnightfile
else:
# look for a fiberflatnight fiberflat
fiberflatnightfile = findfile('fiberflatnight', args.night,
args.expid, camera)
if os.path.isfile(fiberflatnightfile):
input_fiberflat[camera] = fiberflatnightfile
elif args.most_recent_calib:
nightfile = find_most_recent(
args.night, file_type='fiberflatnight')
if nightfile is None:
input_fiberflat[camera] = findcalibfile(
[hdr, camhdr[camera]], 'FIBERFLAT')
else:
input_fiberflat[camera] = nightfile
else:
input_fiberflat[camera] = findcalibfile(
[hdr, camhdr[camera]], 'FIBERFLAT')
log.info("Will use input FIBERFLAT: {}".format(
input_fiberflat[camera]))
if comm is not None:
input_fiberflat = comm.bcast(input_fiberflat, root=0)
timer.stop('find_fiberflat')
#-------------------------------------------------------------------------
#- Apply fiberflat and write fframe file
if args.obstype in ['SCIENCE', 'SKY'] and args.fframe and \
( not args.nofiberflat ) and (not args.noprestdstarfit):
timer.start('apply_fiberflat')
if rank == 0:
log.info('Applying fiberflat at {}'.format(time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
fframefile = findfile('fframe', args.night, args.expid, camera)
if not os.path.exists(fframefile):
framefile = findfile('frame', args.night, args.expid, camera)
fr = desispec.io.read_frame(framefile)
flatfilename = input_fiberflat[camera]
if flatfilename is not None:
ff = desispec.io.read_fiberflat(flatfilename)
fr.meta['FIBERFLT'] = desispec.io.shorten_filename(
flatfilename)
apply_fiberflat(fr, ff)
fframefile = findfile('fframe', args.night, args.expid,
camera)
desispec.io.write_frame(fframefile, fr)
else:
log.warning(
"Missing fiberflat for camera {}".format(camera))
timer.stop('apply_fiberflat')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Select random sky fibers (inplace update of frame file)
#- TODO: move this to a function somewhere
#- TODO: this assigns different sky fibers to each frame of same spectrograph
if (args.obstype in [
'SKY', 'SCIENCE'
]) and (not args.noskysub) and (not args.noprestdstarfit):
timer.start('picksky')
if rank == 0:
log.info('Picking sky fibers at {}'.format(time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
framefile = findfile('frame', args.night, args.expid, camera)
orig_frame = desispec.io.read_frame(framefile)
#- Make a copy so that we can apply fiberflat
fr = deepcopy(orig_frame)
if np.any(fr.fibermap['OBJTYPE'] == 'SKY'):
log.info('{} sky fibers already set; skipping'.format(
os.path.basename(framefile)))
continue
#- Apply fiberflat then select random fibers below a flux cut
flatfilename = input_fiberflat[camera]
if flatfilename is None:
log.error("No fiberflat for {}".format(camera))
continue
ff = desispec.io.read_fiberflat(flatfilename)
apply_fiberflat(fr, ff)
sumflux = np.sum(fr.flux, axis=1)
fluxcut = np.percentile(sumflux, 30)
iisky = np.where(sumflux < fluxcut)[0]
iisky = np.random.choice(iisky, size=100, replace=False)
#- Update fibermap or original frame and write out
orig_frame.fibermap['OBJTYPE'][iisky] = 'SKY'
orig_frame.fibermap['DESI_TARGET'][iisky] |= desi_mask.SKY
desispec.io.write_frame(framefile, orig_frame)
timer.stop('picksky')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Sky subtraction
if args.obstype in [
'SCIENCE', 'SKY'
] and (not args.noskysub) and (not args.noprestdstarfit):
timer.start('skysub')
if rank == 0:
log.info('Starting sky subtraction at {}'.format(time.asctime()))
for i in range(rank, len(args.cameras), size):
camera = args.cameras[i]
framefile = findfile('frame', args.night, args.expid, camera)
hdr = fitsio.read_header(framefile, 'FLUX')
fiberflatfile = input_fiberflat[camera]
if fiberflatfile is None:
log.error("No fiberflat for {}".format(camera))
continue
skyfile = findfile('sky', args.night, args.expid, camera)
cmd = "desi_compute_sky"
cmd += " -i {}".format(framefile)
cmd += " --fiberflat {}".format(fiberflatfile)
cmd += " --o {}".format(skyfile)
if args.no_extra_variance:
cmd += " --no-extra-variance"
if not args.no_sky_wavelength_adjustment:
cmd += " --adjust-wavelength"
if not args.no_sky_lsf_adjustment: cmd += " --adjust-lsf"
runcmd(cmd, inputs=[framefile, fiberflatfile], outputs=[
skyfile,
])
#- sframe = flatfielded sky-subtracted but not flux calibrated frame
#- Note: this re-reads and re-does steps previously done for picking
#- sky fibers; desi_proc is about human efficiency,
#- not I/O or CPU efficiency...
sframefile = desispec.io.findfile('sframe', args.night, args.expid,
camera)
if not os.path.exists(sframefile):
frame = desispec.io.read_frame(framefile)
fiberflat = desispec.io.read_fiberflat(fiberflatfile)
sky = desispec.io.read_sky(skyfile)
apply_fiberflat(frame, fiberflat)
subtract_sky(frame, sky, apply_throughput_correction=True)
frame.meta['IN_SKY'] = shorten_filename(skyfile)
frame.meta['FIBERFLT'] = shorten_filename(fiberflatfile)
desispec.io.write_frame(sframefile, frame)
timer.stop('skysub')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Standard Star Fitting
if args.obstype in ['SCIENCE',] and \
(not args.noskysub ) and \
(not args.nostdstarfit) :
timer.start('stdstarfit')
if rank == 0:
log.info('Starting flux calibration at {}'.format(time.asctime()))
#- Group inputs by spectrograph
framefiles = dict()
skyfiles = dict()
fiberflatfiles = dict()
night, expid = args.night, args.expid #- shorter
for camera in args.cameras:
sp = int(camera[1])
if sp not in framefiles:
framefiles[sp] = list()
skyfiles[sp] = list()
fiberflatfiles[sp] = list()
framefiles[sp].append(findfile('frame', night, expid, camera))
skyfiles[sp].append(findfile('sky', night, expid, camera))
fiberflatfiles[sp].append(input_fiberflat[camera])
#- Hardcoded stdstar model version
starmodels = os.path.join(os.getenv('DESI_BASIS_TEMPLATES'),
'stdstar_templates_v2.2.fits')
#- Fit stdstars per spectrograph (not per-camera)
spectro_nums = sorted(framefiles.keys())
## for sp in spectro_nums[rank::size]:
for i in range(rank, len(spectro_nums), size):
sp = spectro_nums[i]
stdfile = findfile('stdstars', night, expid, spectrograph=sp)
cmd = "desi_fit_stdstars"
cmd += " --frames {}".format(' '.join(framefiles[sp]))
cmd += " --skymodels {}".format(' '.join(skyfiles[sp]))
cmd += " --fiberflats {}".format(' '.join(fiberflatfiles[sp]))
cmd += " --starmodels {}".format(starmodels)
cmd += " --outfile {}".format(stdfile)
cmd += " --delta-color 0.1"
if args.maxstdstars is not None:
cmd += " --maxstdstars {}".format(args.maxstdstars)
inputs = framefiles[sp] + skyfiles[sp] + fiberflatfiles[sp]
runcmd(cmd, inputs=inputs, outputs=[stdfile])
timer.stop('stdstarfit')
if comm is not None:
comm.barrier()
# -------------------------------------------------------------------------
# - Flux calibration
if args.obstype in ['SCIENCE'] and \
(not args.noskysub) and \
(not args.nofluxcalib):
timer.start('fluxcalib')
night, expid = args.night, args.expid #- shorter
#- Compute flux calibration vectors per camera
for camera in args.cameras[rank::size]:
framefile = findfile('frame', night, expid, camera)
skyfile = findfile('sky', night, expid, camera)
spectrograph = int(camera[1])
stdfile = findfile('stdstars',
night,
expid,
spectrograph=spectrograph)
calibfile = findfile('fluxcalib', night, expid, camera)
fiberflatfile = input_fiberflat[camera]
cmd = "desi_compute_fluxcalibration"
cmd += " --infile {}".format(framefile)
cmd += " --sky {}".format(skyfile)
cmd += " --fiberflat {}".format(fiberflatfile)
cmd += " --models {}".format(stdfile)
cmd += " --outfile {}".format(calibfile)
cmd += " --delta-color-cut 0.1"
inputs = [framefile, skyfile, fiberflatfile, stdfile]
runcmd(cmd, inputs=inputs, outputs=[
calibfile,
])
timer.stop('fluxcalib')
if comm is not None:
comm.barrier()
#-------------------------------------------------------------------------
#- Applying flux calibration
if args.obstype in [
'SCIENCE',
] and (not args.noskysub) and (not args.nofluxcalib):
night, expid = args.night, args.expid #- shorter
timer.start('applycalib')
if rank == 0:
log.info('Starting cframe file creation at {}'.format(
time.asctime()))
for camera in args.cameras[rank::size]:
framefile = findfile('frame', night, expid, camera)
skyfile = findfile('sky', night, expid, camera)
spectrograph = int(camera[1])
stdfile = findfile('stdstars',
night,
expid,
spectrograph=spectrograph)
calibfile = findfile('fluxcalib', night, expid, camera)
cframefile = findfile('cframe', night, expid, camera)
fiberflatfile = input_fiberflat[camera]
cmd = "desi_process_exposure"
cmd += " --infile {}".format(framefile)
cmd += " --fiberflat {}".format(fiberflatfile)
cmd += " --sky {}".format(skyfile)
cmd += " --calib {}".format(calibfile)
cmd += " --outfile {}".format(cframefile)
cmd += " --cosmics-nsig 6"
if args.no_xtalk:
cmd += " --no-xtalk"
inputs = [framefile, fiberflatfile, skyfile, calibfile]
runcmd(cmd, inputs=inputs, outputs=[
cframefile,
])
if comm is not None:
comm.barrier()
timer.stop('applycalib')
#-------------------------------------------------------------------------
#- Wrap up
# if rank == 0:
# report = timer.report()
# log.info('Rank 0 timing report:\n' + report)
if comm is not None:
timers = comm.gather(timer, root=0)
else:
timers = [
timer,
]
if rank == 0:
stats = desiutil.timer.compute_stats(timers)
log.info('Timing summary statistics:\n' + json.dumps(stats, indent=2))
if args.timingfile:
if os.path.exists(args.timingfile):
with open(args.timingfile) as fx:
previous_stats = json.load(fx)
#- augment previous_stats with new entries, but don't overwrite old
for name in stats:
if name not in previous_stats:
previous_stats[name] = stats[name]
stats = previous_stats
tmpfile = args.timingfile + '.tmp'
with open(tmpfile, 'w') as fx:
json.dump(stats, fx, indent=2)
os.rename(tmpfile, args.timingfile)
if rank == 0:
log.info('All done at {}'.format(time.asctime()))
def main(args):
log = get_logger()
log.info("read frame")
# read frame
frame = read_frame(args.infile)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel = read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models(
args.models)
if args.chi2cut > 0:
ok = np.where(model_metadata["CHI2DOF"] < args.chi2cut)[0]
if ok.size == 0:
log.error("chi2cut has discarded all stars")
sys.exit(12)
nstars = model_flux.shape[0]
nbad = nstars - ok.size
if nbad > 0:
log.warning("discarding %d star(s) out of %d because of chi2cut" %
(nbad, nstars))
model_flux = model_flux[ok]
model_fibers = model_fibers[ok]
model_metadata = model_metadata[:][ok]
if args.delta_color_cut > 0:
ok = np.where(
np.abs(model_metadata["MODEL_G-R"] -
model_metadata["DATA_G-R"]) < args.delta_color_cut)[0]
nstars = model_flux.shape[0]
nbad = nstars - ok.size
if nbad > 0:
log.warning(
"discarding %d star(s) out of %d because |delta_color|>%f" %
(nbad, nstars, args.delta_color_cut))
model_flux = model_flux[ok]
model_fibers = model_fibers[ok]
model_metadata = model_metadata[:][ok]
# automatically reject stars that ar chi2 outliers
if args.chi2cut_nsig > 0:
mchi2 = np.median(model_metadata["CHI2DOF"])
rmschi2 = np.std(model_metadata["CHI2DOF"])
maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2
ok = np.where(model_metadata["CHI2DOF"] <= maxchi2)[0]
nstars = model_flux.shape[0]
nbad = nstars - ok.size
if nbad > 0:
log.warning(
"discarding %d star(s) out of %d because reduced chi2 outliers (at %d sigma, giving rchi2<%f )"
% (nbad, nstars, args.chi2cut_nsig, maxchi2))
model_flux = model_flux[ok]
model_fibers = model_fibers[ok]
model_metadata = model_metadata[:][ok]
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
## check whether star fibers from args.models are consistent with fibers from fibermap
## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
w = np.where(fibermap["OBJTYPE"][model_fibers % 500] != 'STD')[0]
if len(w) > 0:
for i in model_fibers % 500:
log.error(
"inconsistency with spectrum %d, OBJTYPE='%s' in fibermap" %
(i, fibermap["OBJTYPE"][i]))
sys.exit(12)
fluxcalib = compute_flux_calibration(frame, model_wave, model_flux,
model_fibers % 500)
# QA
if (args.qafile is not None):
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame,
flavor=frame.meta['FLAVOR'])
# Run
#import pdb; pdb.set_trace()
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s" % args.outfile)
def main(args):
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
frame = read_frame(args.infile)
#- Raw scores already added in extraction, but just in case they weren't
#- it is harmless to rerun to make sure we have them.
compute_and_append_frame_scores(frame,suffix="RAW")
if args.cosmics_nsig>0 and args.sky==None : # Reject cosmics (otherwise do it after sky subtraction)
log.info("cosmics ray 1D rejection")
reject_cosmic_rays_1d(frame,args.cosmics_nsig)
if args.fiberflat!=None :
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to all fibers
apply_fiberflat(frame, fiberflat)
compute_and_append_frame_scores(frame,suffix="FFLAT")
if args.sky!=None :
# read sky
skymodel=read_sky(args.sky)
if args.cosmics_nsig>0 :
# use a copy the frame (not elegant but robust)
copied_frame = copy.deepcopy(frame)
# first subtract sky without throughput correction
subtract_sky(copied_frame, skymodel, apply_throughput_correction = False)
# then find cosmics
log.info("cosmics ray 1D rejection after sky subtraction")
reject_cosmic_rays_1d(copied_frame,args.cosmics_nsig)
# copy mask
frame.mask = copied_frame.mask
# and (re-)subtract sky, but just the correction term
subtract_sky(frame, skymodel, apply_throughput_correction = (not args.no_sky_throughput_correction) )
else :
# subtract sky
subtract_sky(frame, skymodel, apply_throughput_correction = (not args.no_sky_throughput_correction) )
compute_and_append_frame_scores(frame,suffix="SKYSUB")
if args.calib!=None :
log.info("calibrate")
# read calibration
fluxcalib=read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
# Ensure that ivars are set to 0 for all values if any designated
# fibermask bit is set. Also flips a bits for each frame.mask value using specmask.BADFIBER
frame = get_fiberbitmasked_frame(frame,bitmask="flux",ivar_framemask=True)
compute_and_append_frame_scores(frame,suffix="CALIB")
# save output
write_frame(args.outfile, frame, units='10**-17 erg/(s cm2 Angstrom)')
log.info("successfully wrote %s"%args.outfile)
def get_skyres(cframes, sub_sky=False, flatten=True):
"""
Args:
cframes: str or list
Single cframe or a list of them
sub_sky: bool, optional
Subtract the sky? This should probably not be done
flatten: bool, optional
Return a flat, 1D array for each variable
combine: bool, optional
combine the individual sky fibers? Median 'smash'
Returns:
wave : ndarray
flux : ndarray
res : ndarray
ivar : ndarray
"""
from desispec.io import read_frame
from desispec.io.sky import read_sky
from desispec.sky import subtract_sky
if isinstance(cframes, list):
all_wave, all_flux, all_res, all_ivar = [], [], [], []
for cframe_file in cframes:
wave, flux, res, ivar = get_skyres(cframe_file, flatten=flatten)
# Save
all_wave.append(wave)
all_flux.append(flux)
all_res.append(res)
all_ivar.append(ivar)
# Concatenate -- Shape is preserved (nfibers, npix)
twave = np.concatenate(all_wave)
tflux = np.concatenate(all_flux)
tres = np.concatenate(all_res)
tivar = np.concatenate(all_ivar)
# Return
return twave, tflux, tres, tivar
cframe = read_frame(cframes, skip_resolution=True)
if cframe.meta['FLAVOR'] in ['flat', 'arc']:
raise ValueError("Bad flavor for exposure: {:s}".format(cframes))
# Sky
sky_file = cframes.replace('cframe', 'sky')
skymodel = read_sky(sky_file)
if sub_sky:
subtract_sky(cframe, skymodel)
# Resid
skyfibers = np.where(cframe.fibermap['OBJTYPE'] == 'SKY')[0]
res = cframe.flux[skyfibers] # Flux calibrated
ivar = cframe.ivar[skyfibers] # Flux calibrated
flux = skymodel.flux[skyfibers] # Residuals; not flux calibrated!
wave = np.outer(np.ones(flux.shape[0]), cframe.wave)
# Combine?
'''
if combine:
res = np.median(res, axis=0)
ivar = np.median(ivar, axis=0)
flux = np.median(flux, axis=0)
wave = np.median(wave, axis=0)
'''
# Return
if flatten:
return wave.flatten(), flux.flatten(), res.flatten(), ivar.flatten()
else:
return wave, flux, res, ivar
def test_subtract_sky(self):
spectra = self._get_spectra()
sky = compute_sky(spectra,add_variance=self.add_variance)
subtract_sky(spectra, sky)
#- allow some slop in the sky subtraction
self.assertTrue(np.allclose(spectra.flux, 0, rtol=1e-5, atol=1e-6))
def run_pa(self,input_frame,skymodel):
from desispec.sky import subtract_sky
subtract_sky(input_frame,skymodel)
return input_frame
def main(args) :
log=get_logger()
cmd = ['desi_compute_fluxcalibration',]
for key, value in args.__dict__.items():
if value is not None:
cmd += ['--'+key, str(value)]
cmd = ' '.join(cmd)
log.info(cmd)
log.info("read frame")
# read frame
frame = read_frame(args.infile)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel=read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux,model_wave,model_fibers,model_metadata=read_stdstar_models(args.models)
if args.chi2cut > 0 :
ok = np.where(model_metadata["CHI2DOF"]<args.chi2cut)[0]
if ok.size == 0 :
log.error("chi2cut has discarded all stars")
sys.exit(12)
nstars=model_flux.shape[0]
nbad=nstars-ok.size
if nbad>0 :
log.warning("discarding %d star(s) out of %d because of chi2cut"%(nbad,nstars))
model_flux=model_flux[ok]
model_fibers=model_fibers[ok]
model_metadata=model_metadata[:][ok]
if args.delta_color_cut > 0 :
ok = np.where(np.abs(model_metadata["MODEL_G-R"]-model_metadata["DATA_G-R"])<args.delta_color_cut)[0]
nstars=model_flux.shape[0]
nbad=nstars-ok.size
if nbad>0 :
log.warning("discarding %d star(s) out of %d because |delta_color|>%f"%(nbad,nstars,args.delta_color_cut))
model_flux=model_flux[ok]
model_fibers=model_fibers[ok]
model_metadata=model_metadata[:][ok]
# automatically reject stars that ar chi2 outliers
if args.chi2cut_nsig > 0 :
mchi2=np.median(model_metadata["CHI2DOF"])
rmschi2=np.std(model_metadata["CHI2DOF"])
maxchi2=mchi2+args.chi2cut_nsig*rmschi2
ok=np.where(model_metadata["CHI2DOF"]<=maxchi2)[0]
nstars=model_flux.shape[0]
nbad=nstars-ok.size
if nbad>0 :
log.warning("discarding %d star(s) out of %d because reduced chi2 outliers (at %d sigma, giving rchi2<%f )"%(nbad,nstars,args.chi2cut_nsig,maxchi2))
model_flux=model_flux[ok]
model_fibers=model_fibers[ok]
model_metadata=model_metadata[:][ok]
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
## check whether star fibers from args.models are consistent with fibers from fibermap
## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
fibermap_std_indices = np.where(isStdStar(fibermap['DESI_TARGET']))[0]
if np.any(~np.in1d(model_fibers%500, fibermap_std_indices)):
for i in model_fibers%500:
log.error("inconsistency with spectrum {}, OBJTYPE='{}', DESI_TARGET={} in fibermap".format(
(i, fibermap["OBJTYPE"][i], fibermap["DESI_TARGET"][i])))
sys.exit(12)
fluxcalib = compute_flux_calibration(frame, model_wave, model_flux, model_fibers%500)
# QA
if (args.qafile is not None):
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR'])
# Run
#import pdb; pdb.set_trace()
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s"%args.outfile)
def main(args):
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is
None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
frame = read_frame(args.infile)
#- Raw scores already added in extraction, but just in case they weren't
#- it is harmless to rerun to make sure we have them.
compute_and_append_frame_scores(frame, suffix="RAW")
if args.cosmics_nsig > 0 and args.sky == None: # Reject cosmics (otherwise do it after sky subtraction)
log.info("cosmics ray 1D rejection")
reject_cosmic_rays_1d(frame, args.cosmics_nsig)
if args.fiberflat != None:
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to all fibers
apply_fiberflat(frame, fiberflat)
compute_and_append_frame_scores(frame, suffix="FFLAT")
if args.sky != None:
# read sky
skymodel = read_sky(args.sky)
if args.cosmics_nsig > 0:
# first subtract sky without throughput correction
subtract_sky(frame, skymodel, throughput_correction=False)
# then find cosmics
log.info("cosmics ray 1D rejection after sky subtraction")
reject_cosmic_rays_1d(frame, args.cosmics_nsig)
if args.sky_throughput_correction:
# and (re-)subtract sky, but just the correction term
subtract_sky(frame,
skymodel,
throughput_correction=True,
default_throughput_correction=0.)
else:
# subtract sky
subtract_sky(frame,
skymodel,
throughput_correction=args.sky_throughput_correction)
compute_and_append_frame_scores(frame, suffix="SKYSUB")
if args.calib != None:
log.info("calibrate")
# read calibration
fluxcalib = read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
compute_and_append_frame_scores(frame, suffix="CALIB")
# save output
write_frame(args.outfile, frame, units='1e-17 erg/(s cm2 Angstrom)')
log.info("successfully wrote %s" % args.outfile)
def frame_skyres(outfil, frame, skymodel, qaframe, quick_look=False):
"""
Generate QA plots and files for sky residuals of a given frame
Parameters
----------
outfil: str
Name of output file
frame: Frame object
skymodel: SkyModel object
qaframe: QAFrame object
"""
from desispec.sky import subtract_sky
# Access metrics
'''
wavg_ivar = np.sum(res_ivar,0)
chi2_wavg = np.sum(wavg_res**2 * wavg_ivar)
dof_wavg = np.sum(wavg_ivar > 0.)
pchi2_wavg = scipy.stats.distributions.chi2.sf(chi2_wavg, dof_wavg)
chi2_med = np.sum(med_res**2 * wavg_ivar)
pchi2_med = scipy.stats.distributions.chi2.sf(chi2_med, dof_wavg)
'''
skyfibers = np.array(qaframe.qa_data['SKYSUB']["METRICS"]["SKYFIBERID"])
subtract_sky(frame, skymodel)
res=frame.flux[skyfibers]
res_ivar=frame.ivar[skyfibers]
if quick_look:
med_res = qaframe.qa_data['SKYSUB']["METRICS"]["MED_RESID_WAVE"]
wavg_res = qaframe.qa_data['SKYSUB']["METRICS"]["WAVG_RES_WAVE"]
else:
med_res = np.median(res,axis=0)
wavg_res = np.sum(res*res_ivar,0) / np.sum(res_ivar,0)
# Plot
if quick_look:
fig = plt.figure(figsize=(8, 10.0))
gs = gridspec.GridSpec(4,2)
else:
fig = plt.figure(figsize=(8, 6.0))
gs = gridspec.GridSpec(2,2)
xmin,xmax = np.min(frame.wave), np.max(frame.wave)
# Simple residual plot
ax0 = plt.subplot(gs[0,:])
ax0.plot(frame.wave, med_res, label='Median Res')
ax0.plot(frame.wave, signal.medfilt(med_res,51), color='black', label='Median**2 Res')
ax0.plot(frame.wave, signal.medfilt(wavg_res,51), color='red', label='Med WAvgRes')
#
ax0.plot([xmin,xmax], [0., 0], '--', color='gray')
ax0.plot([xmin,xmax], [0., 0], '--', color='gray')
ax0.set_xlabel('Wavelength')
ax0.set_ylabel('Sky Residuals (Counts)')
ax0.set_xlim(xmin,xmax)
ax0.set_xlabel('Wavelength')
ax0.set_ylabel('Sky Residuals (Counts)')
ax0.set_xlim(xmin,xmax)
med0 = np.maximum(np.abs(np.median(med_res)), 1.)
ax0.set_ylim(-5.*med0, 5.*med0)
#ax0.text(0.5, 0.85, 'Sky Meanspec',
# transform=ax_flux.transAxes, ha='center')
# Legend
legend = ax0.legend(loc='upper right', borderpad=0.3,
handletextpad=0.3, fontsize='small')
# Histogram of all residuals
ax1 = plt.subplot(gs[1,0])
xmin,xmax = -5., 5.
# Histogram
binsz = qaframe.qa_data['SKYSUB']["PARAMS"]["BIN_SZ"]
if 'DEVS_1D' in qaframe.qa_data['SKYSUB']["METRICS"].keys(): # Online
hist = np.asarray(qaframe.qa_data['SKYSUB']["METRICS"]["DEVS_1D"])
edges = np.asarray(qaframe.qa_data['SKYSUB']["METRICS"]["DEVS_EDGES"])
else: # Generate for offline
gd_res = res_ivar > 0.
devs = res[gd_res] * np.sqrt(res_ivar[gd_res])
i0, i1 = int( np.min(devs) / binsz) - 1, int( np.max(devs) / binsz) + 1
rng = tuple( binsz*np.array([i0,i1]) )
nbin = i1-i0
hist, edges = np.histogram(devs, range=rng, bins=nbin)
xhist = (edges[1:] + edges[:-1])/2.
ax1.hist(xhist, color='blue', bins=edges, weights=hist)#, histtype='step')
# PDF for Gaussian
area = binsz * np.sum(hist)
xppf = np.linspace(scipy.stats.norm.ppf(0.0001), scipy.stats.norm.ppf(0.9999), 100)
ax1.plot(xppf, area*scipy.stats.norm.pdf(xppf), 'r-', alpha=1.0)
ax1.set_xlabel(r'Res/$\sigma$')
ax1.set_ylabel('N')
ax1.set_xlim(xmin,xmax)
# Meta text
#- limit the dictionary to residuals only for meta
qaresid=copy.deepcopy(qaframe)
resid_keys=['NREJ','NSKY_FIB','NBAD_PCHI','MED_RESID','RESID_PER']
qaresid.qa_data['SKYSUB']['METRICS']={key:value for key,value in qaframe.qa_data['SKYSUB']
['METRICS'].items() if key in resid_keys}
ax2 = plt.subplot(gs[1,1])
ax2.set_axis_off()
show_meta(ax2, qaresid, 'SKYSUB', outfil)
if quick_look:
#- SNR Plot
elg_snr_mag = qaframe.qa_data['SKYSUB']["METRICS"]["ELG_SNR_MAG"]
lrg_snr_mag = qaframe.qa_data['SKYSUB']["METRICS"]["LRG_SNR_MAG"]
qso_snr_mag = qaframe.qa_data['SKYSUB']["METRICS"]["QSO_SNR_MAG"]
star_snr_mag = qaframe.qa_data['SKYSUB']["METRICS"]["STAR_SNR_MAG"]
ax3 = plt.subplot(gs[2,0])
ax4 = plt.subplot(gs[2,1])
ax5 = plt.subplot(gs[3,0])
ax6 = plt.subplot(gs[3,1])
ax3.set_ylabel(r'Median S/N')
ax3.set_xlabel('')
ax3.set_title(r'ELG')
if len(elg_snr_mag[1]) > 0: #- at least 1 elg fiber?
select=np.where((elg_snr_mag[1] != np.array(None)) & (~np.isnan(elg_snr_mag[1])) & (np.abs(elg_snr_mag[1])!=np.inf))[0] #- Remove None, nan and inf values in mag
if select.shape[0]>0:
xmin=np.min(elg_snr_mag[1][select])-0.1
xmax=np.max(elg_snr_mag[1][select])+0.1
ax3.set_xlim(xmin,xmax)
ax3.set_ylim(np.min(elg_snr_mag[0][select])-0.1,np.max(elg_snr_mag[0][select])+0.1)
ax3.xaxis.set_ticks(np.arange(int(np.min(elg_snr_mag[1][select])),int(np.max(elg_snr_mag[1][select]))+1,0.5))
ax3.tick_params(axis='x',labelsize=10,labelbottom='on')
ax3.tick_params(axis='y',labelsize=10,labelleft='on')
ax3.plot(elg_snr_mag[1][select],elg_snr_mag[0][select],'b.')
ax4.set_ylabel('')
ax4.set_xlabel('')
ax4.set_title(r'LRG')
if len(lrg_snr_mag[1]) > 0: #- at least 1 lrg fiber?
select=np.where((lrg_snr_mag[1] != np.array(None)) & (~np.isnan(lrg_snr_mag[1])) & (np.abs(lrg_snr_mag[1])!=np.inf))[0]
if select.shape[0]>0:
xmin=np.min(lrg_snr_mag[1][select])-0.1
xmax=np.max(lrg_snr_mag[1][select])+0.1
ax4.set_xlim(xmin,xmax)
ax4.set_ylim(np.min(lrg_snr_mag[0][select])-0.1,np.max(lrg_snr_mag[0][select])+0.1)
ax4.xaxis.set_ticks(np.arange(int(np.min(lrg_snr_mag[1][select])),int(np.max(lrg_snr_mag[1][select]))+1,0.5))
ax4.tick_params(axis='x',labelsize=10,labelbottom='on')
ax4.tick_params(axis='y',labelsize=10,labelleft='on')
ax4.plot(lrg_snr_mag[1][select],lrg_snr_mag[0][select],'r.')
ax5.set_ylabel(r'Median S/N')
ax5.set_xlabel(r'Mag. (DECAM_R)')
ax5.set_title(r'QSO')
if len(qso_snr_mag[1]) > 0: #- at least 1 qso fiber?
select=np.where((qso_snr_mag[1] != np.array(None)) & (~np.isnan(qso_snr_mag[1])) & (np.abs(qso_snr_mag[1])!=np.inf))[0] #- Remove None, nan and inf values
if select.shape[0]>0:
xmin=np.min(qso_snr_mag[1][select])-0.1
xmax=np.max(qso_snr_mag[1][select])+0.1
ax5.set_xlim(xmin,xmax)
ax5.set_ylim(np.min(qso_snr_mag[0][select])-0.1,np.max(qso_snr_mag[0][select])+0.1)
ax5.xaxis.set_ticks(np.arange(int(np.min(qso_snr_mag[1][select])),int(np.max(qso_snr_mag[1][select]))+1,1.0))
ax5.tick_params(axis='x',labelsize=10,labelbottom='on')
ax5.tick_params(axis='y',labelsize=10,labelleft='on')
ax5.plot(qso_snr_mag[1][select],qso_snr_mag[0][select],'g.')
ax6.set_ylabel('')
ax6.set_xlabel('Mag. (DECAM_R)')
ax6.set_title(r'STD')
if len(star_snr_mag[1]) > 0: #- at least 1 std fiber?
select=np.where((star_snr_mag[1] != np.array(None)) & (~np.isnan(star_snr_mag[1])) & (np.abs(star_snr_mag[1])!=np.inf))[0]
if select.shape[0]>0:
xmin=np.min(star_snr_mag[1][select])-0.1
xmax=np.max(star_snr_mag[1][select])+0.1
ax6.set_xlim(xmin,xmax)
ax6.set_ylim(np.min(star_snr_mag[0][select])-0.1,np.max(star_snr_mag[0][select])+0.1)
ax6.xaxis.set_ticks(np.arange(int(np.min(star_snr_mag[1][select])),int(np.max(star_snr_mag[1][select]))+1,0.5))
ax6.tick_params(axis='x',labelsize=10,labelbottom='on')
ax6.tick_params(axis='y',labelsize=10,labelleft='on')
ax6.plot(star_snr_mag[1][select],star_snr_mag[0][select],'k.')
"""
# Meta
xlbl = 0.1
ylbl = 0.85
i0 = outfil.rfind('/')
ax2.text(xlbl, ylbl, outfil[i0+1:], color='black', transform=ax2.transAxes, ha='left')
yoff=0.15
for key in sorted(qaframe.data['SKYSUB']['METRICS'].keys()):
if key in ['QA_FIG']:
continue
# Show
ylbl -= yoff
ax2.text(xlbl+0.1, ylbl, key+': '+str(qaframe.data['SKYSUB']['METRICS'][key]),
transform=ax2.transAxes, ha='left', fontsize='small')
"""
'''
# Residuals
scatt_sz = 0.5
ax_res = plt.subplot(gs[1])
ax_res.get_xaxis().set_ticks([]) # Suppress labeling
res = (sky_model - (true_flux*scl))/(true_flux*scl)
rms = np.sqrt(np.sum(res**2)/len(res))
#ax_res.set_ylim(-3.*rms, 3.*rms)
ax_res.set_ylim(-2, 2)
ax_res.set_ylabel('Frac Res')
# Error
#ax_res.plot(true_wave, 2.*ms_sig/sky_model, color='red')
ax_res.scatter(wave,res, marker='o',s=scatt_sz)
ax_res.plot([xmin,xmax], [0.,0], 'g-')
ax_res.set_xlim(xmin,xmax)
# Relative to error
ax_sig = plt.subplot(gs[2])
ax_sig.set_xlabel('Wavelength')
sig_res = (sky_model - (true_flux*scl))/sky_sig
ax_sig.scatter(wave, sig_res, marker='o',s=scatt_sz)
ax_sig.set_ylabel(r'Res $\delta/\sigma$')
ax_sig.set_ylim(-5., 5.)
ax_sig.plot([xmin,xmax], [0.,0], 'g-')
ax_sig.set_xlim(xmin,xmax)
'''
# Finish
plt.tight_layout(pad=0.1,h_pad=0.0,w_pad=0.0)
outfile = makepath(outfil)
plt.savefig(outfil)
plt.close()
print('Wrote QA SkyRes file: {:s}'.format(outfil))
def run_pa(self,input_frame,skymodel):
from desispec.quicklook.quicksky import subtract_sky
sframe=subtract_sky(input_frame,skymodel)
return sframe
def main(args):
log = get_logger()
cmd = [
'desi_compute_fluxcalibration',
]
for key, value in args.__dict__.items():
if value is not None:
cmd += ['--' + key, str(value)]
cmd = ' '.join(cmd)
log.info(cmd)
log.info("read frame")
# read frame
frame = read_frame(args.infile)
# Set fibermask flagged spectra to have 0 flux and variance
frame = get_fiberbitmasked_frame(frame,
bitmask='flux',
ivar_framemask=True)
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat
apply_fiberflat(frame, fiberflat)
log.info("subtract sky")
# read sky
skymodel = read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
log.info("compute flux calibration")
# read models
model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models(
args.models)
ok = np.ones(len(model_metadata), dtype=bool)
if args.chi2cut > 0:
log.info("Apply cut CHI2DOF<{}".format(args.chi2cut))
ok &= (model_metadata["CHI2DOF"] < args.chi2cut)
if args.delta_color_cut > 0:
log.info("Apply cut |delta color|<{}".format(args.delta_color_cut))
ok &= (np.abs(model_metadata["MODEL_G-R"] - model_metadata["DATA_G-R"])
< args.delta_color_cut)
if args.min_color is not None:
log.info("Apply cut DATA_G-R>{}".format(args.min_color))
ok &= (model_metadata["DATA_G-R"] > args.min_color)
if args.chi2cut_nsig > 0:
# automatically reject stars that ar chi2 outliers
mchi2 = np.median(model_metadata["CHI2DOF"])
rmschi2 = np.std(model_metadata["CHI2DOF"])
maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2
log.info("Apply cut CHI2DOF<{} based on chi2cut_nsig={}".format(
maxchi2, args.chi2cut_nsig))
ok &= (model_metadata["CHI2DOF"] <= maxchi2)
ok = np.where(ok)[0]
if ok.size == 0:
log.error("cuts discarded all stars")
sys.exit(12)
nstars = model_flux.shape[0]
nbad = nstars - ok.size
if nbad > 0:
log.warning("discarding %d star(s) out of %d because of cuts" %
(nbad, nstars))
model_flux = model_flux[ok]
model_fibers = model_fibers[ok]
model_metadata = model_metadata[:][ok]
# check that the model_fibers are actually standard stars
fibermap = frame.fibermap
## check whether star fibers from args.models are consistent with fibers from fibermap
## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
fibermap_std_indices = np.where(isStdStar(fibermap))[0]
if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)):
target_colnames, target_masks, survey = main_cmx_or_sv(fibermap)
colname = target_colnames[0]
for i in model_fibers % 500:
log.error(
"inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap"
.format(i, fibermap["OBJTYPE"][i], colname,
fibermap[colname][i]))
sys.exit(12)
# Make sure the fibers of interest aren't entirely masked.
if np.sum(
np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) ==
frame.nwave) == len(model_fibers):
log.warning('All standard-star spectra are masked!')
return
fluxcalib = compute_flux_calibration(
frame,
model_wave,
model_flux,
model_fibers % 500,
highest_throughput_nstars=args.highest_throughput)
# QA
if (args.qafile is not None):
log.info("performing fluxcalib QA")
# Load
qaframe = load_qa_frame(args.qafile,
frame_meta=frame.meta,
flavor=frame.meta['FLAVOR'])
# Run
#import pdb; pdb.set_trace()
qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
# Write
if args.qafile is not None:
write_qa_frame(args.qafile, qaframe)
log.info("successfully wrote {:s}".format(args.qafile))
# Figure(s)
if args.qafig is not None:
qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)
# write result
write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)
log.info("successfully wrote %s" % args.outfile)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--infile',
type=str,
default=None,
required=True,
help='path of DESI exposure frame fits file')
parser.add_argument('--fiberflat',
type=str,
default=None,
help='path of DESI fiberflat fits file')
parser.add_argument('--sky',
type=str,
default=None,
help='path of DESI sky fits file')
parser.add_argument('--calib',
type=str,
default=None,
help='path of DESI calibration fits file')
parser.add_argument('--outfile',
type=str,
default=None,
required=True,
help='path of DESI sky fits file')
# add calibration here when exists
args = parser.parse_args()
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is
None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
frame = read_frame(args.infile)
if args.fiberflat != None:
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to sky fibers
apply_fiberflat(frame, fiberflat)
if args.sky != None:
log.info("subtract sky")
# read sky
skymodel = read_sky(args.sky)
# subtract sky
subtract_sky(frame, skymodel)
if args.calib != None:
log.info("calibrate")
# read calibration
fluxcalib = read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
# save output
write_frame(args.outfile, frame)
log.info("successfully wrote %s" % args.outfile)
def main(args):
log = get_logger()
if (args.fiberflat is None) and (args.sky is None) and (args.calib is
None):
log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
sys.exit(12)
if (not args.no_tsnr) and (args.calib is None):
log.critical(
'need --fiberflat --sky and --calib to compute template SNR')
sys.exit(12)
frame = read_frame(args.infile)
if not args.no_tsnr:
# tsnr alpha calc. requires uncalibrated + no substraction rame.
uncalibrated_frame = copy.deepcopy(frame)
#- Raw scores already added in extraction, but just in case they weren't
#- it is harmless to rerun to make sure we have them.
compute_and_append_frame_scores(frame, suffix="RAW")
if args.cosmics_nsig > 0 and args.sky == None: # Reject cosmics (otherwise do it after sky subtraction)
log.info("cosmics ray 1D rejection")
reject_cosmic_rays_1d(frame, args.cosmics_nsig)
if args.fiberflat != None:
log.info("apply fiberflat")
# read fiberflat
fiberflat = read_fiberflat(args.fiberflat)
# apply fiberflat to all fibers
apply_fiberflat(frame, fiberflat)
compute_and_append_frame_scores(frame, suffix="FFLAT")
else:
fiberflat = None
if args.no_xtalk:
zero_ivar = (not args.no_zero_ivar)
else:
zero_ivar = False
if args.sky != None:
# read sky
skymodel = read_sky(args.sky)
if args.cosmics_nsig > 0:
# use a copy the frame (not elegant but robust)
copied_frame = copy.deepcopy(frame)
# first subtract sky without throughput correction
subtract_sky(copied_frame,
skymodel,
apply_throughput_correction=False,
zero_ivar=zero_ivar)
# then find cosmics
log.info("cosmics ray 1D rejection after sky subtraction")
reject_cosmic_rays_1d(copied_frame, args.cosmics_nsig)
# copy mask
frame.mask = copied_frame.mask
# and (re-)subtract sky, but just the correction term
subtract_sky(frame,
skymodel,
apply_throughput_correction=(
not args.no_sky_throughput_correction),
zero_ivar=zero_ivar)
else:
# subtract sky
subtract_sky(frame,
skymodel,
apply_throughput_correction=(
not args.no_sky_throughput_correction),
zero_ivar=zero_ivar)
compute_and_append_frame_scores(frame, suffix="SKYSUB")
if not args.no_xtalk:
log.info("fiber crosstalk correction")
correct_fiber_crosstalk(frame, fiberflat)
if not args.no_zero_ivar:
frame.ivar *= (frame.mask == 0)
if args.calib != None:
log.info("calibrate")
# read calibration
fluxcalib = read_flux_calibration(args.calib)
# apply calibration
apply_flux_calibration(frame, fluxcalib)
# Ensure that ivars are set to 0 for all values if any designated
# fibermask bit is set. Also flips a bits for each frame.mask value using specmask.BADFIBER
frame = get_fiberbitmasked_frame(
frame, bitmask="flux", ivar_framemask=(not args.no_zero_ivar))
compute_and_append_frame_scores(frame, suffix="CALIB")
if not args.no_tsnr:
log.info("calculating tsnr")
results, alpha = calc_tsnr2(uncalibrated_frame,
fiberflat=fiberflat,
skymodel=skymodel,
fluxcalib=fluxcalib,
alpha_only=args.alpha_only)
frame.meta['TSNRALPH'] = alpha
comments = {k: "from calc_frame_tsnr" for k in results.keys()}
append_frame_scores(frame, results, comments, overwrite=True)
# record inputs
frame.meta['IN_FRAME'] = shorten_filename(args.infile)
frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat)
frame.meta['IN_SKY'] = shorten_filename(args.sky)
frame.meta['IN_CALIB'] = shorten_filename(args.calib)
# save output
write_frame(args.outfile, frame, units='10**-17 erg/(s cm2 Angstrom)')
log.info("successfully wrote %s" % args.outfile)
