def spline_fit(output_wave,input_wave,input_flux,required_resolution,input_ivar=None,order=3,max_resolution=None):
"""Performs spline fit of input_flux vs. input_wave and resamples at output_wave
Args:
output_wave : 1D array of output wavelength samples
input_wave : 1D array of input wavelengths
input_flux : 1D array of input flux density
required_resolution (float) : resolution for spline knot placement (same unit as wavelength)
Options:
input_ivar : 1D array of weights for input_flux
order (int) : spline order
max_resolution (float) : if not None and first fit fails, try once this resolution
Returns:
output_flux : 1D array of flux sampled at output_wave
"""
if input_ivar is not None :
selection=np.where(input_ivar>0)[0]
if selection.size < 2 :
log=get_logger()
log.error("cannot do spline fit because only {0:d} values with ivar>0".format(selection.size))
raise ValueError
w1=input_wave[selection[0]]
w2=input_wave[selection[-1]]
else :
w1=input_wave[0]
w2=input_wave[-1]
res=required_resolution
n=int((w2-w1)/res)
res=(w2-w1)/(n+1)
knots=w1+res*(0.5+np.arange(n))
## check that nodes are close to pixels
dknots = abs(knots[:,None]-input_wave)
mins = np.amin(dknots,axis=1)
w=mins<res
knots = knots[w]
try :
toto=scipy.interpolate.splrep(input_wave,input_flux,w=input_ivar,k=order,task=-1,t=knots)
output_flux = scipy.interpolate.splev(output_wave,toto)
except ValueError as err :
log=get_logger()
if max_resolution is not None and required_resolution < max_resolution :
log.warning("spline fit failed with resolution={}, retrying with {}".format(required_resolution,max_resolution))
return spline_fit(output_wave,input_wave,input_flux,max_resolution,input_ivar=input_ivar,order=3,max_resolution=None)
else :
log.error("spline fit failed")
raise ValueError
return output_flux
def sim(night, nspec=5, clobber=False):
"""
Simulate data as part of the integration test.
Args:
night (str): YEARMMDD
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
# Create input fibermaps, spectra, and pixel-level raw data
for expid, program in zip([0,1,2], ['flat', 'arc', 'dark']):
cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format(
expid=expid, program=program, nspec=nspec, night=night)
fibermap = io.findfile('fibermap', night, expid)
simspec = '{}/simspec-{:08d}.fits'.format(os.path.dirname(fibermap), expid)
inputs = []
outputs = [fibermap, simspec]
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('newexp-random failed for {} exposure {}'.format(program, expid))
cmd = "pixsim --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, nspec=nspec, night=night)
inputs = [fibermap, simspec]
outputs = list()
outputs.append(fibermap.replace('fibermap-', 'simpix-'))
outputs.append(io.findfile('raw', night, expid))
if runcmd(cmd, inputs=inputs, outputs=outputs, clobber=clobber) != 0:
raise RuntimeError('pixsim failed for {} exposure {}'.format(program, expid))
return
def init_fluxcalib(self, re_init=False):
""" Initialize parameters for FLUXCALIB QA
Args:
re_init: bool, (optional)
Re-initialize parameter dict
Returns:
"""
log=get_logger()
assert self.flavor == 'science'
# Standard FLUXCALIB input parameters
flux_dict = dict(ZP_WAVE=0., # Wavelength for ZP evaluation (camera dependent)
MAX_ZP_OFF=0.2, # Max offset in ZP for individual star
)
if self.camera[0] == 'b':
flux_dict['ZP_WAVE'] = 4800. # Ang
elif self.camera[0] == 'r':
flux_dict['ZP_WAVE'] = 6500. # Ang
elif self.camera[0] == 'z':
flux_dict['ZP_WAVE'] = 8250. # Ang
else:
log.error("Not ready for camera {}!".format(self.camera))
# Init
self.init_qatype('FLUXCALIB', flux_dict, re_init=re_init)
def check_env():
"""Check required environment variables.
Raises:
RuntimeError if any script fails
"""
log = get_logger()
#- template locations
missing_env = False
if 'DESI_BASIS_TEMPLATES' not in os.environ:
log.warning('missing $DESI_BASIS_TEMPLATES needed for simulating spectra')
missing_env = True
if not os.path.isdir(os.getenv('DESI_BASIS_TEMPLATES')):
log.warning('missing $DESI_BASIS_TEMPLATES directory')
log.warning('e.g. see NERSC:/project/projectdirs/desi/spectro/templates/basis_templates/v2.2')
missing_env = True
for name in (
'DESI_SPECTRO_SIM', 'DESI_SPECTRO_REDUX', 'PIXPROD', 'SPECPROD'):
if name not in os.environ:
log.warning("missing ${0}".format(name))
missing_env = True
if missing_env:
log.warning("Why are these needed?")
log.warning(" Simulations written to $DESI_SPECTRO_SIM/$PIXPROD/")
log.warning(" Raw data read from $DESI_SPECTRO_DATA/")
log.warning(" Spectro pipeline output written to $DESI_SPECTRO_REDUX/$SPECPROD/")
log.warning(" Templates are read from $DESI_BASIS_TEMPLATES")
log.critical("missing env vars; exiting without running pipeline")
raise RuntimeError("missing env vars; exiting without running pipeline")
#- Override $DESI_SPECTRO_DATA to match $DESI_SPECTRO_SIM/$PIXPROD
os.environ['DESI_SPECTRO_DATA'] = os.path.join(os.getenv('DESI_SPECTRO_SIM'), os.getenv('PIXPROD'))
def slurp_nights(self, make_frameqa=False, remove=True, write_nights=False, **kwargs):
""" Slurp all the individual QA files, night by night
Loops on nights, generating QANight objects along the way
Args:
make_frameqa: bool, optional
Regenerate the individual QA files (at the frame level first)
remove: bool, optional
Remove the individual QA files?
Returns:
"""
log = get_logger()
# Remake?
if make_frameqa:
self.make_frameqa(**kwargs)
# Reset
log.info("Resetting QA_Night objects")
self.qa_nights = []
# Loop on nights
for night in self.mexp_dict.keys():
qaNight = QA_Night(night)
qaNight.slurp(remove=remove)
# Save nights
self.qa_nights.append(qaNight)
# Write?
if write_nights:
qaNight.write_qa_exposures()
def search_for_framefile(frame_file):
""" Search for an input frame_file in the desispec redux hierarchy
Args:
frame_file: str
Returns:
mfile: str, full path to frame_file if found else raise error
"""
log=get_logger()
# Parse frame file
path, ifile = os.path.split(frame_file)
splits = ifile.split('-')
root = splits[0]
camera = splits[1]
fexposure = int(splits[2].split('.')[0])
# Loop on nights
nights = get_nights()
for night in nights:
for exposure in get_exposures(night):
if exposure == fexposure:
mfile = findfile(root, camera=camera, night=night, expid=exposure)
if os.path.isfile(mfile):
return mfile
else:
log.error("Expected file {:s} not found..".format(mfile))
def compatible(head1,head2) :
log=get_logger()
for k in ["PSFTYPE","NPIX_X","NPIX_Y","HSIZEX","HSIZEY","FIBERMAX","FIBERMIN","FIBERMAX","NPARAMS","LEGDEG","GHDEGX","GHDEGY"] :
if (head1[k] != head2[k]) :
log.warning("different %s : %s , %s"%(k,head1[k],head2[k]))
return False
return True
def slurp(self, make_frameqa=False, remove=True, **kwargs):
""" Slurp all the individual QA files to generate
a list of QA_Exposure objects
Args:
make_frameqa: bool, optional
Regenerate the individual QA files (at the frame level first)
remove: bool, optional
Remove the individual QA files?
Returns:
"""
from desispec.qa import QA_Exposure
log = get_logger()
# Remake?
if make_frameqa:
self.make_frameqa(**kwargs)
# Loop on nights
# Reset
log.info("Resetting QA_Exposure objects")
self.qa_exps = []
# Loop
for night in self.mexp_dict.keys():
# Loop on exposures
for exposure in self.mexp_dict[night].keys():
frames_dict = self.mexp_dict[night][exposure]
if len(frames_dict) == 0:
continue
# Load any frame (for the type)
qa_exp = QA_Exposure(exposure, night,
specprod_dir=self.specprod_dir, remove=remove)
# Append
self.qa_exps.append(qa_exp)
def sim(night, nspec=25, clobber=False):
"""
Simulate data as part of the integration test.
Args:
night (str): YEARMMDD
nspec (int, optional): number of spectra to include
clobber (bool, optional): rerun steps even if outputs already exist
Raises:
RuntimeError if any script fails
"""
log = logging.get_logger()
output_dir = os.path.join('$DESI_SPECTRO_REDUX','calib2d')
# Create input fibermaps, spectra, and quickgen data
for expid, program in zip([0,1,2], ['flat', 'arc', 'dark']):
cmd = "newexp-random --program {program} --nspec {nspec} --night {night} --expid {expid}".format(expid=expid, program=program, nspec=nspec, night=night)
simspec = desisim.io.findfile('simspec', night, expid)
fibermap = '{}/fibermap-{:08d}.fits'.format(os.path.dirname(simspec),expid)
if runcmd(cmd, clobber=clobber) != 0:
raise RuntimeError('newexp failed for {} exposure {}'.format(program, expid))
cmd = "quickgen --simspec {} --fibermap {}".format(simspec,fibermap)
if runcmd(cmd, clobber=clobber) != 0:
raise RuntimeError('quickgen failed for {} exposure {}'.format(program, expid))
return
def _run_defaults(self):
"""See BaseTask.run_defaults.
"""
import glob
log = get_logger()
opts = {}
starmodels = None
if "DESI_BASIS_TEMPLATES" in os.environ:
filenames = sorted(glob.glob(os.environ["DESI_BASIS_TEMPLATES"]+"/stdstar_templates_*.fits"))
if len(filenames) > 0 :
starmodels = filenames[-1]
else:
filenames = sorted(glob.glob(os.environ["DESI_BASIS_TEMPLATES"]+"/star_templates_*.fits"))
log.warning('Unable to find stdstar templates in {}; using star templates instead'.format(
os.getenv('DESI_BASIS_TEMPLATES')))
if len(filenames) > 0 :
starmodels = filenames[-1]
else:
msg = 'Unable to find stdstar or star templates in {}'.format(
os.getenv('DESI_BASIS_TEMPLATES'))
log.error(msg)
raise RuntimeError(msg)
else:
log.error("DESI_BASIS_TEMPLATES not set!")
raise RuntimeError("could not find the stellar templates")
opts["starmodels"] = starmodels
opts["delta-color"] = 0.2
opts["color"] = "G-R"
return opts
def main(args):
log = get_logger()
#- Generate obsconditions with args.program, then override as needed
args.program = args.program.upper()
if args.program in ['ARC', 'FLAT']:
obsconditions = None
else:
obsconditions = desisim.simexp.reference_conditions[args.program]
if args.airmass is not None:
obsconditions['AIRMASS'] = args.airmass
if args.seeing is not None:
obsconditions['SEEING'] = args.seeing
if args.exptime is not None:
obsconditions['EXPTIME'] = args.exptime
if args.moonfrac is not None:
obsconditions['MOONFRAC'] = args.moonfrac
if args.moonalt is not None:
obsconditions['MOONALT'] = args.moonalt
if args.moonsep is not None:
obsconditions['MOONSEP'] = args.moonsep
sim, fibermap, meta, obs, objmeta = desisim.obs.new_exposure(args.program,
nspec=args.nspec, night=args.night, expid=args.expid,
tileid=args.tileid, nproc=args.nproc, seed=args.seed,
obsconditions=obsconditions, outdir=args.outdir)
def _overscan(pix, nsigma=5, niter=3):
'''
returns overscan, readnoise from overscan image pixels
Args:
pix (ndarray) : overscan pixels from CCD image
Optional:
nsigma (float) : number of standard deviations for sigma clipping
niter (int) : number of iterative refits
'''
log=get_logger()
#- normalized median absolute deviation as robust version of RMS
#- see https://en.wikipedia.org/wiki/Median_absolute_deviation
overscan = np.median(pix)
absdiff = np.abs(pix - overscan)
readnoise = 1.4826*np.median(absdiff)
#- input pixels are integers, so iteratively refit
for i in range(niter):
absdiff = np.abs(pix - overscan)
good = absdiff < nsigma*readnoise
if np.sum(good)<5 :
log.error("error in sigma clipping for overscan measurement, return result without clipping")
overscan = np.median(pix)
absdiff = np.abs(pix - overscan)
readnoise = 1.4826*np.median(absdiff)
return overscan,readnoise
overscan = np.mean(pix[good])
readnoise = np.std(pix[good])
#- correct for bias from sigma clipping
readnoise /= _clipped_std_bias(nsigma)
return overscan, readnoise
def obj_s2n_wave(s2n_dict, wv_bins, flux_bins, otype, outfile=None, ax=None):
"""Generate QA of S/N for a given object type
"""
logs = get_logger()
nwv = wv_bins.size
nfx = flux_bins.size
s2n_sum = np.zeros((nwv-1,nfx-1))
s2n_N = np.zeros((nwv-1,nfx-1)).astype(int)
# Loop on exposures+wedges (can do just once if these are identical for each)
for jj, wave in enumerate(s2n_dict['waves']):
w_i = np.digitize(wave, wv_bins) - 1
m_i = np.digitize(s2n_dict['fluxes'][jj], flux_bins) - 1
mmm = []
for ll in range(nfx-1): # Only need to do once
mmm.append(m_i == ll)
#
for kk in range(nwv-1):
all_s2n = s2n_dict['s2n'][jj][:,w_i==kk]
for ll in range(nfx-1):
if np.any(mmm[ll]):
s2n_sum[kk, ll] += np.sum(all_s2n[mmm[ll],:])
s2n_N[kk, ll] += np.sum(mmm[ll]) * all_s2n.shape[1]
sty_otype = get_sty_otype()
# Plot
if ax is None:
fig = plt.figure(figsize=(6, 6.0))
ax= plt.gca()
# Title
fig.suptitle('{:s}: Summary'.format(sty_otype[otype]['lbl']),
fontsize='large')
# Plot em up
wv_cen = (wv_bins + np.roll(wv_bins,-1))/2.
lstys = ['-', '--', '-.', ':', (0, (3, 1, 1, 1))]
mxy = 1e-9
for ss in range(nfx-1):
if np.sum(s2n_N[:,ss]) == 0:
continue
lbl = 'MAG = [{:0.1f},{:0.1f}]'.format(flux_bins[ss], flux_bins[ss+1])
ax.plot(wv_cen[:-1], s2n_sum[:,ss]/s2n_N[:,ss], linestyle=lstys[ss],
label=lbl, color=sty_otype[otype]['color'])
mxy = max(mxy, np.max(s2n_sum[:,ss]/s2n_N[:,ss]))
ax.set_xlabel('Wavelength (Ang)')
#ax.set_xlim(-ylim, ylim)
ax.set_ylabel('Mean S/N per Ang in bins of 20A')
ax.set_yscale("log", nonposy='clip')
ax.set_ylim(0.1, mxy*1.1)
legend = plt.legend(loc='upper left', scatterpoints=1, borderpad=0.3,
handletextpad=0.3, fontsize='medium', numpoints=1)
# Finish
plt.tight_layout(pad=0.2,h_pad=0.2,w_pad=0.3)
plt.subplots_adjust(top=0.92)
if outfile is not None:
plt.savefig(outfile, dpi=600)
print("Wrote: {:s}".format(outfile))
def move_file(filename, dst):
"""Move delivered file from the DTS spool to the final raw data area.
This function will ensure that the destination directory exists.
Parameters
----------
filename : :class:`str`
The name, including full path, of the file to move.
dst : :class:`str`
The destination *directory*.
Returns
-------
:class:`str`
The value returned by :func:`shutil.move`.
"""
from os import mkdir
from os.path import exists, isdir
from shutil import move
from desiutil.log import get_logger
log = get_logger()
if not exists(dst):
log.info("mkdir('{0}', 0o2770)".format(dst))
mkdir(dst, 0o2770)
log.info("move('{0}', '{1}')".format(filename, dst))
return move(filename, dst)
def main():
"""Entry point for :command:`desi_dts_delivery`.
Returns
-------
:class:`int`
An integer suitable for passing to :func:`sys.exit`.
"""
from os import environ
from os.path import dirname, join
from subprocess import Popen
from desiutil.log import get_logger
log = get_logger()
options = parse_delivery()
remote_command = ['desi_{0.nightStatus}_night {0.night}'.format(options)]
if options.prefix is not None:
remote_command = options.prefix + remote_command
remote_command = ('(' +
'; '.join([c + ' &> /dev/null' for c in remote_command]) +
' &)')
command = ['ssh', '-n', '-q', options.nersc_host, remote_command]
log.info("Received file {0.filename} with exposure number {0.exposure:d}.".format(options))
dst = join(environ['DESI_SPECTRO_DATA'], options.night)
log.info("Using {0} as raw data directory.".format(dst))
move_file(options.filename, dst)
exposure_arrived = check_exposure(dst, options.exposure)
if options.nightStatus in ('start', 'end') or exposure_arrived:
log.info("Calling: {0}.".format(' '.join(command)))
proc = Popen(command)
return 0
def load_qa_frame(filename, frame=None, flavor=None):
""" Load an existing QA_Frame or generate one, as needed
Args:
filename: str
frame: Frame object, optional
flavor: str, optional
Type of QA_Frame
Returns:
qa_frame: QA_Frame object
"""
from desispec.qa.qa_frame import QA_Frame
log=get_logger()
if os.path.isfile(filename): # Read from file, if it exists
qaframe = read_qa_frame(filename)
log.info("Loaded QA file {:s}".format(filename))
# Check against frame, if provided
if frame is not None:
for key in ['camera','expid','night','flavor']:
assert getattr(qaframe, key) == frame.meta[key.upper()]
else: # Init
if frame is None:
log.error("QA file {:s} does not exist. Expecting frame input".format(filename))
qaframe = QA_Frame(frame)
# Set flavor?
if flavor is not None:
qaframe.flavor = flavor
# Return
return qaframe
def _insert(self, cursor, props):
"""See BaseTask.insert.
"""
log = get_logger()
name = self.name_join(props)
colstr = '(name'
valstr = "('{}'".format(name)
#cmd='insert or replace into {} values ("{}"'.format(self._type, name)
for k, ktype in zip(self._cols, self._coltypes):
colstr += ', {}'.format(k)
if k == "state":
if k in props:
valstr += ', {}'.format(task_state_to_int[props["state"]])
else:
valstr += ', {}'.format(task_state_to_int["waiting"])
else:
if ktype == "text":
valstr += ", '{}'".format(props[k])
else:
valstr += ', {}'.format(props[k])
colstr += ', submitted)'
valstr += ', 0)'
cmd = 'insert into {} {} values {}'.format(self._type, colstr, valstr)
log.debug(cmd)
cursor.execute(cmd)
return
def _traceset_from_table(wavemin,wavemax,hdu,pname) :
log=get_logger()
head=hdu.header
table=hdu.data
extname=head["EXTNAME"]
i=np.where(table["PARAM"]==pname)[0][0]
if "WAVEMIN" in table.dtype.names :
twavemin=table["WAVEMIN"][i]
if wavemin is not None :
if abs(twavemin-wavemin)>0.001 :
mess="WAVEMIN not matching in hdu {} {}!={}".format(extname,twavemin,wavemin)
log.error(mess)
raise ValueError(mess)
else :
wavemin=twavemin
if "WAVEMAX" in table.dtype.names :
twavemax=table["WAVEMAX"][i]
if wavemax is not None :
if abs(twavemax-wavemax)>0.001 :
mess="WAVEMAX not matching in hdu {} {}!={}".format(extname,twavemax,wavemax)
log.error(mess)
raise ValueError(mess)
else :
wavemax=twavemax
log.info("read {} from hdu {}".format(pname,extname))
return table["COEFF"][i],wavemin,wavemax
def parse(options=None):
parser=argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
#- Required
parser.add_argument('--fiberassign', type=str, required=True,
help="input fiberassign directory or tile file")
parser.add_argument('--mockdir', type=str, required=True,
help="directory with mock targets and truth")
parser.add_argument('--obslist', type=str, required=True,
help="input surveysim obslist file")
parser.add_argument('--expid', type=int, required=True, help="exposure ID")
#- Optional
parser.add_argument('--nside', help='healpixel organization scheme of the mock spectra', type=int, default=64)
parser.add_argument('--outdir', type=str, help="output directory")
parser.add_argument('--nspec', type=int, default=None, help="number of spectra to include")
parser.add_argument('--clobber', action='store_true', help="overwrite any pre-existing output files")
log = get_logger()
if options is None:
args = parser.parse_args()
log.info(' '.join(sys.argv))
else:
args = parser.parse_args(options)
log.info('newexp-mock '+' '.join(options))
return args
def run(self, grph, task, opts, comm=None):
"""
Run the PSF combining.
This is a serial call to libspecex to combine the PSF files.
Args:
grph (dict): pruned graph with this task and dependencies.
task (str): the name of this task.
opts (dict): options to use for this task.
comm (mpi4py.MPI.Comm): optional MPI communicator.
"""
if comm is not None:
if comm.size > 1:
raise RuntimeError("PSFCombine worker should only be called with one process")
log = get_logger()
node = grph[task]
outfile = graph_path(task)
infiles = []
for input in node["in"]:
infiles.append(graph_path(input))
specex.mean_psf(infiles, outfile)
return
def write_xytraceset(outfile,xytraceset) :
"""
Write a traceset fits file and returns path to file written.
Args:
outfile: full path to output file
xytraceset: desispec.xytraceset.XYTraceSet object
Returns:
full filepath of output file that was written
"""
log=get_logger()
outfile = makepath(outfile, 'frame')
hdus = fits.HDUList()
x = fits.PrimaryHDU(xytraceset.x_vs_wave_traceset._coeff.astype('f4'))
x.header['EXTNAME'] = "XTRACE"
hdus.append(x)
hdus.append( fits.ImageHDU(xytraceset.y_vs_wave_traceset._coeff.astype('f4'), name="YTRACE") )
if xytraceset.xsig_vs_wave_traceset is not None : hdus.append( fits.ImageHDU(xytraceset.xsig_vs_wave_traceset._coeff.astype('f4'), name='XSIG') )
if xytraceset.ysig_vs_wave_traceset is not None : hdus.append( fits.ImageHDU(xytraceset.ysig_vs_wave_traceset._coeff.astype('f4'), name='YSIG') )
for hdu in ["XTRACE","YTRACE","XSIG","YSIG"] :
if hdu in hdus :
hdus[hdu].header["WAVEMIN"] = xytraceset.wavemin
hdus[hdu].header["WAVEMAX"] = xytraceset.wavemax
hdus[hdu].header["NPIX_Y"] = xytraceset.npix_y
hdus.writeto(outfile+'.tmp', overwrite=True, checksum=True)
os.rename(outfile+'.tmp', outfile)
log.info("wrote a xytraceset in {}".format(outfile))
return outfile
def __init__(self):
self.log = get_logger()
parser = argparse.ArgumentParser(
description="DESI nightly processing",
usage="""desi_night <command> [options]
Where supported commands are:
update Process an incoming exposure as much as possible. Arc exposures
will trigger PSF estimation. If the nightly PSF exists, then a
flat exposure will be extracted and a fiberflat will be created.
If the nightly PSF exists, then a science exposure will be
extracted. If the nightly fiberflat exists, then a science
exposure will be calibrated.
arcs All arcs are done, proceed with nightly PSF.
flats All flats are done, proceed with nightly fiberflat.
redshifts Regroup spectra and process all updated redshifts.
""")
parser.add_argument("command", help="Subcommand to run")
# parse_args defaults to [1:] for args, but you need to
# exclude the rest of the args too, or validation will fail
args = parser.parse_args(sys.argv[1:2])
if not hasattr(self, args.command):
print("Unrecognized command")
parser.print_help()
sys.exit(errs["usage"])
# use dispatch pattern to invoke method with same name
getattr(self, args.command)()
def default_options(extra={}):
"""
Get the default options for all workers.
Args:
extra (dict): optional extra options to add to the
default options for each worker class.
Returns (dict):
the default options dictionary, suitable for writing
to the default options.yaml file.
"""
log = get_logger()
allopts = {}
for step in step_types:
defwork = default_workers[step]
allopts["{}_worker".format(step)] = defwork
if defwork in extra:
allopts["{}_worker_opts".format(step)] = extra[defwork]
else:
allopts["{}_worker_opts".format(step)] = {}
worker = get_worker(step, None, {})
allopts[step] = worker.default_options()
return allopts
def _option_list(self, name, opts, db):
# we do need db access for spectra
if db is None :
log = get_logger()
log.error("we do need db access for spectra")
raise RuntimeError("we do need db access for spectra")
from .base import task_classes, task_type
# get pixel
props = self.name_split(name)
# get list of exposures and spectrographs by selecting entries in the healpix_frame table with state = 1
# which means that there is a new cframe intersecting the pixel
entries = db.select_healpix_frame({"pixel":props["pixel"],"nside":props["nside"],"state":1})
# now select cframe with same props
cframes = []
for entry in entries :
for band in ["b","r","z"] :
entry_and_band = entry.copy()
entry_and_band["band"] = band
# this will match cframes with same expid and spectro
taskname = task_classes["cframe"].name_join(entry_and_band)
filename = task_classes["cframe"].paths(taskname)[0]
cframes.append(filename)
options = {}
options["infiles"] = cframes
options["outfile"] = self.paths(name)[0]
options["healpix"] = props["pixel"]
options["nside"] = props["nside"]
return option_list(options)
def insert_dlas(wave, zem, rstate=None, seed=None, fNHI=None, debug=False, **kwargs):
""" Insert zero, one or more DLAs into a given spectrum towards a source
with a given redshift
Args:
wave (ndarray): wavelength array in Ang
zem (float): quasar emission redshift
rstate (numpy.random.rstate, optional): for random numberes
seed (int, optional):
fNHI (spline): f_NHI object
**kwargs: Passed to init_fNHI()
Returns:
dlas (list): List of DLA dict's with keys z,N
dla_model (ndarray): normalized specrtrum with DLAs inserted
"""
from scipy import interpolate
log = get_logger()
# Init
if rstate is None:
rstate = np.random.RandomState(seed) # this is breaking the chain of randoms if seed is None
if fNHI is None:
fNHI = init_fNHI(**kwargs)
# Allowed redshift placement
## Cut on zem and 910A rest-frame
zlya = wave/1215.67 - 1
dz = np.roll(zlya,-1)-zlya
dz[-1] = dz[-2]
gdz = (zlya < zem) & (wave > 910.*(1+zem))
# l(z) -- Uses DLA for SLLS too which is fine
lz = calc_lz(zlya[gdz])
cum_lz = np.cumsum(lz*dz[gdz])
tot_lz = cum_lz[-1]
if len(cum_lz)<2:
log.warning('WARNING: cum_lz in insert_dla has only {} element. skyped add DLA.'.format(len(cum_lz)))
dlas,dla_model=[],[]
return dlas,dla_model
fzdla = interpolate.interp1d(cum_lz/tot_lz, zlya[gdz],
bounds_error=False,fill_value=np.min(zlya[gdz]))#
# n DLA
nDLA = rstate.poisson(tot_lz, 1)
# Generate DLAs
dlas = []
for jj in range(nDLA[0]):
# Random z
zabs = float(fzdla(rstate.random_sample()))
# Random NHI
NHI = float(fNHI(rstate.random_sample()))
# Generate and append
dla = dict(z=zabs, N=NHI,dlaid=jj)
dlas.append(dla)
# Generate model of DLAs
dla_model = dla_spec(wave, dlas)
# Return
return dlas, dla_model
def parse(options=None):
parser = argparse.ArgumentParser(description="Fit of standard star spectra in frames.")
parser.add_argument('--frames', type = str, default = None, required=True, nargs='*',
help = 'list of path to DESI frame fits files (needs to be same exposure, spectro)')
parser.add_argument('--skymodels', type = str, default = None, required=True, nargs='*',
help = 'list of path to DESI sky model fits files (needs to be same exposure, spectro)')
parser.add_argument('--fiberflats', type = str, default = None, required=True, nargs='*',
help = 'list of path to DESI fiberflats fits files (needs to be same exposure, spectro)')
parser.add_argument('--starmodels', type = str, help = 'path of spectro-photometric stellar spectra fits')
parser.add_argument('-o','--outfile', type = str, help = 'output file for normalized stdstar model flux')
parser.add_argument('--ncpu', type = int, default = default_nproc, required = False, help = 'use ncpu for multiprocessing')
parser.add_argument('--delta-color', type = float, default = 0.2, required = False, help = 'max delta-color for the selection of standard stars (on top of meas. errors)')
parser.add_argument('--color', type = str, default = "G-R", choices=['G-R', 'R-Z'], required = False, help = 'color for selection of standard stars')
parser.add_argument('--z-max', type = float, default = 0.008, required = False, help = 'max peculiar velocity (blue/red)shift range')
parser.add_argument('--z-res', type = float, default = 0.00002, required = False, help = 'dz grid resolution')
parser.add_argument('--template-error', type = float, default = 0.1, required = False, help = 'fractional template error used in chi2 computation (about 0.1 for BOSS b1)')
log = get_logger()
args = None
if options is None:
args = parser.parse_args()
cmd = ' '.join(sys.argv)
else:
args = parser.parse_args(options)
cmd = 'desi_fit_stdstars ' + ' '.join(options)
log.info('RUNNING {}'.format(cmd))
return args
def __init__(self,wave,flux=None,ivar=None,mask=None,resolution=None):
assert wave.ndim == 1, "Input wavelength should be 1D"
assert (flux is None) or (flux.shape == wave.shape), "wave and flux should have same shape"
assert (ivar is None) or (ivar.shape == wave.shape), "wave and ivar should have same shape"
assert (mask is None) or (mask.shape == wave.shape), "wave and mask should have same shape"
assert (resolution is None) or (isinstance(resolution, desispec.resolution.Resolution))
assert (resolution is None) or (resolution.shape[0] == len(wave)), "resolution size mismatch to wave"
self.wave = wave
self.flux = flux
self.ivar = ivar
self.mask = mask
# if mask is None:
# self.mask = np.zeros(self.flux.shape, dtype=np.uint32)
# else:
# self.mask = util.mask32(mask)
self.resolution = resolution
self.R = resolution #- shorthand
self.log = get_logger()
# Initialize the quantities we will accumulate during co-addition. Note that our
# internal Cinv is a dense matrix.
if ivar is None:
n = len(wave)
self.Cinv = np.zeros((n,n))
self.Cinv_f = np.zeros((n,))
else:
assert flux is not None and resolution is not None,'Missing flux and/or resolution.'
diag_ivar = scipy.sparse.dia_matrix((ivar[np.newaxis,:],[0]),resolution.shape)
self.Cinv = self.resolution.T.dot(diag_ivar.dot(self.resolution))
self.Cinv_f = self.resolution.T.dot(self.ivar*self.flux)
def check_env():
"""
Check required environment variables; raise RuntimeException if missing
"""
log = logging.get_logger()
#- template locations
missing_env = False
if 'DESI_BASIS_TEMPLATES' not in os.environ:
log.warning('missing $DESI_BASIS_TEMPLATES needed for simulating spectra'.format(name))
missing_env = True
if not os.path.isdir(os.getenv('DESI_BASIS_TEMPLATES')):
log.warning('missing $DESI_BASIS_TEMPLATES directory')
log.warning('e.g. see NERSC:/project/projectdirs/desi/spectro/templates/basis_templates/v1.0')
missing_env = True
for name in (
'DESI_SPECTRO_SIM', 'DESI_SPECTRO_REDUX', 'PIXPROD', 'SPECPROD', 'DESIMODEL'):
if name not in os.environ:
log.warning("missing ${0}".format(name))
missing_env = True
if missing_env:
log.warning("Why are these needed?")
log.warning(" Simulations written to $DESI_SPECTRO_SIM/$PIXPROD/")
log.warning(" Raw data read from $DESI_SPECTRO_DATA/")
log.warning(" Spectro pipeline output written to $DESI_SPECTRO_REDUX/$SPECPROD/")
log.warning(" Templates are read from $DESI_BASIS_TEMPLATES")
#- Wait until end to raise exception so that we report everything that
#- is missing before actually failing
if missing_env:
log.critical("missing env vars; exiting without running pipeline")
sys.exit(1)
def merge_psf(inputs, output):
log = get_logger()
npsf = len(inputs)
log.info("Will merge {} PSFs in {}".format(npsf,output))
# we will add/change data to the first PSF
psf_hdulist=fits.open(inputs[0])
for input_filename in inputs[1:] :
log.info("merging {} into {}".format(input_filename,inputs[0]))
other_psf_hdulist=fits.open(input_filename)
# look at what fibers where actually fit
i=np.where(other_psf_hdulist["PSF"].data["PARAM"]=="STATUS")[0][0]
status_of_fibers = \
other_psf_hdulist["PSF"].data["COEFF"][i][:,0].astype(int)
selected_fibers = np.where(status_of_fibers==0)[0]
log.info("fitted fibers in PSF {} = {}".format(input_filename,
selected_fibers))
if selected_fibers.size == 0 :
log.warning("no fiber with status=0 found in {}".format(
input_filename))
other_psf_hdulist.close()
continue
# copy xtrace and ytrace
psf_hdulist["XTRACE"].data[selected_fibers] = \
other_psf_hdulist["XTRACE"].data[selected_fibers]
psf_hdulist["YTRACE"].data[selected_fibers] = \
other_psf_hdulist["YTRACE"].data[selected_fibers]
# copy parameters
parameters = psf_hdulist["PSF"].data["PARAM"]
for param in parameters :
i0=np.where(psf_hdulist["PSF"].data["PARAM"]==param)[0][0]
i1=np.where(other_psf_hdulist["PSF"].data["PARAM"]==param)[0][0]
psf_hdulist["PSF"].data["COEFF"][i0][selected_fibers] = \
other_psf_hdulist["PSF"].data["COEFF"][i1][selected_fibers]
# copy bundle chi2
i = np.where(other_psf_hdulist["PSF"].data["PARAM"]=="BUNDLE")[0][0]
bundles = np.unique(other_psf_hdulist["PSF"].data["COEFF"][i]\
[selected_fibers,0].astype(int))
log.info("fitted bundles in PSF {} = {}".format(input_filename,
bundles))
for b in bundles :
for key in [ "B{:02d}RCHI2".format(b), "B{:02d}NDATA".format(b),
"B{:02d}NPAR".format(b) ]:
psf_hdulist["PSF"].header[key] = \
other_psf_hdulist["PSF"].header[key]
# close file
other_psf_hdulist.close()
# write
psf_hdulist.writeto(output,overwrite=True)
log.info("Wrote PSF {}".format(output))
return
def run_task(name, opts, comm=None, logfile=None, db=None):
"""Run a single task.
Based on the name of the task, call the appropriate run function for that
task. Log output to the specified file. Run using the specified MPI
communicator and optionally update state to the specified database.
Note: This function DOES NOT check the database or filesystem to see if
the task has been completed or if its dependencies exist. It assumes that
some higher-level code has done that if necessary.
Args:
name (str): the name of this task.
opts (dict): options to use for this task.
comm (mpi4py.MPI.Comm): optional MPI communicator.
logfile (str): output log file. If None, do not redirect output to a
file.
db (pipeline.db.DB): The optional database to update.
Returns:
int: the total number of processes that failed.
"""
from .tasks.base import task_classes, task_type
log = get_logger()
ttype = task_type(name)
nproc = 1
rank = 0
if comm is not None:
nproc = comm.size
rank = comm.rank
if rank == 0:
if (logfile is not None) and os.path.isfile(logfile):
os.remove(logfile)
# Mark task as in progress
if db is not None:
task_classes[ttype].state_set(db=db,name=name,state="running")
failcount = 0
if logfile is None:
# No redirection
if db is None:
failcount = task_classes[ttype].run(name, opts, comm=comm)
else:
failcount = task_classes[ttype].run_and_update(db, name, opts,
comm=comm)
else:
with stdouterr_redirected(to=logfile, comm=comm):
if db is None:
failcount = task_classes[ttype].run(name, opts, comm=comm)
else:
failcount = task_classes[ttype].run_and_update(db, name, opts,
comm=comm)
return failcount
def read_frame(filename, nspec=None, skip_resolution=False):
"""Reads a frame fits file and returns its data.
Args:
filename: path to a file, or (night, expid, camera) tuple where
night = string YEARMMDD
expid = integer exposure ID
camera = b0, r1, .. z9
skip_resolution: bool, option
Speed up read time (>5x) by avoiding the Resolution matrix
Returns:
desispec.Frame object with attributes wave, flux, ivar, etc.
"""
log = get_logger()
#- check if filename is (night, expid, camera) tuple instead
if not isinstance(filename, str):
night, expid, camera = filename
filename = findfile('frame', night, expid, camera)
if not os.path.isfile(filename):
raise IOError("cannot open" + filename)
fx = fits.open(filename, uint=True, memmap=False)
hdr = fx[0].header
flux = native_endian(fx['FLUX'].data.astype('f8'))
ivar = native_endian(fx['IVAR'].data.astype('f8'))
wave = native_endian(fx['WAVELENGTH'].data.astype('f8'))
if 'MASK' in fx:
mask = native_endian(fx['MASK'].data)
else:
mask = None #- let the Frame object create the default mask
# Init
resolution_data = None
qwsigma = None
qndiag = None
fibermap = None
chi2pix = None
scores = None
scores_comments = None
if skip_resolution:
pass
elif 'RESOLUTION' in fx:
resolution_data = native_endian(fx['RESOLUTION'].data.astype('f8'))
elif 'QUICKRESOLUTION' in fx:
qr = fx['QUICKRESOLUTION'].header
qndiag = qr['NDIAG']
qwsigma = native_endian(fx['QUICKRESOLUTION'].data.astype('f4'))
if 'FIBERMAP' in fx:
fibermap = Table(fx['FIBERMAP'].data)
if 'DESIGN_X' in fibermap.colnames:
fibermap.rename_column('DESIGN_X', 'FIBERASSIGN_X')
if 'DESIGN_Y' in fibermap.colnames:
fibermap.rename_column('DESIGN_Y', 'FIBERASSIGN_Y')
else:
fibermap = None
if 'CHI2PIX' in fx:
chi2pix = native_endian(fx['CHI2PIX'].data.astype('f8'))
else:
chi2pix = None
if 'SCORES' in fx:
scores = fx['SCORES'].data
# I need to open the header to read the comments
scores_comments = dict()
head = fx['SCORES'].header
for i in range(1, len(scores.columns) + 1):
k = 'TTYPE' + str(i)
scores_comments[head[k]] = head.comments[k]
else:
scores = None
scores_comments = None
fx.close()
if nspec is not None:
flux = flux[0:nspec]
ivar = ivar[0:nspec]
if resolution_data is not None:
resolution_data = resolution_data[0:nspec]
else:
qwsigma = qwsigma[0:nspec]
if chi2pix is not None:
chi2pix = chi2pix[0:nspec]
if mask is not None:
mask = mask[0:nspec]
# return flux,ivar,wave,resolution_data, hdr
frame = Frame(wave,
flux,
ivar,
mask,
resolution_data,
meta=hdr,
fibermap=fibermap,
chi2pix=chi2pix,
scores=scores,
scores_comments=scores_comments,
wsigma=qwsigma,
ndiag=qndiag,
suppress_res_warning=skip_resolution)
# Vette
diagnosis = frame.vet()
if diagnosis != 0:
warnings.warn(
"Frame did not pass simple vetting test. diagnosis={:d}".format(
diagnosis))
log.error(
"Frame did not pass simple vetting test. diagnosis={:d}".format(
diagnosis))
# Return
return frame
def mean_psf(inputs, output):
log = get_logger()
npsf = len(inputs)
log.info("Will compute the average of {} PSFs".format(npsf))
refhead = None
tables = []
xtrace = []
ytrace = []
wavemins = []
wavemaxs = []
hdulist = None
bundle_rchi2 = []
nbundles = None
nfibers_per_bundle = None
for input in inputs:
log.info("Adding {}".format(input))
if not os.path.isfile(input):
log.warning("missing {}".format(input))
continue
psf = fits.open(input)
if refhead is None:
hdulist = psf
refhead = psf["PSF"].header
nfibers = \
(psf["PSF"].header["FIBERMAX"]-psf["PSF"].header["FIBERMIN"])+1
PSFVER = int(refhead["PSFVER"])
if (PSFVER < 3):
log.error("ERROR NEED PSFVER>=3")
sys.exit(1)
else:
if not compatible(psf["PSF"].header, refhead):
log.error("psfs {} and {} are not compatible".format(
inputs[0], input))
sys.exit(12)
tables.append(psf["PSF"].data)
wavemins.append(psf["PSF"].header["WAVEMIN"])
wavemaxs.append(psf["PSF"].header["WAVEMAX"])
if "XTRACE" in psf:
xtrace.append(psf["XTRACE"].data)
if "YTRACE" in psf:
ytrace.append(psf["YTRACE"].data)
rchi2 = []
b = 0
while "B{:02d}RCHI2".format(b) in psf["PSF"].header:
rchi2.append(psf["PSF"].header["B{:02d}RCHI2".format(b)])
b += 1
rchi2 = np.array(rchi2)
nbundles = rchi2.size
bundle_rchi2.append(rchi2)
npsf = len(tables)
bundle_rchi2 = np.array(bundle_rchi2)
log.debug("bundle_rchi2= {}".format(str(bundle_rchi2)))
median_bundle_rchi2 = np.median(bundle_rchi2)
rchi2_threshold = median_bundle_rchi2 + 1.
log.debug("median chi2={} threshold={}".format(median_bundle_rchi2,
rchi2_threshold))
WAVEMIN = refhead["WAVEMIN"]
WAVEMAX = refhead["WAVEMAX"]
FIBERMIN = int(refhead["FIBERMIN"])
FIBERMAX = int(refhead["FIBERMAX"])
fibers_in_bundle = {}
i = np.where(tables[0]["PARAM"] == "BUNDLE")[0][0]
bundle_of_fibers = tables[0]["COEFF"][i][:, 0].astype(int)
bundles = np.unique(bundle_of_fibers)
for b in bundles:
fibers_in_bundle[b] = np.where(bundle_of_fibers == b)[0]
for entry in range(tables[0].size):
PARAM = tables[0][entry]["PARAM"]
log.info("Averaging '{}' coefficients".format(PARAM))
coeff = [tables[0][entry]["COEFF"]]
npar = coeff[0][1].size
for p in range(1, npsf):
if wavemins[p] == WAVEMIN and wavemaxs[p] == WAVEMAX:
coeff.append(tables[p][entry]["COEFF"])
else:
log.info("need to refit legendre polynomial ...")
icoeff = tables[p][entry]["COEFF"]
ocoeff = np.zeros(icoeff.shape)
# need to reshape legpol
iu = np.linspace(-1, 1, npar + 3)
iwavemin = wavemins[p]
iwavemax = wavemaxs[p]
wave = (iu + 1.) / 2. * (iwavemax - iwavemin) + iwavemin
ou = (wave - WAVEMIN) / (WAVEMAX - WAVEMIN) * 2. - 1.
for f in range(icoeff.shape[0]):
val = legval(iu, icoeff[f])
ocoeff[f] = legfit(ou, val, deg=npar - 1)
coeff.append(ocoeff)
coeff = np.array(coeff)
output_rchi2 = np.zeros((bundle_rchi2.shape[1]))
output_coeff = np.zeros(tables[0][entry]["COEFF"].shape)
# now merge, using rchi2 as selection score
for bundle in fibers_in_bundle.keys():
ok = np.where(bundle_rchi2[:, bundle] < rchi2_threshold)[0]
#ok=np.array([0,1]) # debug
if entry == 0:
log.info("for fiber bundle {}, {} valid PSFs".format(
bundle, ok.size))
if ok.size >= 2: # use median
log.debug("bundle #{} : use median".format(bundle))
for f in fibers_in_bundle[bundle]:
output_coeff[f] = np.median(coeff[ok, f], axis=0)
output_rchi2[bundle] = np.median(bundle_rchi2[ok, bundle])
elif ok.size == 1: # copy
log.debug("bundle #{} : use only one psf ".format(bundle))
for f in fibers_in_bundle[bundle]:
output_coeff[f] = coeff[ok[0], f]
output_rchi2[bundle] = bundle_rchi2[ok[0], bundle]
else: # we have a problem here, take the smallest rchi2
log.debug("bundle #{} : take smallest chi2 ".format(bundle))
i = np.argmin(bundle_rchi2[:, bundle])
for f in fibers_in_bundle[bundle]:
output_coeff[f] = coeff[i, f]
output_rchi2[bundle] = bundle_rchi2[i, bundle]
# now copy this in output table
hdulist["PSF"].data["COEFF"][entry] = output_coeff
# change bundle chi2
for bundle in range(output_rchi2.size):
hdulist["PSF"].header["B{:02d}RCHI2".format(bundle)] = \
output_rchi2[bundle]
if len(xtrace) > 0:
xtrace = np.array(xtrace)
ytrace = np.array(ytrace)
for p in range(xtrace.shape[0]):
if wavemins[p] == WAVEMIN and wavemaxs[p] == WAVEMAX:
continue
# need to reshape legpol
iu = np.linspace(-1, 1, npar + 3)
iwavemin = wavemins[p]
iwavemax = wavemaxs[p]
wave = (iu + 1.) / 2. * (iwavemax - iwavemin) + iwavemin
ou = (wave - WAVEMIN) / (WAVEMAX - WAVEMIN) * 2. - 1.
for f in range(icoeff.shape[0]):
val = legval(iu, xtrace[f])
xtrace[f] = legfit(ou, val, deg=npar - 1)
val = legval(iu, ytrace[f])
ytrace[f] = legfit(ou, val, deg=npar - 1)
hdulist["xtrace"].data = np.median(np.array(xtrace), axis=0)
hdulist["ytrace"].data = np.median(np.array(ytrace), axis=0)
# alter other keys in header
hdulist["PSF"].header[
"EXPID"] = 0. # it's a mix, need to add the expids
for hdu in ["XTRACE", "YTRACE", "PSF"]:
if hdu in hdulist:
for input in inputs:
hdulist[hdu].header["comment"] = "inc {}".format(input)
# save output PSF
hdulist.writeto(output, overwrite=True)
log.info("wrote {}".format(output))
return
def merge_psf(inputs, output):
log = get_logger()
npsf = len(inputs)
log.info("Will merge {} PSFs in {}".format(npsf, output))
# we will add/change data to the first PSF
psf_hdulist = fits.open(inputs[0])
for input_filename in inputs[1:]:
log.info("merging {} into {}".format(input_filename, inputs[0]))
other_psf_hdulist = fits.open(input_filename)
# look at what fibers where actually fit
i = np.where(other_psf_hdulist["PSF"].data["PARAM"] == "STATUS")[0][0]
status_of_fibers = \
other_psf_hdulist["PSF"].data["COEFF"][i][:,0].astype(int)
selected_fibers = np.where(status_of_fibers == 0)[0]
log.info("fitted fibers in PSF {} = {}".format(input_filename,
selected_fibers))
if selected_fibers.size == 0:
log.warning(
"no fiber with status=0 found in {}".format(input_filename))
other_psf_hdulist.close()
continue
# copy xtrace and ytrace
psf_hdulist["XTRACE"].data[selected_fibers] = \
other_psf_hdulist["XTRACE"].data[selected_fibers]
psf_hdulist["YTRACE"].data[selected_fibers] = \
other_psf_hdulist["YTRACE"].data[selected_fibers]
# copy parameters
parameters = psf_hdulist["PSF"].data["PARAM"]
for param in parameters:
i0 = np.where(psf_hdulist["PSF"].data["PARAM"] == param)[0][0]
i1 = np.where(
other_psf_hdulist["PSF"].data["PARAM"] == param)[0][0]
psf_hdulist["PSF"].data["COEFF"][i0][selected_fibers] = \
other_psf_hdulist["PSF"].data["COEFF"][i1][selected_fibers]
# copy bundle chi2
i = np.where(other_psf_hdulist["PSF"].data["PARAM"] == "BUNDLE")[0][0]
bundles = np.unique(other_psf_hdulist["PSF"].data["COEFF"][i]\
[selected_fibers,0].astype(int))
log.info("fitted bundles in PSF {} = {}".format(
input_filename, bundles))
for b in bundles:
for key in [
"B{:02d}RCHI2".format(b), "B{:02d}NDATA".format(b),
"B{:02d}NPAR".format(b)
]:
psf_hdulist["PSF"].header[key] = \
other_psf_hdulist["PSF"].header[key]
# close file
other_psf_hdulist.close()
# write
psf_hdulist.writeto(output, overwrite=True)
log.info("Wrote PSF {}".format(output))
return
def main(args, comm=None):
log = get_logger()
imgfile = args.input_image
outfile = args.output_psf
if args.input_psf is not None:
inpsffile = args.input_psf
else:
from desispec.calibfinder import findcalibfile
hdr = fits.getheader(imgfile)
inpsffile = findcalibfile([
hdr,
], 'PSF')
optarray = []
if args.extra is not None:
optarray = args.extra.split()
specmin = int(args.specmin)
nspec = int(args.nspec)
bundlesize = int(args.bundlesize)
specmax = specmin + nspec
# Now we divide our spectra into bundles
checkbundles = set()
checkbundles.update(
np.floor_divide(np.arange(specmin, specmax),
bundlesize * np.ones(nspec)).astype(int))
bundles = sorted(checkbundles)
nbundle = len(bundles)
bspecmin = {}
bnspec = {}
for b in bundles:
if specmin > b * bundlesize:
bspecmin[b] = specmin
else:
bspecmin[b] = b * bundlesize
if (b + 1) * bundlesize > specmax:
bnspec[b] = specmax - bspecmin[b]
else:
bnspec[b] = (b + 1) * bundlesize - bspecmin[b]
# Now we assign bundles to processes
nproc = 1
rank = 0
if comm is not None:
nproc = comm.size
rank = comm.rank
mynbundle = int(nbundle / nproc)
myfirstbundle = 0
leftover = nbundle % nproc
if rank < leftover:
mynbundle += 1
myfirstbundle = rank * mynbundle
else:
myfirstbundle = ((mynbundle + 1) * leftover) + \
(mynbundle * (rank - leftover))
if rank == 0:
# Print parameters
log.info("specex: using {} processes".format(nproc))
log.info("specex: input image = {}".format(imgfile))
log.info("specex: input PSF = {}".format(inpsffile))
log.info("specex: output = {}".format(outfile))
log.info("specex: bundlesize = {}".format(bundlesize))
log.info("specex: specmin = {}".format(specmin))
log.info("specex: specmax = {}".format(specmax))
if args.broken_fibers:
log.info("specex: broken fibers = {}".format(args.broken_fibers))
# get the root output file
outpat = re.compile(r'(.*)\.fits')
outmat = outpat.match(outfile)
if outmat is None:
raise RuntimeError("specex output file should have .fits extension")
outroot = outmat.group(1)
outdir = os.path.dirname(outroot)
if rank == 0:
if not os.path.isdir(outdir):
os.makedirs(outdir)
failcount = 0
for b in range(myfirstbundle, myfirstbundle + mynbundle):
outbundle = "{}_{:02d}".format(outroot, b)
outbundlefits = "{}.fits".format(outbundle)
com = ['desi_psf_fit']
com.extend(['-a', imgfile])
com.extend(['--in-psf', inpsffile])
com.extend(['--out-psf', outbundlefits])
com.extend(['--first-bundle', "{}".format(b)])
com.extend(['--last-bundle', "{}".format(b)])
com.extend(['--first-fiber', "{}".format(bspecmin[b])])
com.extend(['--last-fiber', "{}".format(bspecmin[b] + bnspec[b] - 1)])
if args.broken_fibers:
com.extend(['--broken-fibers', "{}".format(args.broken_fibers)])
if args.debug:
com.extend(['--debug'])
com.extend(optarray)
log.debug("proc {} calling {}".format(rank, " ".join(com)))
argc = len(com)
arg_buffers = [ct.create_string_buffer(com[i].encode('ascii')) \
for i in range(argc)]
addrlist = [ ct.cast(x, ct.POINTER(ct.c_char)) for x in \
map(ct.addressof, arg_buffers) ]
arg_pointers = (ct.POINTER(ct.c_char) * argc)(*addrlist)
retval = libspecex.cspecex_desi_psf_fit(argc, arg_pointers)
if retval != 0:
comstr = " ".join(com)
log.error("desi_psf_fit on process {} failed with return "
"value {} running {}".format(rank, retval, comstr))
failcount += 1
if comm is not None:
from mpi4py import MPI
failcount = comm.allreduce(failcount, op=MPI.SUM)
if failcount > 0:
# all processes throw
raise RuntimeError("some bundles failed desi_psf_fit")
if rank == 0:
outfits = "{}.fits".format(outroot)
inputs = ["{}_{:02d}.fits".format(outroot, x) for x in bundles]
if args.disable_merge:
log.info("don't merge")
else:
#- Empirically it appears that files written by one rank sometimes
#- aren't fully buffer-flushed and closed before getting here,
#- despite the MPI allreduce barrier. Pause to let I/O catch up.
log.info('HACK: taking a 20 sec pause before merging')
sys.stdout.flush()
time.sleep(20.)
merge_psf(inputs, outfits)
log.info('done merging')
if failcount == 0:
# only remove the per-bundle files if the merge was good
for f in inputs:
if os.path.isfile(f):
os.remove(f)
if comm is not None:
failcount = comm.bcast(failcount, root=0)
if failcount > 0:
# all processes throw
raise RuntimeError("merging of per-bundle files failed")
return
def parse(options=None):
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--simspec',
type=str,
required=True,
help="input simspec file")
parser.add_argument('--fibermap', type=str, help='input fibermap file')
parser.add_argument(
'-n',
'--nspec',
type=int,
default=100,
help='number of spectra to be simulated, starting from first')
parser.add_argument('--nstart',
type=int,
default=0,
help='starting spectra # for simulation 0-4999')
parser.add_argument('--spectrograph',
type=int,
default=None,
help='Spectrograph no. 0-9')
parser.add_argument('--config',
type=str,
default='desi',
help='specsim configuration')
parser.add_argument('-b',
'--n_fibers',
type=int,
default=650,
help='total number of fibers')
parser.add_argument('-t',
'--telescope',
type=str,
default='1m',
help='telescope',
choices=['1m', '160mm'])
parser.add_argument('-l',
'--location',
type=str,
default='APO',
help='site location',
choices=['APO', 'LCO'])
parser.add_argument('-s',
'--seed',
type=int,
default=0,
help="random seed")
# Only produce uncalibrated output
parser.add_argument('--frameonly',
action="store_true",
help="only output frame files")
# Moon options if bright or gray time
parser.add_argument('--moon-phase',
type=float,
help='moon phase (0=full, 1=new)',
default=None,
metavar='')
parser.add_argument('--moon-angle',
type=float,
help='separation angle to the moon (0-180 deg)',
default=None,
metavar='')
parser.add_argument('--moon-zenith',
type=float,
help='zenith angle of the moon (0-90 deg)',
default=None,
metavar='')
parser.add_argument(
'--objtype',
type=str,
help='ELG, LRG, QSO, BGS, MWS, WD, DARK_MIX, or BRIGHT_MIX',
default='DARK_MIX',
metavar='')
parser.add_argument('-a',
'--airmass',
type=float,
help='airmass',
default=None,
metavar='')
parser.add_argument('-e',
'--exptime',
type=float,
help='exposure time (s)',
default=None,
metavar='')
parser.add_argument('-o',
'--outdir',
type=str,
help='output directory',
default='.',
metavar='')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='toggle on verbose output')
parser.add_argument(
'--outdir-truth',
type=str,
help='optional alternative output directory for truth files',
metavar='')
# Object type specific options
parser.add_argument('--zrange-qso',
type=float,
default=(0.5, 4.0),
nargs=2,
metavar='',
help='minimum and maximum redshift range for QSO')
parser.add_argument('--zrange-elg',
type=float,
default=(0.6, 1.6),
nargs=2,
metavar='',
help='minimum and maximum redshift range for ELG')
parser.add_argument('--zrange-lrg',
type=float,
default=(0.5, 1.1),
nargs=2,
metavar='',
help='minimum and maximum redshift range for LRG')
parser.add_argument('--zrange-bgs',
type=float,
default=(0.01, 0.4),
nargs=2,
metavar='',
help='minimum and maximum redshift range for BGS')
parser.add_argument(
'--rmagrange-bgs',
type=float,
default=(15.0, 19.5),
nargs=2,
metavar='',
help='Minimum and maximum BGS r-band (AB) magnitude range')
parser.add_argument(
'--sne-rfluxratiorange',
type=float,
default=(0.1, 1.0),
nargs=2,
metavar='',
help='r-band flux ratio of the SNeIa spectrum relative to the galaxy')
parser.add_argument('--add-SNeIa',
action='store_true',
help='include SNeIa spectra')
if options is None:
args = parser.parse_args()
else:
args = parser.parse_args(options)
log = get_logger()
if args.simspec:
args.objtype = None
if args.fibermap is None:
dirname = os.path.dirname(os.path.abspath(args.simspec))
filename = os.path.basename(args.simspec).replace(
'simspec', 'fibermap')
args.fibermap = os.path.join(dirname, filename)
log.warning(
'deriving fibermap {} from simspec input filename'.format(
args.fibermap))
return args
def compute_fiber_bundle_trace_shifts_using_psf(fibers,
line,
psf,
image,
maxshift=2.):
"""
Computes trace shifts along x and y from a preprocessed image, a PSF (with trace coords), and a given emission line,
by doing a forward model of the image.
Args:
fibers : 1D array with list of fibers
line : float, wavelength of an emission line (in Angstrom)
psf : specter psf object
image : DESI preprocessed image object
Optional:
maxshift : float maximum shift in pixels for 2D chi2 scan
Returns:
x : 1D array of x coordinates on CCD (axis=1 in numpy image array, AXIS=0 in FITS, cross-dispersion axis = fiber number direction)
y : 1D array of y coordinates on CCD (axis=0 in numpy image array, AXIS=1 in FITS, wavelength dispersion axis)
dx : 1D array of shifts along x coordinates on CCD
dy : 1D array of shifts along y coordinates on CCD
sx : 1D array of uncertainties on dx
sy : 1D array of uncertainties on dy
"""
log = get_logger()
#log.info("compute_fiber_bundle_offsets fibers={} line={}".format(fibers,line))
# get central coordinates of bundle for interpolation of offsets on CCD
x, y = psf.xy([
int(np.median(fibers)),
], line)
try:
nfibers = len(fibers)
# compute stamp coordinates
xstart = None
xstop = None
ystart = None
ystop = None
xs = []
ys = []
pix = []
xx = []
yy = []
for fiber in fibers:
txs, tys, tpix = psf.xypix(fiber, line)
xs.append(txs)
ys.append(tys)
pix.append(tpix)
if xstart is None:
xstart = txs.start
xstop = txs.stop
ystart = tys.start
ystop = tys.stop
else:
xstart = min(xstart, txs.start)
xstop = max(xstop, txs.stop)
ystart = min(ystart, tys.start)
ystop = max(ystop, tys.stop)
# load stamp data, with margins to avoid problems with shifted psf
margin = int(maxshift) + 1
stamp = np.zeros(
(ystop - ystart + 2 * margin, xstop - xstart + 2 * margin))
stampivar = np.zeros(stamp.shape)
stamp[margin:-margin, margin:-margin] = image.pix[ystart:ystop,
xstart:xstop]
stampivar[margin:-margin, margin:-margin] = image.ivar[ystart:ystop,
xstart:xstop]
# will use a fixed footprint despite changes of psf stamps
# so that chi2 always based on same data set
footprint = np.zeros(stamp.shape)
for i in range(nfibers):
footprint[margin - ystart + ys[i].start:margin - ystart +
ys[i].stop, margin - xstart + xs[i].start:margin -
xstart + xs[i].stop] = 1
#plt.imshow(footprint) ; plt.show() ; sys.exit(12)
# define grid of shifts to test
res = 0.5
nshift = int(maxshift / res)
dx = res * np.tile(
np.arange(2 * nshift + 1) - nshift, (2 * nshift + 1, 1))
dy = dx.T
original_shape = dx.shape
dx = dx.ravel()
dy = dy.ravel()
chi2 = np.zeros(dx.shape)
A = np.zeros((nfibers, nfibers))
B = np.zeros((nfibers))
mods = np.zeros(np.zeros(nfibers).shape + stamp.shape)
debugging = False
if debugging: # FOR DEBUGGING KEEP MODELS
models = []
# loop on possible shifts
# refit fluxes and compute chi2
for d in range(len(dx)):
# print(d,dx[d],dy[d])
A *= 0
B *= 0
mods *= 0
for i, fiber in enumerate(fibers):
# apply the PSF shift
psf._cache = {} # reset cache !!
psf.coeff['X']._coeff[fiber][0] += dx[d]
psf.coeff['Y']._coeff[fiber][0] += dy[d]
# compute pix and paste on stamp frame
xx, yy, pix = psf.xypix(fiber, line)
mods[i][margin - ystart + yy.start:margin - ystart + yy.stop,
margin - xstart + xx.start:margin - xstart +
xx.stop] = pix
# undo the PSF shift
psf.coeff['X']._coeff[fiber][0] -= dx[d]
psf.coeff['Y']._coeff[fiber][0] -= dy[d]
B[i] = np.sum(stampivar * stamp * mods[i])
for j in range(i + 1):
A[i, j] = np.sum(stampivar * mods[i] * mods[j])
if j != i:
A[j, i] = A[i, j]
Ai = np.linalg.inv(A)
flux = Ai.dot(B)
model = np.zeros(stamp.shape)
for i in range(nfibers):
model += flux[i] * mods[i]
chi2[d] = np.sum(stampivar * (stamp - model)**2)
if debugging:
models.append(model)
if debugging:
schi2 = chi2.reshape(original_shape).copy() # FOR DEBUGGING
sdx = dx.copy()
sdy = dy.copy()
# find minimum chi2 grid point
k = chi2.argmin()
j, i = np.unravel_index(k, ((2 * nshift + 1), (2 * nshift + 1)))
#print("node dx,dy=",dx.reshape(original_shape)[j,i],dy.reshape(original_shape)[j,i])
# cut a region around minimum
delta = 1
istart = max(0, i - delta)
istop = min(2 * nshift + 1, i + delta + 1)
jstart = max(0, j - delta)
jstop = min(2 * nshift + 1, j + delta + 1)
chi2 = chi2.reshape(original_shape)[jstart:jstop, istart:istop].ravel()
dx = dx.reshape(original_shape)[jstart:jstop, istart:istop].ravel()
dy = dy.reshape(original_shape)[jstart:jstop, istart:istop].ravel()
# fit 2D polynomial of deg2
m = np.array([dx * 0 + 1, dx, dy, dx**2, dy**2, dx * dy]).T
c, r, rank, s = np.linalg.lstsq(m, chi2)
if c[3] > 0 and c[4] > 0:
# get minimum
# dchi2/dx=0 : c[1]+2*c[3]*dx+c[5]*dy = 0
# dchi2/dy=0 : c[2]+2*c[4]*dy+c[5]*dx = 0
a = np.array([[2 * c[3], c[5]], [c[5], 2 * c[4]]])
b = np.array([c[1], c[2]])
t = -np.linalg.inv(a).dot(b)
dx = t[0]
dy = t[1]
sx = 1. / np.sqrt(c[3])
sy = 1. / np.sqrt(c[4])
#print("interp dx,dy=",dx,dy)
if debugging: # FOR DEBUGGING
import matplotlib.pyplot as plt
plt.figure()
plt.subplot(2, 2, 1, title="chi2")
plt.imshow(schi2,
extent=(-nshift * res, nshift * res, -nshift * res,
nshift * res),
origin=0,
interpolation="nearest")
plt.plot(dx, dy, "+", color="white", ms=20)
plt.xlabel("x")
plt.ylabel("y")
plt.subplot(2, 2, 2, title="data")
plt.imshow(stamp * footprint,
origin=0,
interpolation="nearest")
plt.grid()
k0 = np.argmin(sdx**2 + sdy**2)
plt.subplot(2, 2, 3, title="original psf")
plt.imshow(models[k0], origin=0, interpolation="nearest")
plt.grid()
plt.subplot(2, 2, 4, title="shifted psf")
plt.imshow(models[k], origin=0, interpolation="nearest")
plt.grid()
plt.show()
else:
log.warning(
"fit failed (bad chi2 surf.) for fibers [%d:%d] line=%dA" %
(fibers[0], fibers[-1] + 1, int(line)))
dx = 0.
dy = 0.
sx = 10.
sy = 10.
except LinAlgError:
log.warning(
"fit failed (masked or missing data) for fibers [%d:%d] line=%dA" %
(fibers[0], fibers[-1] + 1, int(line)))
dx = 0.
dy = 0.
sx = 10.
sy = 10.
return x, y, dx, dy, sx, sy
def quietDesiLogger(loglvl=20):
from desiutil.log import get_logger
get_logger(level=loglvl)
def compute_uniform_sky(frame,
nsig_clipping=4.,
max_iterations=100,
model_ivar=False,
add_variance=True):
"""Compute a sky model.
Sky[fiber,i] = R[fiber,i,j] Flux[j]
Input flux are expected to be flatfielded!
We don't check this in this routine.
Args:
frame : Frame object, which includes attributes
- wave : 1D wavelength grid in Angstroms
- flux : 2D flux[nspec, nwave] density
- ivar : 2D inverse variance of flux
- mask : 2D inverse mask flux (0=good)
- resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers)
nsig_clipping : [optional] sigma clipping value for outlier rejection
Optional:
max_iterations : int , number of iterations
model_ivar : replace ivar by a model to avoid bias due to correlated flux and ivar. this has a negligible effect on sims.
add_variance : evaluate calibration error and add this to the sky model variance
returns SkyModel object with attributes wave, flux, ivar, mask
"""
log = get_logger()
log.info("starting")
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nwave = frame.nwave
nfibers = len(skyfibers)
current_ivar = frame.ivar[skyfibers].copy() * (frame.mask[skyfibers] == 0)
flux = frame.flux[skyfibers]
Rsky = frame.R[skyfibers]
input_ivar = None
if model_ivar:
log.info(
"use a model of the inverse variance to remove bias due to correlated ivar and flux"
)
input_ivar = current_ivar.copy()
median_ivar_vs_wave = np.median(current_ivar, axis=0)
median_ivar_vs_fiber = np.median(current_ivar, axis=1)
median_median_ivar = np.median(median_ivar_vs_fiber)
for f in range(current_ivar.shape[0]):
threshold = 0.01
current_ivar[f] = median_ivar_vs_fiber[
f] / median_median_ivar * median_ivar_vs_wave
# keep input ivar for very low weights
ii = (input_ivar[f] <= (threshold * median_ivar_vs_wave))
#log.info("fiber {} keep {}/{} original ivars".format(f,np.sum(ii),current_ivar.shape[1]))
current_ivar[f][ii] = input_ivar[f][ii]
sqrtw = np.sqrt(current_ivar)
sqrtwflux = sqrtw * flux
chi2 = np.zeros(flux.shape)
nout_tot = 0
for iteration in range(max_iterations):
# the matrix A is 1/2 of the second derivative of the chi2 with respect to the parameters
# A_ij = 1/2 d2(chi2)/di/dj
# A_ij = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * d(model)/dj[fiber,w]
# the vector B is 1/2 of the first derivative of the chi2 with respect to the parameters
# B_i = 1/2 d(chi2)/di
# B_i = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * (flux[fiber,w]-model[fiber,w])
# the model is model[fiber]=R[fiber]*sky
# and the parameters are the unconvolved sky flux at the wavelength i
# so, d(model)/di[fiber,w] = R[fiber][w,i]
# this gives
# A_ij = sum_fiber sum_wave_w ivar[fiber,w] R[fiber][w,i] R[fiber][w,j]
# A = sum_fiber ( diag(sqrt(ivar))*R[fiber] ) ( diag(sqrt(ivar))* R[fiber] )^t
# A = sum_fiber sqrtwR[fiber] sqrtwR[fiber]^t
# and
# B = sum_fiber sum_wave_w ivar[fiber,w] R[fiber][w] * flux[fiber,w]
# B = sum_fiber sum_wave_w sqrt(ivar)[fiber,w]*flux[fiber,w] sqrtwR[fiber,wave]
#A=scipy.sparse.lil_matrix((nwave,nwave)).tocsr()
A = np.zeros((nwave, nwave))
B = np.zeros((nwave))
# diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
SD = scipy.sparse.lil_matrix((nwave, nwave))
# loop on fiber to handle resolution
for fiber in range(nfibers):
if fiber % 10 == 0:
log.info("iter %d sky fiber %d/%d" %
(iteration, fiber, nfibers))
R = Rsky[fiber]
# diagonal sparse matrix with content = sqrt(ivar)
SD.setdiag(sqrtw[fiber])
sqrtwR = SD * R # each row r of R is multiplied by sqrtw[r]
A += (sqrtwR.T * sqrtwR).todense()
B += sqrtwR.T * sqrtwflux[fiber]
log.info("iter %d solving" % iteration)
w = A.diagonal() > 0
A_pos_def = A[w, :]
A_pos_def = A_pos_def[:, w]
parameters = B * 0
try:
parameters[w] = cholesky_solve(A_pos_def, B[w])
except:
log.info("cholesky failed, trying svd in iteration {}".format(
iteration))
parameters[w] = np.linalg.lstsq(A_pos_def, B[w])[0]
log.info("iter %d compute chi2" % iteration)
for fiber in range(nfibers):
# the parameters are directly the unconvolve sky flux
# so we simply have to reconvolve it
fiber_convolved_sky_flux = Rsky[fiber].dot(parameters)
chi2[fiber] = current_ivar[fiber] * (flux[fiber] -
fiber_convolved_sky_flux)**2
log.info("rejecting")
nout_iter = 0
if iteration < 1:
# only remove worst outlier per wave
# apply rejection iteratively, only one entry per wave among fibers
# find waves with outlier (fastest way)
nout_per_wave = np.sum(chi2 > nsig_clipping**2, axis=0)
selection = np.where(nout_per_wave > 0)[0]
for i in selection:
worst_entry = np.argmax(chi2[:, i])
current_ivar[worst_entry, i] = 0
sqrtw[worst_entry, i] = 0
sqrtwflux[worst_entry, i] = 0
nout_iter += 1
else:
# remove all of them at once
bad = (chi2 > nsig_clipping**2)
current_ivar *= (bad == 0)
sqrtw *= (bad == 0)
sqrtwflux *= (bad == 0)
nout_iter += np.sum(bad)
nout_tot += nout_iter
sum_chi2 = float(np.sum(chi2))
ndf = int(np.sum(chi2 > 0) - nwave)
chi2pdf = 0.
if ndf > 0:
chi2pdf = sum_chi2 / ndf
log.info("iter #%d chi2=%f ndf=%d chi2pdf=%f nout=%d" %
(iteration, sum_chi2, ndf, chi2pdf, nout_iter))
if nout_iter == 0:
break
log.info("nout tot=%d" % nout_tot)
# we know have to compute the sky model for all fibers
# and propagate the uncertainties
# no need to restore the original ivar to compute the model errors when modeling ivar
# the sky inverse variances are very similar
log.info("compute the parameter covariance")
# we may have to use a different method to compute this
# covariance
try:
parameter_covar = cholesky_invert(A)
# the above is too slow
# maybe invert per block, sandwich by R
except np.linalg.linalg.LinAlgError:
log.warning(
"cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv"
)
parameter_covar = np.linalg.pinv(A)
log.info("compute mean resolution")
# we make an approximation for the variance to save CPU time
# we use the average resolution of all fibers in the frame:
mean_res_data = np.mean(frame.resolution_data, axis=0)
Rmean = Resolution(mean_res_data)
log.info("compute convolved sky and ivar")
# The parameters are directly the unconvolved sky
# First convolve with average resolution :
convolved_sky_covar = Rmean.dot(parameter_covar).dot(Rmean.T.todense())
# and keep only the diagonal
convolved_sky_var = np.diagonal(convolved_sky_covar)
# inverse
convolved_sky_ivar = (convolved_sky_var > 0) / (convolved_sky_var +
(convolved_sky_var == 0))
# and simply consider it's the same for all spectra
cskyivar = np.tile(convolved_sky_ivar,
frame.nspec).reshape(frame.nspec, nwave)
# The sky model for each fiber (simple convolution with resolution of each fiber)
cskyflux = np.zeros(frame.flux.shape)
for i in range(frame.nspec):
cskyflux[i] = frame.R[i].dot(parameters)
# look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1
if skyfibers.size > 1 and add_variance:
modified_cskyivar = _model_variance(frame, cskyflux, cskyivar,
skyfibers)
else:
modified_cskyivar = cskyivar.copy()
# need to do better here
mask = (cskyivar == 0).astype(np.uint32)
return SkyModel(
frame.wave.copy(),
cskyflux,
modified_cskyivar,
mask,
nrej=nout_tot,
stat_ivar=cskyivar) # keep a record of the statistical ivar for QA
def select_targets(infiles, numproc=4, cmxdir=None, noqso=False):
"""Process input files in parallel to select commissioning (cmx) targets
Parameters
----------
infiles : :class:`list` or `str`
List of input filenames (tractor/sweep files) OR one filename.
numproc : :class:`int`, optional, defaults to 4
The number of parallel processes to use.
cmxdir : :class:`str`, optional, defaults to :envvar:`CMX_DIR`
Directory to find commmissioning files to which to match, such
as the CALSPEC stars. If not specified, the cmx directory is
taken to be the value of :envvar:`CMX_DIR`.
noqso : :class:`boolean`, optional, defaults to ``False``
If passed, do not run the quasar selection. All QSO bits will be
set to zero. Intended use is to speed unit tests.
Returns
-------
:class:`~numpy.ndarray`
The subset of input targets which pass the cmx cuts, including an extra
column for `CMX_TARGET`.
Notes
-----
- if numproc==1, use serial code instead of parallel.
"""
from desiutil.log import get_logger
log = get_logger()
# -Convert single file to list of files.
if isinstance(infiles, str):
infiles = [
infiles,
]
# -Sanity check that files exist before going further.
for filename in infiles:
if not os.path.exists(filename):
raise ValueError("{} doesn't exist".format(filename))
# ADM retrieve/check the cmxdir.
cmxdir = _get_cmxdir(cmxdir)
def _finalize_targets(objects, cmx_target, priority_shift):
# -desi_target includes BGS_ANY and MWS_ANY, so we can filter just
# -on desi_target != 0
keep = (cmx_target != 0)
objects = objects[keep]
cmx_target = cmx_target[keep]
priority_shift = priority_shift[keep]
# -Add *_target mask columns
# ADM note that only cmx_target is defined for commissioning
# ADM so just pass that around
targets = finalize(objects,
cmx_target,
cmx_target,
cmx_target,
survey='cmx')
# ADM shift the priorities of targets with functional priorities.
targets["PRIORITY_INIT"] += priority_shift
return targets
# -functions to run on every brick/sweep file
def _select_targets_file(filename):
'''Returns targets in filename that pass the cuts'''
objects = io.read_tractor(filename)
cmx_target, priority_shift = apply_cuts(objects,
cmxdir=cmxdir,
noqso=noqso)
return _finalize_targets(objects, cmx_target, priority_shift)
# Counter for number of bricks processed;
# a numpy scalar allows updating nbrick in python 2
# c.f https://www.python.org/dev/peps/pep-3104/
nbrick = np.zeros((), dtype='i8')
t0 = time()
def _update_status(result):
''' wrapper function for the critical reduction operation,
that occurs on the main parallel process '''
if nbrick % 20 == 0 and nbrick > 0:
elapsed = time() - t0
rate = elapsed / nbrick
log.info(
'{} files; {:.1f} secs/file; {:.1f} total mins elapsed'.format(
nbrick, rate, elapsed / 60.))
nbrick[...] += 1 # this is an in-place modification
return result
# -Parallel process input files
if numproc > 1:
pool = sharedmem.MapReduce(np=numproc)
with pool:
targets = pool.map(_select_targets_file,
infiles,
reduce=_update_status)
else:
targets = list()
for x in infiles:
targets.append(_update_status(_select_targets_file(x)))
targets = np.concatenate(targets)
return targets
from datetime import datetime
from pkg_resources import resource_filename
from time import time
from astropy.io import ascii
from glob import glob
import healpy as hp
from desitarget import io
from desitarget.internal import sharedmem
from desimodel.footprint import radec2pix
from desitarget.geomask import add_hp_neighbors, radec_match_to, nside2nside
# ADM set up the DESI default logger
from desiutil.log import get_logger
log = get_logger()
# ADM start the clock
start = time()
# ADM columns contained in our version of the Tycho fits files.
tychodatamodel = np.array([], dtype=[
('TYC1', '>i2'), ('TYC2', '>i2'), ('TYC3', '|u1'),
('RA', '>f8'), ('DEC', '>f8'),
('MEAN_RA', '>f8'), ('MEAN_DEC', '>f8'),
('SIGMA_RA', '>f4'), ('SIGMA_DEC', '>f4'),
# ADM these are converted to be in mas/yr for consistency with Gaia.
('PM_RA', '>f4'), ('PM_DEC', '>f4'),
('SIGMA_PM_RA', '>f4'), ('SIGMA_PM_DEC', '>f4'),
('EPOCH_RA', '>f4'), ('EPOCH_DEC', '>f4'),
('MAG_BT', '>f4'), ('MAG_VT', '>f4'), ('MAG_HP', '>f4'), ('ISGALAXY', '|u1'),
def add_missing_frames(frames):
'''
Adds any missing frames with ivar=0 FrameLite objects with correct shape
to match those that do exist.
Args:
frames: dict of FrameLite objects, keyed by (night,expid,camera)
Modifies `frames` in-place.
Example: if `frames` has keys (2020,1,'b0') and (2020,1,'r0') but
not (2020,1,'z0'), this will add a blank FrameLite object for z0.
The purpose of this is to facilitate frames2spectra, which needs
*something* for every spectro camera for every exposure that is included.
'''
log = get_logger()
#- First figure out the number of wavelengths per band
wave = dict()
ndiag = dict()
for (night, expid, camera), frame in frames.items():
band = camera[0]
if band not in wave:
wave[band] = frame.wave
if band not in ndiag:
ndiag[band] = frame.rdat.shape[1]
#- Now loop through all frames, filling in any missing bands
bands = sorted(list(wave.keys()))
for (night, expid, camera), frame in list(frames.items()):
band = camera[0]
spectro = camera[1:]
for x in bands:
if x == band:
continue
xcam = x + spectro
if (night, expid, xcam) in frames:
continue
log.warning('Creating blank data for missing frame {}'.format(
(night, expid, xcam)))
nwave = len(wave[x])
nspec = frame.flux.shape[0]
flux = np.zeros((nspec, nwave), dtype='f4')
ivar = np.zeros((nspec, nwave), dtype='f4')
mask = np.zeros((nspec, nwave), dtype='u4') + specmask.NODATA
rdat = np.zeros((nspec, ndiag[x], nwave), dtype='f4')
#- Copy the header and correct the camera keyword
header = fitsio.FITSHDR(frame.header)
header['camera'] = xcam
#- Make new blank scores, replacing trailing band _B/R/Z
dtype = list()
if frame.scores is not None:
for name in frame.scores.dtype.names:
if name.endswith('_' + band.upper()):
xname = name[0:-1] + x.upper()
dtype.append((xname, type(frame.scores[name][0])))
scores = np.zeros(nspec, dtype=dtype)
else:
scores = None
#- Add the blank FrameLite object
frames[(night, expid, xcam)] = FrameLite(wave[x], flux, ivar, mask,
rdat, frame.fibermap,
header, scores)
def stdouterr_redirected(to=None, comm=None):
"""
Redirect stdout and stderr to a file.
The general technique is based on:
http://stackoverflow.com/questions/5081657
http://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/
One difference here is that each process in the communicator
redirects to a different temporary file, and the upon exit
from the context the rank zero process concatenates these
in order to the file result.
Args:
to (str): The output file name.
comm (mpi4py.MPI.Comm): The optional MPI communicator.
"""
# The currently active POSIX file descriptors
fd_out = sys.stdout.fileno()
fd_err = sys.stderr.fileno()
# The DESI logger
log = get_logger()
def _redirect(out_to, err_to):
# Flush the C-level buffers
if c_stdout is not None:
libc.fflush(c_stdout)
if c_stderr is not None:
libc.fflush(c_stderr)
# This closes the python file handles, and marks the POSIX
# file descriptors for garbage collection- UNLESS those
# are the special file descriptors for stderr/stdout.
sys.stdout.close()
sys.stderr.close()
# Close fd_out/fd_err if they are open, and copy the
# input file descriptors to these.
os.dup2(out_to, fd_out)
os.dup2(err_to, fd_err)
# Create a new sys.stdout / sys.stderr that points to the
# redirected POSIX file descriptors. In Python 3, these
# are actually higher level IO objects.
if sys.version_info[0] < 3:
sys.stdout = os.fdopen(fd_out, "wb")
sys.stderr = os.fdopen(fd_err, "wb")
else:
# Python 3 case
sys.stdout = io.TextIOWrapper(os.fdopen(fd_out, 'wb'))
sys.stderr = io.TextIOWrapper(os.fdopen(fd_err, 'wb'))
# update DESI logging to use new stdout
while len(log.handlers) > 0:
h = log.handlers[0]
log.removeHandler(h)
# Add the current stdout.
ch = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter(
"%(levelname)s:%(filename)s:%(lineno)s:%(funcName)s: %(message)s")
ch.setFormatter(formatter)
log.addHandler(ch)
# redirect both stdout and stderr to the same file
if to is None:
to = "/dev/null"
if (comm is None) or (comm.rank == 0):
log.debug("Begin log redirection to {} at {}".format(
to, time.asctime()))
# Save the original file descriptors so we can restore them later
saved_fd_out = os.dup(fd_out)
saved_fd_err = os.dup(fd_err)
try:
pto = to
if comm is not None:
if to != "/dev/null":
pto = "{}_{}".format(to, comm.rank)
# open python file, which creates low-level POSIX file
# descriptor.
file = open(pto, "w")
# redirect stdout/stderr to this new file descriptor.
_redirect(out_to=file.fileno(), err_to=file.fileno())
yield # allow code to be run with the redirected output
# close python file handle, which will mark POSIX file
# descriptor for garbage collection. That is fine since
# we are about to overwrite those in the finally clause.
file.close()
finally:
# restore old stdout and stderr
_redirect(out_to=saved_fd_out, err_to=saved_fd_err)
if comm is not None:
# concatenate per-process files
comm.barrier()
if comm.rank == 0:
with open(to, "w") as outfile:
for p in range(comm.size):
outfile.write(
"================= Process {} =================\n".
format(p))
fname = "{}_{}".format(to, p)
with open(fname) as infile:
outfile.write(infile.read())
os.remove(fname)
comm.barrier()
if (comm is None) or (comm.rank == 0):
log.debug("End log redirection to {} at {}".format(
to, time.asctime()))
# flush python handles for good measure
sys.stdout.flush()
sys.stderr.flush()
return
def write_frame(outfile, frame, header=None, fibermap=None, units=None):
"""Write a frame fits file and returns path to file written.
Args:
outfile: full path to output file, or tuple (night, expid, channel)
frame: desispec.frame.Frame object with wave, flux, ivar...
Optional:
header: astropy.io.fits.Header or dict to override frame.header
fibermap: table to store as FIBERMAP HDU
Returns:
full filepath of output file that was written
Note:
to create a Frame object to pass into write_frame,
frame = Frame(wave, flux, ivar, resolution_data)
"""
log = get_logger()
outfile = makepath(outfile, 'frame')
#- Ignore some known and harmless units warnings
import warnings
warnings.filterwarnings(
'ignore', message="'.*nanomaggies.* did not parse as fits unit.*")
warnings.filterwarnings(
'ignore', message=".*'10\*\*6 arcsec.* did not parse as fits unit.*")
if header is not None:
hdr = fitsheader(header)
else:
hdr = fitsheader(frame.meta)
add_dependencies(hdr)
# Vette
diagnosis = frame.vet()
if diagnosis != 0:
raise IOError(
"Frame did not pass simple vetting test. diagnosis={:d}".format(
diagnosis))
hdus = fits.HDUList()
x = fits.PrimaryHDU(frame.flux.astype('f4'), header=hdr)
x.header['EXTNAME'] = 'FLUX'
if units is not None:
units = str(units)
if 'BUNIT' in hdr and hdr['BUNIT'] != units:
log.warning('BUNIT {bunit} != units {units}; using {units}'.format(
bunit=hdr['BUNIT'], units=units))
x.header['BUNIT'] = units
hdus.append(x)
hdus.append(fits.ImageHDU(frame.ivar.astype('f4'), name='IVAR'))
# hdus.append( fits.CompImageHDU(frame.mask, name='MASK') )
hdus.append(fits.ImageHDU(frame.mask, name='MASK'))
hdus.append(fits.ImageHDU(frame.wave.astype('f8'), name='WAVELENGTH'))
hdus[-1].header['BUNIT'] = 'Angstrom'
if frame.resolution_data is not None:
hdus.append(
fits.ImageHDU(frame.resolution_data.astype('f4'),
name='RESOLUTION'))
elif frame.wsigma is not None:
log.debug("Using ysigma from qproc")
qrimg = fits.ImageHDU(frame.wsigma.astype('f4'), name='YSIGMA')
qrimg.header["NDIAG"] = frame.ndiag
hdus.append(qrimg)
if fibermap is not None:
fibermap = encode_table(fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append(fits.convenience.table_to_hdu(fibermap))
elif frame.fibermap is not None:
fibermap = encode_table(frame.fibermap) #- unicode -> bytes
fibermap.meta['EXTNAME'] = 'FIBERMAP'
hdus.append(fits.convenience.table_to_hdu(fibermap))
elif frame.spectrograph is not None:
x.header[
'FIBERMIN'] = 500 * frame.spectrograph # Hard-coded (as in desispec.frame)
else:
log.error(
"You are likely writing a frame without sufficient fiber info")
if frame.chi2pix is not None:
hdus.append(fits.ImageHDU(frame.chi2pix.astype('f4'), name='CHI2PIX'))
if frame.scores is not None:
scores_tbl = encode_table(frame.scores) #- unicode -> bytes
scores_tbl.meta['EXTNAME'] = 'SCORES'
hdus.append(fits.convenience.table_to_hdu(scores_tbl))
if frame.scores_comments is not None: # add comments in header
hdu = hdus['SCORES']
for i in range(1, 999):
key = 'TTYPE' + str(i)
if key in hdu.header:
value = hdu.header[key]
if value in frame.scores_comments.keys():
hdu.header[key] = (value, frame.scores_comments[value])
hdus.writeto(outfile + '.tmp', overwrite=True, checksum=True)
os.rename(outfile + '.tmp', outfile)
return outfile
def compute_fiberflat(frame,
nsig_clipping=10.,
accuracy=5.e-4,
minval=0.1,
maxval=10.,
max_iterations=100,
smoothing_res=5.,
max_bad=100,
max_rej_it=5,
min_sn=0,
diag_epsilon=1e-3):
"""Compute fiber flat by deriving an average spectrum and dividing all fiber data by this average.
Input data are expected to be on the same wavelength grid, with uncorrelated noise.
They however do not have exactly the same resolution.
Args:
frame (desispec.Frame): input Frame object with attributes
wave, flux, ivar, resolution_data
nsig_clipping : [optional] sigma clipping value for outlier rejection
accuracy : [optional] accuracy of fiberflat (end test for the iterative loop)
minval: [optional] mask pixels with flux < minval * median fiberflat.
maxval: [optional] mask pixels with flux > maxval * median fiberflat.
max_iterations: [optional] maximum number of iterations
smoothing_res: [optional] spacing between spline fit nodes for smoothing the fiberflat
max_bad: [optional] mask entire fiber if more than max_bad-1 initially unmasked pixels are masked during the iterations
max_rej_it: [optional] reject at most the max_rej_it worst pixels in each iteration
min_sn: [optional] mask portions with signal to noise less than min_sn
diag_epsilon: [optional] size of the regularization term in the deconvolution
Returns:
desispec.FiberFlat object with attributes
wave, fiberflat, ivar, mask, meanspec
Notes:
- we first iteratively :
- compute a deconvolved mean spectrum
- compute a fiber flat using the resolution convolved mean spectrum for each fiber
- smooth the fiber flat along wavelength
- clip outliers
- then we compute a fiberflat at the native fiber resolution (not smoothed)
- the routine returns the fiberflat, its inverse variance , mask, and the deconvolved mean spectrum
- the fiberflat is the ratio data/mean , so this flat should be divided to the data
NOTE THAT THIS CODE HAS NOT BEEN TESTED WITH ACTUAL FIBER TRANSMISSION VARIATIONS,
OUTLIER PIXELS, DEAD COLUMNS ...
"""
log = get_logger()
log.info("starting")
#
# chi2 = sum_(fiber f) sum_(wavelenght i) w_fi ( D_fi - F_fi (R_f M)_i )
#
# where
# w = inverse variance
# D = flux data (at the resolution of the fiber)
# F = smooth fiber flat
# R = resolution data
# M = mean deconvolved spectrum
#
# M = A^{-1} B
# with
# A_kl = sum_(fiber f) sum_(wavelenght i) w_fi F_fi^2 (R_fki R_fli)
# B_k = sum_(fiber f) sum_(wavelenght i) w_fi D_fi F_fi R_fki
#
# defining R'_fi = sqrt(w_fi) F_fi R_fi
# and D'_fi = sqrt(w_fi) D_fi
#
# A = sum_(fiber f) R'_f R'_f^T
# B = sum_(fiber f) R'_f D'_f
# (it's faster that way, and we try to use sparse matrices as much as possible)
#
#- Shortcuts
nwave = frame.nwave
nfibers = frame.nspec
wave = frame.wave.copy() #- this will become part of output too
flux = frame.flux.copy()
ivar = frame.ivar * (frame.mask == 0)
# iterative fitting and clipping to get precise mean spectrum
# we first need to iterate to converge on a solution of mean spectrum
# and smooth fiber flat. several interations are needed when
# throughput AND resolution vary from fiber to fiber.
# the end test is that the fiber flat has varied by less than accuracy
# of previous iteration for all wavelength
# we also have a max. number of iterations for this code
nout_tot = 0
chi2pdf = 0.
smooth_fiberflat = np.ones((flux.shape))
chi2 = np.zeros((flux.shape))
## mask low sn portions
w = flux * np.sqrt(ivar) < min_sn
ivar[w] = 0
## 0th pass: reject pixels according to minval and maxval
mean_spectrum = np.zeros(flux.shape[1])
nbad = np.zeros(nfibers, dtype=int)
for iteration in range(max_iterations):
for i in range(flux.shape[1]):
w = ivar[:, i] > 0
if w.sum() > 0:
mean_spectrum[i] = np.median(flux[w, i])
nbad_it = 0
for fib in range(nfibers):
w = ((flux[fib, :] < minval * mean_spectrum) |
(flux[fib, :] > maxval * mean_spectrum)) & (ivar[fib, :] > 0)
nbad_it += w.sum()
nbad[fib] += w.sum()
if w.sum() > 0:
ivar[fib, w] = 0
log.warning("0th pass: masking {} pixels in fiber {}".format(
w.sum(), fib))
if nbad[fib] >= max_bad:
ivar[fib, :] = 0
log.warning(
"0th pass: masking entire fiber {} (nbad={})".format(
fib, nbad[fib]))
if nbad_it == 0:
break
# 1st pass is median for spectrum, flat field without resolution
# outlier rejection
for iteration in range(max_iterations):
# use median for spectrum
mean_spectrum = np.zeros((flux.shape[1]))
for i in range(flux.shape[1]):
w = ivar[:, i] > 0
if w.sum() > 0:
mean_spectrum[i] = np.median(flux[w, i])
nbad_it = 0
sum_chi2 = 0
# not more than max_rej_it pixels per fiber at a time
for fib in range(nfibers):
w = ivar[fib, :] > 0
if w.sum() == 0:
continue
F = flux[fib, :] * 0
w = (mean_spectrum != 0) & (ivar[fib, :] > 0)
F[w] = flux[fib, w] / mean_spectrum[w]
smooth_fiberflat[fib, :] = spline_fit(
wave, wave[w], F[w], smoothing_res,
ivar[fib, w] * mean_spectrum[w]**2)
chi2 = ivar[fib, :] * (flux[fib, :] -
mean_spectrum * smooth_fiberflat[fib, :])**2
w = np.isnan(chi2)
bad = np.where(chi2 > nsig_clipping**2)[0]
if bad.size > 0:
if bad.size > max_rej_it: # not more than 5 pixels at a time
ii = np.argsort(chi2[bad])
bad = bad[ii[-max_rej_it:]]
ivar[fib, bad] = 0
log.warning(
"1st pass: rejecting {} pixels from fiber {}".format(
len(bad), fib))
nbad[fib] += len(bad)
if nbad[fib] >= max_bad:
ivar[fib, :] = 0
log.warning(
"1st pass: rejecting fiber {} due to too many (new) bad pixels"
.format(fib))
nbad_it += len(bad)
sum_chi2 += chi2.sum()
ndf = int((ivar > 0).sum() - nwave - nfibers * (nwave / smoothing_res))
chi2pdf = 0.
if ndf > 0:
chi2pdf = sum_chi2 / ndf
log.info(
"1st pass iter #{} chi2={}/{} chi2pdf={} nout={} (nsig={})".format(
iteration, sum_chi2, ndf, chi2pdf, nbad_it, nsig_clipping))
if nbad_it == 0:
break
## flatten fiberflat
## normalize smooth_fiberflat:
mean = np.ones(smooth_fiberflat.shape[1])
for i in range(smooth_fiberflat.shape[1]):
w = ivar[:, i] > 0
if w.sum() > 0:
mean[i] = np.median(smooth_fiberflat[w, i])
smooth_fiberflat = smooth_fiberflat / mean
median_spectrum = mean_spectrum * 1.
previous_smooth_fiberflat = smooth_fiberflat * 0
log.info("after 1st pass : nout = %d/%d" %
(np.sum(ivar == 0), np.size(ivar.flatten())))
# 2nd pass is full solution including deconvolved spectrum, no outlier rejection
for iteration in range(max_iterations):
## reset sum_chi2
sum_chi2 = 0
log.info("2nd pass, iter %d : mean deconvolved spectrum" % iteration)
# fit mean spectrum
A = scipy.sparse.lil_matrix((nwave, nwave)).tocsr()
B = np.zeros((nwave))
# diagonal sparse matrix with content = sqrt(ivar)*flat of a given fiber
SD = scipy.sparse.lil_matrix((nwave, nwave))
# this is to go a bit faster
sqrtwflat = np.sqrt(ivar) * smooth_fiberflat
# loop on fiber to handle resolution (this is long)
for fiber in range(nfibers):
if fiber % 10 == 0:
log.info("2nd pass, filling matrix, iter %d fiber %d" %
(iteration, fiber))
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
SD.setdiag(sqrtwflat[fiber])
sqrtwflatR = SD * R # each row r of R is multiplied by sqrtwflat[r]
A = A + (sqrtwflatR.T * sqrtwflatR).tocsr()
B += sqrtwflatR.T.dot(np.sqrt(ivar[fiber]) * flux[fiber])
A_pos_def = A.todense()
log.info("deconvolving")
w = A.diagonal() > 0
A_pos_def = A_pos_def[w, :]
A_pos_def = A_pos_def[:, w]
mean_spectrum = np.zeros(nwave)
try:
mean_spectrum[w] = cholesky_solve(A_pos_def, B[w])
except:
mean_spectrum[w] = np.linalg.lstsq(A_pos_def, B[w])[0]
log.info("cholesky failes, trying svd inverse in iter {}".format(
iteration))
for fiber in range(nfibers):
if np.sum(ivar[fiber] > 0) == 0:
continue
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
M = R.dot(mean_spectrum)
ok = (M != 0) & (ivar[fiber, :] > 0)
if ok.sum() == 0:
continue
smooth_fiberflat[fiber] = spline_fit(
wave, wave[ok], flux[fiber, ok] / M[ok], smoothing_res,
ivar[fiber, ok] * M[ok]**2) * (ivar[fiber, :] * M**2 > 0)
chi2 = ivar[fiber] * (flux[fiber] - smooth_fiberflat[fiber] * M)**2
sum_chi2 += chi2.sum()
w = np.isnan(smooth_fiberflat[fiber])
if w.sum() > 0:
ivar[fiber] = 0
smooth_fiberflat[fiber] = 1
# normalize to get a mean fiberflat=1
mean = np.ones(smooth_fiberflat.shape[1])
for i in range(nwave):
w = ivar[:, i] > 0
if w.sum() > 0:
mean[i] = np.median(smooth_fiberflat[w, i])
ok = np.where(mean != 0)[0]
smooth_fiberflat[:, ok] /= mean[ok]
# this is the max difference between two iterations
max_diff = np.max(
np.abs(smooth_fiberflat - previous_smooth_fiberflat) * (ivar > 0.))
previous_smooth_fiberflat = smooth_fiberflat.copy()
ndf = int(np.sum(ivar > 0) - nwave - nfibers * (nwave / smoothing_res))
chi2pdf = 0.
if ndf > 0:
chi2pdf = sum_chi2 / ndf
log.info("2nd pass, iter %d, chi2=%f ndf=%d chi2pdf=%f" %
(iteration, sum_chi2, ndf, chi2pdf))
if max_diff < accuracy:
break
log.info(
"2nd pass, iter %d, max diff. = %g > requirement = %g, continue iterating"
% (iteration, max_diff, accuracy))
log.info("Total number of masked pixels=%d" % nout_tot)
log.info("3rd pass, final computation of fiber flat")
# now use mean spectrum to compute flat field correction without any smoothing
# because sharp feature can arise if dead columns
fiberflat = np.ones((flux.shape))
fiberflat_ivar = np.zeros((flux.shape))
mask = np.zeros((flux.shape), dtype='uint32')
# reset ivar
ivar = frame.ivar
fiberflat_mask = 12 # place holder for actual mask bit when defined
nsig_for_mask = nsig_clipping # only mask out N sigma outliers
for fiber in range(nfibers):
if np.sum(ivar[fiber] > 0) == 0:
continue
### R = Resolution(resolution_data[fiber])
R = frame.R[fiber]
M = np.array(np.dot(R.todense(), mean_spectrum)).flatten()
fiberflat[fiber] = (M != 0) * flux[fiber] / (M + (M == 0)) + (M == 0)
fiberflat_ivar[fiber] = ivar[fiber] * M**2
nbad_tot = 0
iteration = 0
while iteration < 500:
w = fiberflat_ivar[fiber, :] > 0
if w.sum() < 100:
break
smooth_fiberflat = spline_fit(wave, wave[w], fiberflat[fiber, w],
smoothing_res, fiberflat_ivar[fiber,
w])
chi2 = fiberflat_ivar[fiber] * (fiberflat[fiber] -
smooth_fiberflat)**2
bad = np.where(chi2 > nsig_for_mask**2)[0]
if bad.size > 0:
nbadmax = 1
if bad.size > nbadmax: # not more than nbadmax pixels at a time
ii = np.argsort(chi2[bad])
bad = bad[ii[-nbadmax:]]
mask[fiber, bad] += fiberflat_mask
fiberflat_ivar[fiber, bad] = 0.
nbad_tot += bad.size
else:
break
iteration += 1
log.info("3rd pass : fiber #%d , number of iterations %d" %
(fiber, iteration))
# set median flat to 1
log.info("3rd pass : set median fiberflat to 1")
mean = np.ones((flux.shape[1]))
for i in range(flux.shape[1]):
ok = np.where((mask[:, i] == 0) & (ivar[:, i] > 0))[0]
if ok.size > 0:
mean[i] = np.median(fiberflat[ok, i])
ok = np.where(mean != 0)[0]
for fiber in range(nfibers):
fiberflat[fiber, ok] /= mean[ok]
log.info("3rd pass : interpolating over masked pixels")
for fiber in range(nfibers):
if np.sum(ivar[fiber] > 0) == 0:
continue
# replace bad by smooth fiber flat
bad = np.where((mask[fiber] > 0) | (fiberflat_ivar[fiber] == 0)
| (fiberflat[fiber] < minval)
| (fiberflat[fiber] > maxval))[0]
if bad.size > 0:
fiberflat_ivar[fiber, bad] = 0
# find max length of segment with bad pix
length = 0
for i in range(bad.size):
ib = bad[i]
ilength = 1
tmp = ib
for jb in bad[i + 1:]:
if jb == tmp + 1:
ilength += 1
tmp = jb
else:
break
length = max(length, ilength)
if length > 10:
log.info(
"3rd pass : fiber #%d has a max length of bad pixels=%d" %
(fiber, length))
smoothing_res = float(max(100, length))
x = np.arange(wave.size)
ok = fiberflat_ivar[fiber] > 0
if ok.sum() == 0:
continue
try:
smooth_fiberflat = spline_fit(x, x[ok], fiberflat[fiber, ok],
smoothing_res,
fiberflat_ivar[fiber, ok])
fiberflat[fiber, bad] = smooth_fiberflat[bad]
except:
fiberflat[fiber, bad] = 1
fiberflat_ivar[fiber, bad] = 0
if nbad_tot > 0:
log.info(
"3rd pass : fiber #%d masked pixels = %d (%d iterations)" %
(fiber, nbad_tot, iteration))
# set median flat to 1
log.info("set median fiberflat to 1")
mean = np.ones((flux.shape[1]))
for i in range(flux.shape[1]):
ok = np.where((mask[:, i] == 0) & (ivar[:, i] > 0))[0]
if ok.size > 0:
mean[i] = np.median(fiberflat[ok, i])
ok = np.where(mean != 0)[0]
for fiber in range(nfibers):
fiberflat[fiber, ok] /= mean[ok]
log.info("done fiberflat")
return FiberFlat(wave,
fiberflat,
fiberflat_ivar,
mask,
mean_spectrum,
chi2pdf=chi2pdf)
def __init__(self,
wave,
flux,
ivar,
mask=None,
resolution_data=None,
fibers=None,
spectrograph=None,
meta=None,
fibermap=None,
chi2pix=None,
scores=None,
scores_comments=None,
wsigma=None,
ndiag=21,
suppress_res_warning=False):
"""
Lightweight wrapper for multiple spectra on a common wavelength grid
x.wave, x.flux, x.ivar, x.mask, x.resolution_data, x.header, sp.R
Args:
wave: 1D[nwave] wavelength in Angstroms
flux: 2D[nspec, nwave] flux
ivar: 2D[nspec, nwave] inverse variance of flux
Optional:
mask: 2D[nspec, nwave] integer bitmask of flux. 0=good.
resolution_data: 3D[nspec, ndiag, nwave]
diagonals of resolution matrix data
fibers: ndarray of which fibers these spectra are
spectrograph: integer, which spectrograph [0-9]
meta: dict-like object (e.g. FITS header)
fibermap: fibermap table
chi2pix: 2D[nspec, nwave] chi2 of 2D model to pixel-level data
for pixels that contributed to each flux bin
scores: dictionnary of 1D arrays of size nspec
scores_comments: dictionnary of string (explaining the scores)
suppress_res_warning: bool to suppress Warning message when the Resolution image is not read
Parameters below allow on-the-fly resolution calculation
wsigma: 2D[nspec,nwave] sigma widths for each wavelength bin for all fibers
Notes:
spectrograph input is used only if fibers is None. In this case,
it assumes nspec_per_spectrograph = flux.shape[0] and calculates
the fibers array for this spectrograph, i.e.
fibers = spectrograph * flux.shape[0] + np.arange(flux.shape[0])
Attributes:
All input args become object attributes.
nspec : number of spectra, flux.shape[0]
nwave : number of wavelengths, flux.shape[1]
specmin : minimum fiber number
R: array of sparse Resolution matrix objects converted
from resolution_data
fibermap: fibermap table if provided
"""
assert wave.ndim == 1
assert flux.ndim == 2
assert wave.shape[0] == flux.shape[1]
assert ivar.shape == flux.shape
assert (mask is None) or mask.shape == flux.shape
assert (mask is None) or mask.dtype in \
(int, np.int64, np.int32, np.uint64, np.uint32), "Bad mask type "+str(mask.dtype)
self.wave = wave
self.flux = flux
self.ivar = ivar
self.meta = meta
self.fibermap = fibermap
self.nspec, self.nwave = self.flux.shape
self.chi2pix = chi2pix
self.scores = scores
self.scores_comments = scores_comments
self.ndiag = ndiag
fibers_per_spectrograph = 500 #- hardcode; could get from desimodel
if mask is None:
self.mask = np.zeros(flux.shape, dtype=np.uint32)
else:
self.mask = util.mask32(mask)
if resolution_data is not None:
if resolution_data.ndim != 3 or \
resolution_data.shape[0] != self.nspec or \
resolution_data.shape[2] != self.nwave:
raise ValueError(
"Wrong dimensions for resolution_data[nspec, ndiag, nwave]"
)
#- Maybe setup non-None identity matrix resolution matrix instead?
self.wsigma = wsigma
self.resolution_data = resolution_data
if resolution_data is not None:
self.wsigma = None #ignore width coefficients if resolution data is given explicitly
self.ndiag = None
self.R = np.array([Resolution(r) for r in resolution_data])
elif wsigma is not None:
from desispec.quicklook.qlresolution import QuickResolution
assert ndiag is not None
r = []
for sigma in wsigma:
r.append(QuickResolution(sigma=sigma, ndiag=self.ndiag))
self.R = np.array(r)
else:
#SK I believe this should be error, but looking at the
#tests frame objects are allowed to not to have resolution data
# thus I changed value error to a simple warning message.
if not suppress_res_warning:
log = get_logger()
log.warning("Frame object is constructed without resolution data or respective "\
"sigma widths. Resolution will not be available")
# raise ValueError("Need either resolution_data or coefficients to generate it")
self.spectrograph = spectrograph
# Deal with Fibers (these must be set!)
if fibers is not None:
fibers = np.asarray(fibers)
if len(fibers) != self.nspec:
raise ValueError("len(fibers) != nspec ({} != {})".format(
len(fibers), self.nspec))
if fibermap is not None and np.any(fibers != fibermap['FIBER']):
raise ValueError("fibermap doesn't match fibers")
if (spectrograph is not None):
minfiber = spectrograph * fibers_per_spectrograph
maxfiber = (spectrograph + 1) * fibers_per_spectrograph
if np.any(fibers < minfiber) or np.any(maxfiber <= fibers):
raise ValueError('fibers inconsistent with spectrograph')
self.fibers = fibers
else:
if fibermap is not None:
self.fibers = fibermap['FIBER']
elif spectrograph is not None:
self.fibers = spectrograph * fibers_per_spectrograph + np.arange(
self.nspec, dtype=int)
elif (self.meta is not None) and ('FIBERMIN' in self.meta):
self.fibers = self.meta['FIBERMIN'] + np.arange(self.nspec,
dtype=int)
else:
raise ValueError("Must set fibers by one of the methods!")
if self.meta is not None:
self.meta['FIBERMIN'] = np.min(self.fibers)
def qa_fiberflat(param, frame, fiberflat):
""" Calculate QA on FiberFlat object
Args:
param: dict of QA parameters
frame: Frame
fiberflat: FiberFlat
Returns:
qadict: dict of QA outputs
Need to record simple Python objects for yaml (str, float, int)
"""
from desimodel.focalplane import fiber_area_arcsec2
log = get_logger()
# x, y, area
fibermap = frame.fibermap
x = fibermap['X_TARGET']
y = fibermap['Y_TARGET']
area = fiber_area_arcsec2(x, y)
mean_area = np.mean(area)
norm_area = area / mean_area
npix = fiberflat.fiberflat.shape[1]
# Normalize
norm_flat = fiberflat.fiberflat / np.outer(norm_area, np.ones(npix))
# Output dict
qadict = {}
# Check amplitude of the meanspectrum
qadict['MAX_MEANSPEC'] = float(np.max(fiberflat.meanspec))
if qadict['MAX_MEANSPEC'] < 100000:
log.warning("Low counts in meanspec = {:g}".format(
qadict['MAX_MEANSPEC']))
# Record chi2pdf
try:
qadict['CHI2PDF'] = float(fiberflat.chi2pdf)
except TypeError:
qadict['CHI2PDF'] = 0.
# N mask
qadict['N_MASK'] = int(np.sum(fiberflat.mask > 0))
if qadict['N_MASK'] > param['MAX_N_MASK']: # Arbitrary
log.warning("High rejection rate: {:d}".format(qadict['N_MASK']))
# Scale (search for low/high throughput)
gdp = fiberflat.mask == 0
rtio = (frame.flux / np.outer(norm_area, np.ones(npix))) / np.outer(
np.ones(fiberflat.nspec), fiberflat.meanspec)
scale = np.median(rtio * gdp, axis=1)
MAX_SCALE_OFF = float(np.max(np.abs(scale - 1.)))
fiber = int(np.argmax(np.abs(scale - 1.)))
qadict['MAX_SCALE_OFF'] = [MAX_SCALE_OFF, fiber]
if qadict['MAX_SCALE_OFF'][0] > param['MAX_SCALE_OFF']:
log.warning("Discrepant flux in fiberflat: {:g}, {:d}".format(
qadict['MAX_SCALE_OFF'][0], qadict['MAX_SCALE_OFF'][1]))
# Offset in fiberflat
qadict['MAX_OFF'] = float(np.max(np.abs(norm_flat - 1.)))
if qadict['MAX_OFF'] > param['MAX_OFF']:
log.warning("Large offset in fiberflat: {:g}".format(
qadict['MAX_OFF']))
# Offset in mean of fiberflat
#mean = np.mean(fiberflat.fiberflat*gdp,axis=1)
mean = np.mean(norm_flat * gdp, axis=1)
fiber = int(np.argmax(np.abs(mean - 1.)))
qadict['MAX_MEAN_OFF'] = [float(np.max(np.abs(mean - 1.))), fiber]
if qadict['MAX_MEAN_OFF'][0] > param['MAX_MEAN_OFF']:
log.warning("Discrepant mean in fiberflat: {:g}, {:d}".format(
qadict['MAX_MEAN_OFF'][0], qadict['MAX_MEAN_OFF'][1]))
# RMS in individual fibers
rms = np.std(gdp * (norm_flat - np.outer(mean, np.ones(fiberflat.nwave))),
axis=1)
#rms = np.std(gdp*(fiberflat.fiberflat-
# np.outer(mean, np.ones(fiberflat.nwave))),axis=1)
fiber = int(np.argmax(rms))
qadict['MAX_RMS'] = [float(np.max(rms)), fiber]
if qadict['MAX_RMS'][0] > param['MAX_RMS']:
log.warning("Large RMS in fiberflat: {:g}, {:d}".format(
qadict['MAX_RMS'][0], qadict['MAX_RMS'][1]))
# Return
return qadict
def get_fiberbitmasked_frame_arrays(frame,
bitmask=None,
ivar_framemask=True,
return_mask=False):
"""
Function that takes a frame object and a bitmask and
returns ivar (and optionally mask) array(s) that have fibers with
offending bits in fibermap['FIBERSTATUS'] set to
0 in ivar and optionally flips a bit in mask.
input:
frame: frame object
bitmask: int32 or list/array of int32's derived from desispec.maskbits.fibermask
OR string indicating a keyword for get_fiberbitmask_comparison_value()
ivar_framemask: bool (default=True), tells code whether to multiply the output
variance by (frame.mask==0)
return_mask: bool, (default=False). Returns the frame.mask with the logic of
FIBERSTATUS applied.
output:
ivar: frame.ivar where the fibers with FIBERSTATUS & bitmask > 0
set to zero ivar
mask: (optional) frame.mask logically OR'ed with BADFIBER bit in cases with
a bad FIBERSTATUS
example bitmask list:
bitmask = [fmsk.BROKENFIBER,fmsk.UNASSIGNED,fmsk.BADFIBER,\
fmsk.BADTRACE,fmsk.MANYBADCOL, fmsk.MANYREJECTED]
bitmask = get_fiberbitmask_comparison_value(kind='fluxcalib')
bitmask = 'fluxcalib'
bitmask = 4128780
"""
ivar = frame.ivar.copy()
mask = frame.mask.copy()
if ivar_framemask and frame.mask is not None:
ivar *= (frame.mask == 0)
fmap = Table(frame.fibermap)
if frame.fibermap is None:
log = get_logger()
log.warning("No fibermap was given, so no FIBERSTATUS check applied.")
if bitmask is None or frame.fibermap is None:
if return_mask:
return ivar, mask
else:
return ivar
if type(bitmask) in [int, np.int32]:
bad = bitmask
elif type(bitmask) == str:
if bitmask.isnumeric():
bad = np.int32(bitmask)
else:
bad = get_fiberbitmask_comparison_value(kind=bitmask)
else:
bad = bitmask[0]
for bit in bitmask[1:]:
bad |= bit
# find if any fibers have an intersection with the bad bits
badfibers = fmap['FIBER'][(fmap['FIBERSTATUS'] & bad) > 0].data
badfibers = badfibers % 500
# For the bad fibers, loop through and nullify them
for fiber in badfibers:
mask[fiber] |= specmask.BADFIBER
if ivar_framemask:
ivar[fiber] = 0.
if return_mask:
return ivar, mask
else:
return ivar
def average_fiberflat(fiberflats):
"""Average several fiberflats
Args:
fiberflats : list of `desispec.FiberFlat` object
returns a desispec.FiberFlat object
"""
log = get_logger()
log.info("starting")
if len(fiberflats) == 0:
message = "input fiberflat list is empty"
log.critical(message)
raise ValueError(message)
if len(fiberflats) == 1:
log.warning("only one fiberflat to average??")
return fiberflats[0]
# check wavelength range
for fflat in fiberflats[1:]:
if not np.allclose(fiberflats[0].wave, fflat.wave):
message = "fiberflats do not have the same wavelength arrays"
log.critical(message)
raise ValueError(message)
wave = fiberflats[0].wave
fiberflat = None
ivar = None
if len(fiberflats) > 2:
log.info("{} fiberflat to average, use masked median".format(
len(fiberflats)))
tmp_fflat = []
tmp_ivar = []
tmp_mask = []
for tmp in fiberflats:
tmp_fflat.append(tmp.fiberflat)
tmp_ivar.append(tmp.ivar)
tmp_mask.append(tmp.mask)
fiberflat = masked_median(np.array(tmp_fflat), np.array(tmp_mask))
ivar = np.sum(np.array(tmp_ivar), axis=0)
ivar *= 2. / np.pi # penalty for using a median instead of a mean
else:
log.info("{} fiberflat to average, use weighted mean".format(
len(fiberflats)))
sw = None
swf = None
for tmp in fiberflats:
w = (tmp.ivar) * (tmp.mask == 0)
if sw is None:
sw = w
swf = w * tmp.fiberflat
mask = tmp.mask
else:
sw += w
swf += w * tmp.fiberflat
fiberflat = swf / (sw + (sw == 0))
ivar = sw
# combined mask
mask = None
for tmp in fiberflats:
if mask is None:
mask = tmp.mask
else:
ii = (mask > 0) & (tmp.mask > 0)
mask[ii] |= tmp.mask[ii]
mask[
tmp.mask ==
0] = 0 # mask=0 on fiber and wave data point where at list one fiberflat has mask=0
return FiberFlat(wave,
fiberflat,
ivar,
mask,
header=fiberflats[0].header,
fibers=fiberflats[0].fibers,
spectrograph=fiberflats[0].spectrograph)
def main(args):
# Set up the logger
if args.verbose:
log = get_logger(DEBUG)
else:
log = get_logger()
# Make sure all necessary environment variables are set
DESI_SPECTRO_REDUX_DIR = "./quickGen"
if 'DESI_SPECTRO_REDUX' not in os.environ:
log.info('DESI_SPECTRO_REDUX environment is not set.')
else:
DESI_SPECTRO_REDUX_DIR = os.environ['DESI_SPECTRO_REDUX']
if os.path.exists(DESI_SPECTRO_REDUX_DIR):
if not os.path.isdir(DESI_SPECTRO_REDUX_DIR):
raise RuntimeError("Path %s Not a directory" %
DESI_SPECTRO_REDUX_DIR)
else:
try:
os.makedirs(DESI_SPECTRO_REDUX_DIR)
except:
raise
SPECPROD_DIR = 'specprod'
if 'SPECPROD' not in os.environ:
log.info('SPECPROD environment is not set.')
else:
SPECPROD_DIR = os.environ['SPECPROD']
prod_Dir = specprod_root()
if os.path.exists(prod_Dir):
if not os.path.isdir(prod_Dir):
raise RuntimeError("Path %s Not a directory" % prod_Dir)
else:
try:
os.makedirs(prod_Dir)
except:
raise
# Initialize random number generator to use.
np.random.seed(args.seed)
random_state = np.random.RandomState(args.seed)
# Derive spectrograph number from nstart if needed
if args.spectrograph is None:
args.spectrograph = args.nstart / args.n_fibers
# Read fibermapfile to get object type, night and expid
if args.fibermap:
log.info("Reading fibermap file {}".format(args.fibermap))
fibermap = read_fibermap(args.fibermap)
objtype = get_source_types(fibermap)
stdindx = np.where(objtype == 'STD') # match STD with STAR
mwsindx = np.where(objtype == 'MWS_STAR') # match MWS_STAR with STAR
bgsindx = np.where(objtype == 'BGS') # match BGS with LRG
objtype[stdindx] = 'STAR'
objtype[mwsindx] = 'STAR'
objtype[bgsindx] = 'LRG'
NIGHT = fibermap.meta['NIGHT']
EXPID = fibermap.meta['EXPID']
TILEID = fibermap.meta['TILEID']
else:
# Create a blank fake fibermap
fibermap = empty_fibermap(args.nspec)
targetids = random_state.randint(2**62, size=args.nspec)
fibermap['TARGETID'] = targetids
night = get_night()
expid = 0
log.info("Initializing SpecSim with config {}".format(args.config))
desiparams = load_desiparams(config=args.config, telescope=args.telescope)
qsim = get_simulator(args.config, num_fibers=1, params=desiparams)
if args.simspec:
# Read the input file
log.info('Reading input file {}'.format(args.simspec))
simspec = desisim.io.read_simspec(args.simspec)
nspec = simspec.nspec
if simspec.flavor == 'arc':
log.warning("quickgen doesn't generate flavor=arc outputs")
return
else:
wavelengths = simspec.wave
spectra = simspec.flux
if nspec < args.nspec:
log.info("Only {} spectra in input file".format(nspec))
args.nspec = nspec
else:
# Initialize the output truth table.
spectra = []
wavelengths = qsim.source.wavelength_out.to(u.Angstrom).value
npix = len(wavelengths)
truth = dict()
meta = Table()
truth['OBJTYPE'] = np.zeros(args.nspec, dtype=(str, 10))
truth['FLUX'] = np.zeros((args.nspec, npix))
truth['WAVE'] = wavelengths
jj = list()
for thisobj in set(true_objtype):
ii = np.where(true_objtype == thisobj)[0]
nobj = len(ii)
truth['OBJTYPE'][ii] = thisobj
log.info('Generating {} template'.format(thisobj))
# Generate the templates
if thisobj == 'ELG':
elg = desisim.templates.ELG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = elg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_elg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'LRG':
lrg = desisim.templates.LRG(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = lrg.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_lrg,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'QSO':
qso = desisim.templates.QSO(wave=wavelengths)
flux, tmpwave, meta1 = qso.make_templates(
nmodel=nobj, seed=args.seed, zrange=args.zrange_qso)
elif thisobj == 'BGS':
bgs = desisim.templates.BGS(wave=wavelengths,
add_SNeIa=args.add_SNeIa)
flux, tmpwave, meta1 = bgs.make_templates(
nmodel=nobj,
seed=args.seed,
zrange=args.zrange_bgs,
rmagrange=args.rmagrange_bgs,
sne_rfluxratiorange=args.sne_rfluxratiorange)
elif thisobj == 'STD':
std = desisim.templates.STD(wave=wavelengths)
flux, tmpwave, meta1 = std.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'QSO_BAD': # use STAR template no color cuts
star = desisim.templates.STAR(wave=wavelengths)
flux, tmpwave, meta1 = star.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'MWS_STAR' or thisobj == 'MWS':
mwsstar = desisim.templates.MWS_STAR(wave=wavelengths)
flux, tmpwave, meta1 = mwsstar.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'WD':
wd = desisim.templates.WD(wave=wavelengths)
flux, tmpwave, meta1 = wd.make_templates(nmodel=nobj,
seed=args.seed)
elif thisobj == 'SKY':
flux = np.zeros((nobj, npix))
meta1 = Table(dict(REDSHIFT=np.zeros(nobj, dtype=np.float32)))
elif thisobj == 'TEST':
flux = np.zeros((args.nspec, npix))
indx = np.where(wave > 5800.0 - 1E-6)[0][0]
ref_integrated_flux = 1E-10
ref_cst_flux_density = 1E-17
single_line = (np.arange(args.nspec) % 2 == 0).astype(
np.float32)
continuum = (np.arange(args.nspec) % 2 == 1).astype(np.float32)
for spec in range(args.nspec):
flux[spec, indx] = single_line[
spec] * ref_integrated_flux / np.gradient(wavelengths)[
indx] # single line
flux[spec] += continuum[
spec] * ref_cst_flux_density # flat continuum
meta1 = Table(
dict(REDSHIFT=np.zeros(args.nspec, dtype=np.float32),
LINE=wave[indx] *
np.ones(args.nspec, dtype=np.float32),
LINEFLUX=single_line * ref_integrated_flux,
CONSTFLUXDENSITY=continuum * ref_cst_flux_density))
else:
log.fatal('Unknown object type {}'.format(thisobj))
sys.exit(1)
# Pack it in.
truth['FLUX'][ii] = flux
meta = vstack([meta, meta1])
jj.append(ii.tolist())
# Sanity check on units; templates currently return ergs, not 1e-17 ergs...
# assert (thisobj == 'SKY') or (np.max(truth['FLUX']) < 1e-6)
# Sort the metadata table.
jj = sum(jj, [])
meta_new = Table()
for k in range(args.nspec):
index = int(np.where(np.array(jj) == k)[0])
meta_new = vstack([meta_new, meta[index]])
meta = meta_new
# Add TARGETID and the true OBJTYPE to the metadata table.
meta.add_column(
Column(true_objtype, dtype=(str, 10), name='TRUE_OBJTYPE'))
meta.add_column(Column(targetids, name='TARGETID'))
# Rename REDSHIFT -> TRUEZ anticipating later table joins with zbest.Z
meta.rename_column('REDSHIFT', 'TRUEZ')
# explicitly set location on focal plane if needed to support airmass
# variations when using specsim v0.5
if qsim.source.focal_xy is None:
qsim.source.focal_xy = (u.Quantity(0, 'mm'), u.Quantity(100, 'mm'))
# Set simulation parameters from the simspec header or desiparams
bright_objects = ['bgs', 'mws', 'bright', 'BGS', 'MWS', 'BRIGHT_MIX']
gray_objects = ['gray', 'grey']
if args.simspec is None:
object_type = objtype
flavor = None
elif simspec.flavor == 'science':
object_type = None
flavor = simspec.header['PROGRAM']
else:
object_type = None
flavor = simspec.flavor
log.warning(
'Maybe using an outdated simspec file with flavor={}'.format(
flavor))
# Set airmass
if args.airmass is not None:
qsim.atmosphere.airmass = args.airmass
elif args.simspec and 'AIRMASS' in simspec.header:
qsim.atmosphere.airmass = simspec.header['AIRMASS']
else:
qsim.atmosphere.airmass = 1.25 # Science Req. Doc L3.3.2
# Set site location
if args.location is not None:
qsim.observation.observatory = args.location
else:
qsim.observation.observatory = 'APO'
# Set exptime
if args.exptime is not None:
qsim.observation.exposure_time = args.exptime * u.s
elif args.simspec and 'EXPTIME' in simspec.header:
qsim.observation.exposure_time = simspec.header['EXPTIME'] * u.s
elif objtype in bright_objects:
qsim.observation.exposure_time = desiparams['exptime_bright'] * u.s
else:
qsim.observation.exposure_time = desiparams['exptime_dark'] * u.s
# Set Moon Phase
if args.moon_phase is not None:
qsim.atmosphere.moon.moon_phase = args.moon_phase
elif args.simspec and 'MOONFRAC' in simspec.header:
qsim.atmosphere.moon.moon_phase = simspec.header['MOONFRAC']
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_phase = 0.7
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_phase = 0.1
else:
qsim.atmosphere.moon.moon_phase = 0.5
# Set Moon Zenith
if args.moon_zenith is not None:
qsim.atmosphere.moon.moon_zenith = args.moon_zenith * u.deg
elif args.simspec and 'MOONALT' in simspec.header:
qsim.atmosphere.moon.moon_zenith = simspec.header['MOONALT'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.moon_zenith = 30 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.moon_zenith = 80 * u.deg
else:
qsim.atmosphere.moon.moon_zenith = 100 * u.deg
# Set Moon - Object Angle
if args.moon_angle is not None:
qsim.atmosphere.moon.separation_angle = args.moon_angle * u.deg
elif args.simspec and 'MOONSEP' in simspec.header:
qsim.atmosphere.moon.separation_angle = simspec.header[
'MOONSEP'] * u.deg
elif flavor in bright_objects or object_type in bright_objects:
qsim.atmosphere.moon.separation_angle = 50 * u.deg
elif flavor in gray_objects:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
else:
qsim.atmosphere.moon.separation_angle = 60 * u.deg
# Initialize per-camera output arrays that will be saved
waves, trueflux, noisyflux, obsivar, resolution, sflux = {}, {}, {}, {}, {}, {}
maxbin = 0
nmax = args.nspec
for camera in qsim.instrument.cameras:
# Lookup this camera's resolution matrix and convert to the sparse
# format used in desispec.
R = Resolution(camera.get_output_resolution_matrix())
resolution[camera.name] = np.tile(R.to_fits_array(),
[args.nspec, 1, 1])
waves[camera.name] = (camera.output_wavelength.to(
u.Angstrom).value.astype(np.float32))
nwave = len(waves[camera.name])
maxbin = max(maxbin, len(waves[camera.name]))
nobj = np.zeros((nmax, 3, maxbin)) # object photons
nsky = np.zeros((nmax, 3, maxbin)) # sky photons
nivar = np.zeros((nmax, 3, maxbin)) # inverse variance (object+sky)
cframe_observedflux = np.zeros(
(nmax, 3, maxbin)) # calibrated object flux
cframe_ivar = np.zeros(
(nmax, 3, maxbin)) # inverse variance of calibrated object flux
cframe_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to calibrated flux
sky_ivar = np.zeros((nmax, 3, maxbin)) # inverse variance of sky
sky_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to sky only
frame_rand_noise = np.zeros(
(nmax, 3, maxbin)) # random Gaussian noise to nobj+nsky
trueflux[camera.name] = np.empty(
(args.nspec, nwave)) # calibrated flux
noisyflux[camera.name] = np.empty(
(args.nspec, nwave)) # observed flux with noise
obsivar[camera.name] = np.empty(
(args.nspec, nwave)) # inverse variance of flux
if args.simspec:
for i in range(10):
cn = camera.name + str(i)
if cn in simspec.cameras:
dw = np.gradient(simspec.cameras[cn].wave)
break
else:
raise RuntimeError(
'Unable to find a {} camera in input simspec'.format(
camera))
else:
sflux = np.empty((args.nspec, npix))
#- Check if input simspec is for a continuum flat lamp instead of science
#- This does not convolve to per-fiber resolution
if args.simspec:
if simspec.flavor == 'flat':
log.info("Simulating flat lamp exposure")
for i, camera in enumerate(qsim.instrument.cameras):
channel = camera.name #- from simspec, b/r/z not b0/r1/z9
assert camera.output_wavelength.unit == u.Angstrom
num_pixels = len(waves[channel])
phot = list()
for j in range(10):
cn = camera.name + str(j)
if cn in simspec.cameras:
camwave = simspec.cameras[cn].wave
dw = np.gradient(camwave)
phot.append(simspec.cameras[cn].phot)
if len(phot) == 0:
raise RuntimeError(
'Unable to find a {} camera in input simspec'.format(
camera))
else:
phot = np.vstack(phot)
meanspec = resample_flux(waves[channel], camwave,
np.average(phot / dw, axis=0))
fiberflat = random_state.normal(loc=1.0,
scale=1.0 / np.sqrt(meanspec),
size=(nspec, num_pixels))
ivar = np.tile(meanspec, [nspec, 1])
mask = np.zeros((simspec.nspec, num_pixels), dtype=np.uint32)
for kk in range((args.nspec + args.nstart - 1) //
args.n_fibers + 1):
camera = channel + str(kk)
outfile = desispec.io.findfile('fiberflat', NIGHT, EXPID,
camera)
start = max(args.n_fibers * kk, args.nstart)
end = min(args.n_fibers * (kk + 1), nmax)
if (args.spectrograph <= kk):
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, kk, start, end))
ff = FiberFlat(waves[channel],
fiberflat[start:end, :],
ivar[start:end, :],
mask[start:end, :],
meanspec,
header=dict(CAMERA=camera))
write_fiberflat(outfile, ff)
filePath = desispec.io.findfile("fiberflat", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(filePath))
sys.exit(0)
# Repeat the simulation for all spectra
fluxunits = 1e-17 * u.erg / (u.s * u.cm**2 * u.Angstrom)
for j in range(args.nspec):
thisobjtype = objtype[j]
sys.stdout.flush()
if flavor == 'arc':
qsim.source.update_in('Quickgen source {0}'.format, 'perfect',
wavelengths * u.Angstrom,
spectra * fluxunits)
else:
qsim.source.update_in('Quickgen source {0}'.format(j),
thisobjtype.lower(),
wavelengths * u.Angstrom,
spectra[j, :] * fluxunits)
qsim.source.update_out()
qsim.simulate()
qsim.generate_random_noise(random_state)
for i, output in enumerate(qsim.camera_output):
assert output['observed_flux'].unit == 1e17 * fluxunits
# Extract the simulation results needed to create our uncalibrated
# frame output file.
num_pixels = len(output)
nobj[j, i, :num_pixels] = output['num_source_electrons'][:, 0]
nsky[j, i, :num_pixels] = output['num_sky_electrons'][:, 0]
nivar[j, i, :num_pixels] = 1.0 / output['variance_electrons'][:, 0]
# Get results for our flux-calibrated output file.
cframe_observedflux[
j, i, :num_pixels] = 1e17 * output['observed_flux'][:, 0]
cframe_ivar[
j,
i, :num_pixels] = 1e-34 * output['flux_inverse_variance'][:, 0]
# Fill brick arrays from the results.
camera = output.meta['name']
trueflux[camera][j][:] = 1e17 * output['observed_flux'][:, 0]
noisyflux[camera][j][:] = 1e17 * (
output['observed_flux'][:, 0] +
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
obsivar[camera][j][:] = 1e-34 * output['flux_inverse_variance'][:,
0]
# Use the same noise realization in the cframe and frame, without any
# additional noise from sky subtraction for now.
frame_rand_noise[
j, i, :num_pixels] = output['random_noise_electrons'][:, 0]
cframe_rand_noise[j, i, :num_pixels] = 1e17 * (
output['flux_calibration'][:, 0] *
output['random_noise_electrons'][:, 0])
# The sky output file represents a model fit to ~40 sky fibers.
# We reduce the variance by a factor of 25 to account for this and
# give the sky an independent (Gaussian) noise realization.
sky_ivar[
j,
i, :num_pixels] = 25.0 / (output['variance_electrons'][:, 0] -
output['num_source_electrons'][:, 0])
sky_rand_noise[j, i, :num_pixels] = random_state.normal(
scale=1.0 / np.sqrt(sky_ivar[j, i, :num_pixels]),
size=num_pixels)
armName = {"b": 0, "r": 1, "z": 2}
for channel in 'brz':
#Before writing, convert from counts/bin to counts/A (as in Pixsim output)
#Quicksim Default:
#FLUX - input spectrum resampled to this binning; no noise added [1e-17 erg/s/cm2/s/Ang]
#COUNTS_OBJ - object counts in 0.5 Ang bin
#COUNTS_SKY - sky counts in 0.5 Ang bin
num_pixels = len(waves[channel])
dwave = np.gradient(waves[channel])
nobj[:, armName[channel], :num_pixels] /= dwave
frame_rand_noise[:, armName[channel], :num_pixels] /= dwave
nivar[:, armName[channel], :num_pixels] *= dwave**2
nsky[:, armName[channel], :num_pixels] /= dwave
sky_rand_noise[:, armName[channel], :num_pixels] /= dwave
sky_ivar[:, armName[channel], :num_pixels] /= dwave**2
# Now write the outputs in DESI standard file system. None of the output file can have more than 500 spectra
# Looping over spectrograph
for ii in range((args.nspec + args.nstart - 1) // args.n_fibers + 1):
start = max(args.n_fibers * ii,
args.nstart) # first spectrum for a given spectrograph
end = min(args.n_fibers * (ii + 1),
nmax) # last spectrum for the spectrograph
if (args.spectrograph <= ii):
camera = "{}{}".format(channel, ii)
log.info(
"Writing files for channel:{}, spectrograph:{}, spectra:{} to {}"
.format(channel, ii, start, end))
num_pixels = len(waves[channel])
# Write frame file
framefileName = desispec.io.findfile("frame", NIGHT, EXPID,
camera)
frame_flux=nobj[start:end,armName[channel],:num_pixels]+ \
nsky[start:end,armName[channel],:num_pixels] + \
frame_rand_noise[start:end,armName[channel],:num_pixels]
frame_ivar = nivar[start:end, armName[channel], :num_pixels]
sh1 = frame_flux.shape[
0] # required for slicing the resolution metric, resolusion matrix has (nspec,ndiag,wave)
# for example if nstart =400, nspec=150: two spectrographs:
# 400-499=> 0 spectrograph, 500-549 => 1
if (args.nstart == start):
resol = resolution[channel][:sh1, :, :]
else:
resol = resolution[channel][-sh1:, :, :]
# must create desispec.Frame object
frame=Frame(waves[channel], frame_flux, frame_ivar,\
resolution_data=resol, spectrograph=ii, \
fibermap=fibermap[start:end], \
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor) )
desispec.io.write_frame(framefileName, frame)
framefilePath = desispec.io.findfile("frame", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(framefilePath))
if args.frameonly or simspec.flavor == 'arc':
continue
# Write cframe file
cframeFileName = desispec.io.findfile("cframe", NIGHT, EXPID,
camera)
cframeFlux = cframe_observedflux[
start:end,
armName[channel], :num_pixels] + cframe_rand_noise[
start:end, armName[channel], :num_pixels]
cframeIvar = cframe_ivar[start:end,
armName[channel], :num_pixels]
# must create desispec.Frame object
cframe = Frame(waves[channel], cframeFlux, cframeIvar, \
resolution_data=resol, spectrograph=ii,
fibermap=fibermap[start:end],
meta=dict(CAMERA=camera, FLAVOR=simspec.flavor, NIGHT=NIGHT, EXPID=EXPID, TILEID=TILEID) )
desispec.io.frame.write_frame(cframeFileName, cframe)
cframefilePath = desispec.io.findfile("cframe", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(cframefilePath))
# Write sky file
skyfileName = desispec.io.findfile("sky", NIGHT, EXPID, camera)
skyflux=nsky[start:end,armName[channel],:num_pixels] + \
sky_rand_noise[start:end,armName[channel],:num_pixels]
skyivar = sky_ivar[start:end, armName[channel], :num_pixels]
skymask = np.zeros(skyflux.shape, dtype=np.uint32)
# must create desispec.Sky object
skymodel = SkyModel(waves[channel],
skyflux,
skyivar,
skymask,
header=dict(CAMERA=camera))
desispec.io.sky.write_sky(skyfileName, skymodel)
skyfilePath = desispec.io.findfile("sky", NIGHT, EXPID, camera)
log.info("Wrote file {}".format(skyfilePath))
# Write calib file
calibVectorFile = desispec.io.findfile("calib", NIGHT, EXPID,
camera)
flux = cframe_observedflux[start:end,
armName[channel], :num_pixels]
phot = nobj[start:end, armName[channel], :num_pixels]
calibration = np.zeros_like(phot)
jj = (flux > 0)
calibration[jj] = phot[jj] / flux[jj]
#- TODO: what should calibivar be?
#- For now, model it as the noise of combining ~10 spectra
calibivar = 10 / cframe_ivar[start:end,
armName[channel], :num_pixels]
#mask=(1/calibivar>0).astype(int)??
mask = np.zeros(calibration.shape, dtype=np.uint32)
# write flux calibration
fluxcalib = FluxCalib(waves[channel], calibration, calibivar,
mask)
write_flux_calibration(calibVectorFile, fluxcalib)
calibfilePath = desispec.io.findfile("calib", NIGHT, EXPID,
camera)
log.info("Wrote file {}".format(calibfilePath))
def write_raw(filename, rawdata, header, camera=None, primary_header=None):
'''
Write raw pixel data to a DESI raw data file
Args:
filename : file name to write data; if this exists, append a new HDU
rawdata : 2D ndarray of raw pixel data including overscans
header : dict-like object or fits.Header with keywords
CCDSECx, BIASSECx, DATASECx where x=1,2,3, or 4
Options:
camera : b0, r1 .. z9 - override value in header
primary_header : header to write in HDU0 if filename doesn't yet exist
The primary utility of this function over raw fits calls is to ensure
that all necessary keywords are present before writing the file.
CCDSECx, BIASSECx, DATASECx where x=1,2,3, or 4
DATE-OBS, GAINx and RDNOISEx will generate a non-fatal warning if missing
'''
log = get_logger()
header = desispec.io.util.fitsheader(header)
primary_header = desispec.io.util.fitsheader(primary_header)
if rawdata.dtype not in (np.int16, np.int32, np.int64):
message = 'dtype {} not supported for raw data'.format(rawdata.dtype)
log.fatal(message)
raise ValueError(message)
fail_message = ''
for required_key in ['DOSVER', 'FEEVER', 'DETECTOR']:
if required_key not in primary_header:
if required_key in header:
primary_header[required_key] = header[required_key]
else:
fail_message = fail_message + \
'Keyword {} must be in header or primary_header\n'.format(required_key)
if fail_message != '':
raise ValueError(fail_message)
#- Check required keywords before writing anything
missing_keywords = list()
if camera is None and 'CAMERA' not in header:
log.error("Must provide camera keyword or header['CAMERA']")
missing_keywords.append('CAMERA')
for amp in ['1', '2', '3', '4']:
for prefix in ['CCDSEC', 'BIASSEC', 'DATASEC']:
keyword = prefix + amp
if keyword not in header:
log.error('Missing keyword ' + keyword)
missing_keywords.append(keyword)
#- Missing DATE-OBS is warning but not error
if 'DATE-OBS' not in primary_header:
if 'DATE-OBS' in header:
primary_header['DATE-OBS'] = header['DATE-OBS']
else:
log.warning('missing keyword DATE-OBS')
#- Missing GAINx is warning but not error
for amp in ['1', '2', '3', '4']:
keyword = 'GAIN' + amp
if keyword not in header:
log.warning('Gain keyword {} missing; using 1.0'.format(keyword))
header[keyword] = 1.0
#- Missing RDNOISEx is warning but not error
for amp in ['1', '2', '3', '4']:
keyword = 'RDNOISE' + amp
if keyword not in header:
log.warning('Readnoise keyword {} missing'.format(keyword))
#- Stop if any keywords are missing
if len(missing_keywords) > 0:
raise KeyError('missing required keywords {}'.format(missing_keywords))
#- Set EXTNAME=camera
if camera is not None:
header['CAMERA'] = camera.lower()
extname = camera.upper()
else:
if header['CAMERA'] != header['CAMERA'].lower():
log.warning('Converting CAMERA {} to lowercase'.format(
header['CAMERA']))
header['CAMERA'] = header['CAMERA'].lower()
extname = header['CAMERA'].upper()
header['INHERIT'] = True
#- fits.CompImageHDU doesn't know how to fill in default keywords, so
#- temporarily generate an uncompressed HDU to get those keywords
header = fits.ImageHDU(rawdata, header=header, name=extname).header
#- Bizarrely, compression of 64-bit integers isn't supported.
#- downcast to 32-bit if that won't lose precision.
#- Real raw data should be 32-bit or 16-bit anyway
if rawdata.dtype == np.int64:
if np.max(np.abs(rawdata)) < 2**31:
rawdata = rawdata.astype(np.int32)
if rawdata.dtype in (np.int16, np.int32):
dataHDU = fits.CompImageHDU(rawdata, header=header, name=extname)
elif rawdata.dtype == np.int64:
log.warning(
'Image compression not supported for 64-bit; writing uncompressed')
dataHDU = fits.ImageHDU(rawdata, header=header, name=extname)
else:
log.error("How did we get this far with rawdata dtype {}?".format(
rawdata.dtype))
dataHDU = fits.ImageHDU(rawdata, header=header, name=extname)
#- Actually write or update the file
if os.path.exists(filename):
hdus = fits.open(filename, mode='append', memmap=False)
if extname in hdus:
hdus.close()
raise ValueError('Camera {} already in {}'.format(
camera, filename))
else:
hdus.append(dataHDU)
hdus.flush()
hdus.close()
else:
hdus = fits.HDUList()
add_dependencies(primary_header)
hdus.append(fits.PrimaryHDU(None, header=primary_header))
hdus.append(dataHDU)
hdus.writeto(filename)
def get_exp2healpix_map(nights=None, specprod_dir=None, nside=64, comm=None):
'''
Returns table with columns NIGHT EXPID SPECTRO HEALPIX NTARGETS
Options:
nights: list of YEARMMDD to scan for exposures
specprod_dir: override $DESI_SPECTRO_REDUX/$SPECPROD
nside: healpix nside, must be power of 2
comm: MPI communicator
Note: This could be replaced by a DB query when the production DB exists.
'''
log = get_logger()
if comm is None:
rank, size = 0, 1
else:
rank, size = comm.rank, comm.size
if specprod_dir is None:
specprod_dir = io.specprod_root()
if nights is None and rank == 0:
nights = io.get_nights(specprod_dir=specprod_dir)
if comm:
nights = comm.bcast(nights, root=0)
#-----
#- Distribute nights over ranks, scanning their exposures to build
#- map of exposures -> healpix
#- Rows to add to the output table
rows = list()
#- for tracking exposures that we've already mapped in a different band
night_expid_spectro = set()
for night in nights[rank::size]:
night = str(night)
nightdir = os.path.join(specprod_dir, 'exposures', night)
for expid in io.get_exposures(night,
specprod_dir=specprod_dir,
raw=False):
tmpframe = io.findfile('cframe',
night,
expid,
'r0',
specprod_dir=specprod_dir)
expdir = os.path.split(tmpframe)[0]
cframefiles = sorted(glob.glob(expdir + '/cframe*.fits'))
for filename in cframefiles:
#- parse 'path/night/expid/cframe-r0-12345678.fits'
camera = os.path.basename(filename).split('-')[1]
channel, spectro = camera[0], int(camera[1])
#- skip if we already have this expid/spectrograph
if (night, expid, spectro) in night_expid_spectro:
continue
else:
night_expid_spectro.add((night, expid, spectro))
log.debug('Rank {} mapping {} {}'.format(
rank, night, os.path.basename(filename)))
sys.stdout.flush()
#- Determine healpix, allowing for NaN
columns = ['TARGET_RA', 'TARGET_DEC']
fibermap = fitsio.read(filename, 'FIBERMAP', columns=columns)
ra, dec = fibermap['TARGET_RA'], fibermap['TARGET_DEC']
ok = ~np.isnan(ra) & ~np.isnan(dec)
ra, dec = ra[ok], dec[ok]
allpix = desimodel.footprint.radec2pix(nside, ra, dec)
#- Add rows for final output
for pix, ntargets in sorted(Counter(allpix).items()):
rows.append((night, expid, spectro, pix, ntargets))
#- Collect rows from individual ranks back to rank 0
if comm:
rank_rows = comm.gather(rows, root=0)
if rank == 0:
rows = list()
for r in rank_rows:
rows.extend(r)
else:
rows = None
rows = comm.bcast(rows, root=0)
#- Create the final output table
exp2healpix = np.array(rows,
dtype=[('NIGHT', 'i4'), ('EXPID', 'i8'),
('SPECTRO', 'i4'), ('HEALPIX', 'i8'),
('NTARGETS', 'i8')])
return exp2healpix
def parse(options=None):
"""
Can change night or number of spectra to be simulated and delete all output of test
Won't overwrite exisiting data unless overwrite argument provided
QuickLook data read from $QL_SPEC_DATA
QuickLook output written to $QL_SPEC_REDUX
Environment Variable check included here
"""
parser=argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--night',type=str,default='20160728',help='night to be simulated')
parser.add_argument('--nspec',type=int,default=5,help='number of spectra to be simulated, starting from first')
parser.add_argument('--overwrite', action='store_true', help='overwrite existing files')
parser.add_argument('--delete', action='store_true', help='delete all files generated by this test')
if options is None:
args = parser.parse_args()
else:
args = parser.parse_args(options)
log = logging.get_logger()
log.setLevel(logging.DEBUG)
missing_env = False
if 'DESI_BASIS_TEMPLATES' not in os.environ:
log.warning('missing $DESI_BASIS_TEMPLATES needed for simulating spectra'.format(name))
missing_env = True
if not os.path.isdir(os.getenv('DESI_BASIS_TEMPLATES')):
log.warning('missing $DESI_BASIS_TEMPLATES directory')
log.warning('e.g. see NERSC:/project/projectdirs/desi/spectro/templates/basis_templates/v1.0')
missing_env = True
for name in (
'DESI_SPECTRO_SIM', 'PIXPROD', 'DESIMODEL'):
if name not in os.environ:
log.warning("missing ${}".format(name))
missing_env = True
if 'QL_SPEC_REDUX' not in os.environ:
log.warning("missing ${}".format('QL_SPEC_REDUX'))
missing_env = True
if 'QL_SPEC_DATA' not in os.environ:
log.warning("missing ${}".format('QL_SPEC_DATA'))
missing_env = True
if missing_env:
log.warning("Why are these needed?")
log.warning(" Simulations written to $DESI_SPECTRO_SIM/$PIXPROD")
log.warning(" Raw data read from $QL_SPEC_DATA")
log.warning(" Spectro/QuickLook pipeline output written to $QL_SPEC_REDUX")
log.warning(" PSF files are found in $DESIMODEL")
log.warning(" Templates are read from $DESI_BASIS_TEMPLATES")
#- Wait until end to raise exception so that we report everything that
#- is missing before actually failing
if missing_env:
log.critical("missing env vars; exiting without running simulations or quicklook pipeline")
sys.exit(1)
sim_dir = os.path.join(os.environ['DESI_SPECTRO_SIM'],os.environ['PIXPROD'],args.night)
data_dir = os.path.join(os.environ['QL_SPEC_DATA'],args.night)
output_dir = os.environ['QL_SPEC_REDUX']
if args.overwrite:
if os.path.exists(sim_dir):
sim_files = os.listdir(sim_dir)
for file in range(len(sim_files)):
sim_file = os.path.join(sim_dir,sim_files[file])
os.remove(sim_file)
os.rmdir(sim_dir)
if os.path.exists(data_dir):
data_files = os.listdir(data_dir)
for file in range(len(data_files)):
data_file = os.path.join(data_dir,data_files[file])
os.remove(data_file)
os.rmdir(data_dir)
if os.path.exists(output_dir):
exp_dir = os.path.join(output_dir,'exposures',args.night)
calib_dir = os.path.join(output_dir,'calib2d',args.night)
if os.path.exists(exp_dir):
id_dir = os.path.join(exp_dir,'00000002')
if os.path.exists(id_dir):
id_files = os.listdir(id_dir)
for file in range(len(id_files)):
id_file = os.path.join(id_dir,id_files[file])
os.remove(id_file)
os.rmdir(id_dir)
exp_files = os.listdir(exp_dir)
for file in range(len(exp_files)):
exp_file = os.path.join(exp_dir,exp_files[file])
os.remove(exp_file)
os.rmdir(exp_dir)
if os.path.exists(calib_dir):
calib_files = os.listdir(calib_dir)
for file in range(len(calib_files)):
calib_file = os.path.join(calib_dir,calib_files[file])
os.remove(calib_file)
os.rmdir(calib_dir)
else:
if os.path.exists(sim_dir) or os.path.exists(data_dir) or os.path.exists(output_dir):
raise RuntimeError('Files already exist for this night! Can overwrite or change night if necessary')
return args
#!/usr/bin/env python
import argparse
from timedomain.filters import *
from timedomain.iterators import *
from timedomain.sp_utils import *
from timedomain.fs_utils import *
import timedomain.config as config
import sys
from astropy.table import Table
from desiutil.log import get_logger, DEBUG
log = get_logger(DEBUG)
def main(args):
""" Main entry point of the app """
print("Start ", args)
logic = getattr(sys.modules[__name__], args.logic)
iterator_ = getattr(sys.modules[__name__], args.iterator)
prunelogic = args.logic[0:args.logic.find("Logic")]
### Get the tile and array from the arguments
if args.obsdates_tilenumbers != None:
obsdates_tilenumbers_str = args.obsdates_tilenumbers
obsdates_tilenumbers = np.chararray((len(obsdates_tilenumbers_str), 2),
itemsize=10,
unicode=True)
for i in range(len(obsdates_tilenumbers_str)):
obsdates_tilenumbers[i, :] = obsdates_tilenumbers_str[i].split('|')
def get_nodes_per_exp(nnodes,nexposures,ncameras,user_nodes_per_comm_exp=None):
"""
Calculate how many nodes to use per exposure
Args:
nnodes: number of nodes in MPI COMM_WORLD (not number of ranks)
nexposures: number of exposures to process
ncameras: number of cameras per exposure
user_nodes_per_comm_exp (int, optional): user override of number of
nodes to use; used to check requirements
Returns number of nodes to include in sub-communicators used to process
individual exposures
Notes:
* Uses the largest number of nodes per exposure that will still
result in efficient node usage
* requires that (nexposures*ncameras) / nnodes = int
* the derived nodes_per_comm_exp * nexposures / nodes = int
* See desisim.test.test_pixsim.test_get_nodes_per_exp() for examples
* if user_nodes_per_comm_exp is given, requires that
GreatestCommonDivisor(nnodes, ncameras) / user_nodes_per_comm_exp = int
"""
from math import gcd
import desiutil.log as logging
log = logging.get_logger()
log.setLevel(logging.INFO)
#check if nframes is evenly divisible by nnodes
nframes = ncameras*nexposures
if nframes % nnodes !=0:
### msg=("nframes {} must be evenly divisible by nnodes {}, try again".format(nframes, nnodes))
### raise ValueError(msg)
msg=("nframes {} is not evenly divisible by nnodes {}; packing will be inefficient".format(nframes, nnodes))
log.warning(msg)
else:
log.debug("nframes {} is evenly divisible by nnodes {}, check passed".format(nframes, nnodes))
#find greatest common divisor between nnodes and ncameras
#greatest common divisor = greatest common factor
#we use python's built in gcd
greatest_common_factor=gcd(nnodes,ncameras)
#the greatest common factor must be greater than one UNLESS we are on one node
if nnodes > 1:
if greatest_common_factor == 1:
msg=("greatest common factor {} between nnodes {} and nframes {} must be larger than one, try again".format(greatest_common_factor, nnodes, nframes))
raise ValueError(msg)
else:
log.debug("greatest common factor {} between nnodes {} and nframes {} is greater than one, check passed".format(greatest_common_factor, nnodes, nframes))
#check to make sure the user hasn't specified a really asinine value of user_nodes_per_comm_exp
if user_nodes_per_comm_exp is not None:
if greatest_common_factor % user_nodes_per_comm_exp !=0:
msg=("user-specified value of user_nodes_per_comm_exp {} is bad, try again".format(user_nodes_per_comm_exp))
raise ValueError(msg)
else:
log.debug("user-specified value of user_nodes_per_comm_exp {} is good, check passed".format(user_nodes_per_comm_exp))
nodes_per_comm_exp=user_nodes_per_comm_exp
#if the user didn't specify anything, use the greatest common factor
if user_nodes_per_comm_exp is None:
nodes_per_comm_exp=greatest_common_factor
#finally check to make sure exposures*gcf/nnodes is an integer to avoid inefficient node use
if (nexposures*nodes_per_comm_exp) % nnodes != 0:
### msg=("nexposures {} * nodes_per_comm_exp {} does not divide evenly into nnodes {}, try again".format(nexposures, nodes_per_comm_exp, nnodes))
### raise ValueError(msg)
msg=("nexposures {} * nodes_per_comm_exp {} does not divide evenly into nnodes {}; packing will be inefficient".format(nexposures, nodes_per_comm_exp, nnodes))
log.warning(msg)
else:
log.debug("nexposures {} * nodes_per_comm_exp {} divides evenly into nnodes {}, check passed".format(nexposures, nodes_per_comm_exp, nnodes))
return nodes_per_comm_exp
def subtract_sky(frame,
skymodel,
throughput_correction=False,
default_throughput_correction=1.):
"""Subtract skymodel from frame, altering frame.flux, .ivar, and .mask
Args:
frame : desispec.Frame object
skymodel : desispec.SkyModel object
Option:
throughput_correction : if True, fit for an achromatic throughput correction. This is to absorb variations of Focal Ratio Degradation with fiber flexure.
default_throughput_correction : float, default value of correction if the fit on sky lines failed.
"""
assert frame.nspec == skymodel.nspec
assert frame.nwave == skymodel.nwave
log = get_logger()
log.info("starting")
# check same wavelength, die if not the case
if not np.allclose(frame.wave, skymodel.wave):
message = "frame and sky not on same wavelength grid"
log.error(message)
raise ValueError(message)
if throughput_correction:
# need to fit for a multiplicative factor of the sky model
# before subtraction
# we are going to use a set of bright sky lines,
# and fit a multiplicative factor + background around
# each of them individually, and then combine the results
# with outlier rejection in case a source emission line
# coincides with one of the sky lines.
# it's more robust to have a hardcoded set of sky lines here
# these are all the sky lines with a flux >5% of the max flux
# except in b where we add an extra weaker line at 5199.4A
skyline = np.array([
5199.4, 5578.4, 5656.4, 5891.4, 5897.4, 6302.4, 6308.4, 6365.4,
6500.4, 6546.4, 6555.4, 6618.4, 6663.4, 6679.4, 6690.4, 6765.4,
6831.4, 6836.4, 6865.4, 6925.4, 6951.4, 6980.4, 7242.4, 7247.4,
7278.4, 7286.4, 7305.4, 7318.4, 7331.4, 7343.4, 7360.4, 7371.4,
7394.4, 7404.4, 7440.4, 7526.4, 7714.4, 7719.4, 7752.4, 7762.4,
7782.4, 7796.4, 7810.4, 7823.4, 7843.4, 7855.4, 7862.4, 7873.4,
7881.4, 7892.4, 7915.4, 7923.4, 7933.4, 7951.4, 7966.4, 7982.4,
7995.4, 8016.4, 8028.4, 8064.4, 8280.4, 8284.4, 8290.4, 8298.4,
8301.4, 8313.4, 8346.4, 8355.4, 8367.4, 8384.4, 8401.4, 8417.4,
8432.4, 8454.4, 8467.4, 8495.4, 8507.4, 8627.4, 8630.4, 8634.4,
8638.4, 8652.4, 8657.4, 8662.4, 8667.4, 8672.4, 8677.4, 8683.4,
8763.4, 8770.4, 8780.4, 8793.4, 8829.4, 8835.4, 8838.4, 8852.4,
8870.4, 8888.4, 8905.4, 8922.4, 8945.4, 8960.4, 8990.4, 9003.4,
9040.4, 9052.4, 9105.4, 9227.4, 9309.4, 9315.4, 9320.4, 9326.4,
9340.4, 9378.4, 9389.4, 9404.4, 9422.4, 9442.4, 9461.4, 9479.4,
9505.4, 9521.4, 9555.4, 9570.4, 9610.4, 9623.4, 9671.4, 9684.4,
9693.4, 9702.4, 9714.4, 9722.4, 9740.4, 9748.4, 9793.4, 9802.4,
9814.4, 9820.4
])
sw = []
swf = []
sws = []
sws2 = []
swsf = []
# half width of wavelength region around each sky line
# larger values give a better statistical precision
# but also a larger sensitivity to source features
# best solution on one dark night exposure obtained with
# a half width of 4A.
hw = 4 #A
tivar = frame.ivar
if frame.mask is not None:
tivar *= (frame.mask == 0)
tivar *= (skymodel.ivar > 0)
# we precompute the quantities needed to fit each sky line + continuum
# the sky "line profile" is the actual sky model
# and we consider an additive constant
for line in skyline:
if line <= frame.wave[0] or line >= frame.wave[-1]: continue
ii = np.where((frame.wave >= line - hw)
& (frame.wave <= line + hw))[0]
if ii.size < 2: continue
sw.append(np.sum(tivar[:, ii], axis=1))
swf.append(np.sum(tivar[:, ii] * frame.flux[:, ii], axis=1))
swsf.append(
np.sum(tivar[:, ii] * frame.flux[:, ii] * skymodel.flux[:, ii],
axis=1))
sws.append(np.sum(tivar[:, ii] * skymodel.flux[:, ii], axis=1))
sws2.append(np.sum(tivar[:, ii] * skymodel.flux[:, ii]**2, axis=1))
nlines = len(sw)
for fiber in range(frame.flux.shape[0]):
# we solve the 2x2 linear system for each fiber and sky line
# and save the results for each fiber
coef = [] # list of scale values
var = [] # list of variance on scale values
for line in range(nlines):
if sw[line][fiber] <= 0: continue
A = np.array([[sw[line][fiber], sws[line][fiber]],
[sws[line][fiber], sws2[line][fiber]]])
B = np.array([swf[line][fiber], swsf[line][fiber]])
try:
Ai = np.linalg.inv(A)
X = Ai.dot(B)
coef.append(
X[1]
) # the scale coef (marginalized over cst background)
var.append(Ai[1, 1])
except:
pass
if len(coef) == 0:
log.warning("cannot corr. throughput. for fiber %d" % fiber)
continue
coef = np.array(coef)
var = np.array(var)
ivar = (var > 0) / (var + (var == 0) + 0.005**2)
ivar_for_outliers = (var > 0) / (var + (var == 0) + 0.02**2)
# loop for outlier rejection
failed = False
for loop in range(50):
a = np.sum(ivar)
if a <= 0:
log.warning(
"cannot corr. throughput. ivar=0 everywhere on sky lines for fiber %d"
% fiber)
failed = True
break
mcoef = np.sum(ivar * coef) / a
mcoeferr = 1 / np.sqrt(a)
nsig = 3.
chi2 = ivar_for_outliers * (coef - mcoef)**2
worst = np.argmax(chi2)
if chi2[worst] > nsig**2 * np.median(
chi2[chi2 > 0]): # with rough scaling of errors
#log.debug("discard a bad measurement for fiber %d"%(fiber))
ivar[worst] = 0
ivar_for_outliers[worst] = 0
else:
break
if failed:
continue
log.info(
"fiber #%03d throughput corr = %5.4f +- %5.4f (mean fiber flux=%f)"
% (fiber, mcoef, mcoeferr, np.median(frame.flux[fiber])))
'''
if np.abs(mcoef)>0.01 :
print(fiber,"mean coef=",mcoef,"all coef=",coef)
print(fiber,"all err=",np.sqrt(var))
print(fiber,"mean coef=",mcoef,"selected coef=",coef[ivar>0])
print(fiber,"select err=",np.sqrt(var[ivar>0]))
import matplotlib.pyplot as plt
x=np.arange(coef.size)
plt.errorbar(x,coef,np.sqrt(var),fmt="o")
plt.errorbar(x[ivar>0],coef[ivar>0],np.sqrt(var[ivar>0]),fmt="o")
plt.axhline(0.)
plt.axhline(mcoef)
plt.ylim(-0.11,0.11)
plt.grid()
plt.show()
'''
if mcoeferr > 0.01:
log.warning(
"throughput corr error = %5.4f is too large for fiber #%03d, do not apply correction"
% (mcoeferr, fiber))
throughput_correction_value = default_throughput_correction
else:
throughput_correction_value = mcoef
# apply this correction to the sky model even if we have not fit it (default can be 1 or 0)
skymodel.flux[fiber] *= throughput_correction_value
frame.flux -= skymodel.flux
frame.ivar = util.combine_ivar(frame.ivar, skymodel.ivar)
frame.mask |= skymodel.mask
log.info("done")
def write_image(outfile, image, meta=None):
"""Writes image object to outfile
Args:
outfile : output file string
image : desispec.image.Image object
(or any object with 2D array attributes image, ivar, mask)
Optional:
meta : dict-like object with metadata key/values (e.g. FITS header)
"""
log = get_logger()
if meta is not None:
hdr = fitsheader(meta)
else:
hdr = fitsheader(image.meta)
add_dependencies(hdr)
#- Work around fitsio>1.0 writing blank keywords, e.g. on 20191212
for key in hdr.keys():
if type(hdr[key]) == fits.card.Undefined:
log.warning('Setting blank keyword {} to None'.format(key))
hdr[key] = None
outdir = os.path.dirname(os.path.abspath(outfile))
if not os.path.isdir(outdir):
os.makedirs(outdir)
hx = fits.HDUList()
hdu = fits.ImageHDU(image.pix.astype(np.float32), name='IMAGE', header=hdr)
if 'CAMERA' not in hdu.header:
hdu.header.append(
('CAMERA', image.camera.lower(), 'Spectrograph Camera'))
if 'RDNOISE' not in hdu.header and np.isscalar(image.readnoise):
hdu.header.append(
('RDNOISE', image.readnoise, 'Read noise [RMS electrons/pixel]'))
hx.append(hdu)
hx.append(fits.ImageHDU(image.ivar.astype(np.float32), name='IVAR'))
hx.append(fits.CompImageHDU(image.mask.astype(np.int16), name='MASK'))
if not np.isscalar(image.readnoise):
hx.append(
fits.ImageHDU(image.readnoise.astype(np.float32),
name='READNOISE'))
if hasattr(image, 'fibermap'):
if isinstance(image.fibermap, Table):
fmhdu = fits.convenience.table_to_hdu(image.fibermap)
fmhdu.name = 'FIBERMAP'
else:
fmhdu = fits.BinTableHDU(image.fibermap, name='FIBERMAP')
hx.append(fmhdu)
hx.writeto(outfile + '.tmp', overwrite=True, checksum=True)
os.rename(outfile + '.tmp', outfile)
return outfile
def compute_polynomial_times_sky(frame,
nsig_clipping=4.,
max_iterations=30,
model_ivar=False,
add_variance=True,
angular_variation_deg=1,
chromatic_variation_deg=1):
"""Compute a sky model.
Sky[fiber,i] = R[fiber,i,j] Polynomial(x[fiber],y[fiber],wavelength[j]) Flux[j]
Input flux are expected to be flatfielded!
We don't check this in this routine.
Args:
frame : Frame object, which includes attributes
- wave : 1D wavelength grid in Angstroms
- flux : 2D flux[nspec, nwave] density
- ivar : 2D inverse variance of flux
- mask : 2D inverse mask flux (0=good)
- resolution_data : 3D[nspec, ndiag, nwave] (only sky fibers)
nsig_clipping : [optional] sigma clipping value for outlier rejection
Optional:
max_iterations : int , number of iterations
model_ivar : replace ivar by a model to avoid bias due to correlated flux and ivar. this has a negligible effect on sims.
add_variance : evaluate calibration error and add this to the sky model variance
returns SkyModel object with attributes wave, flux, ivar, mask
"""
log = get_logger()
log.info("starting")
# Grab sky fibers on this frame
skyfibers = np.where(frame.fibermap['OBJTYPE'] == 'SKY')[0]
assert np.max(skyfibers) < 500 #- indices, not fiber numbers
nwave = frame.nwave
nfibers = len(skyfibers)
current_ivar = frame.ivar[skyfibers].copy() * (frame.mask[skyfibers] == 0)
flux = frame.flux[skyfibers]
Rsky = frame.R[skyfibers]
input_ivar = None
if model_ivar:
log.info(
"use a model of the inverse variance to remove bias due to correlated ivar and flux"
)
input_ivar = current_ivar.copy()
median_ivar_vs_wave = np.median(current_ivar, axis=0)
median_ivar_vs_fiber = np.median(current_ivar, axis=1)
median_median_ivar = np.median(median_ivar_vs_fiber)
for f in range(current_ivar.shape[0]):
threshold = 0.01
current_ivar[f] = median_ivar_vs_fiber[
f] / median_median_ivar * median_ivar_vs_wave
# keep input ivar for very low weights
ii = (input_ivar[f] <= (threshold * median_ivar_vs_wave))
#log.info("fiber {} keep {}/{} original ivars".format(f,np.sum(ii),current_ivar.shape[1]))
current_ivar[f][ii] = input_ivar[f][ii]
# need focal plane coordinates
x = frame.fibermap["FIBERASSIGN_X"]
y = frame.fibermap["FIBERASSIGN_Y"]
# normalize for numerical stability
xm = np.mean(x)
ym = np.mean(y)
xs = np.std(x)
ys = np.std(y)
if xs == 0: xs = 1
if ys == 0: ys = 1
x = (x - xm) / xs
y = (y - ym) / ys
w = (frame.wave - frame.wave[0]) / (frame.wave[-1] -
frame.wave[0]) * 2. - 1
# precompute the monomials for the sky fibers
log.debug("compute monomials for deg={} and {}".format(
angular_variation_deg, chromatic_variation_deg))
monomials = []
for dx in range(angular_variation_deg + 1):
for dy in range(angular_variation_deg + 1 - dx):
xypol = (x**dx) * (y**dy)
for dw in range(chromatic_variation_deg + 1):
wpol = w**dw
monomials.append(np.outer(xypol, wpol))
ncoef = len(monomials)
coef = np.zeros((ncoef))
allfibers_monomials = np.array(monomials)
log.debug("shape of allfibers_monomials = {}".format(
allfibers_monomials.shape))
skyfibers_monomials = allfibers_monomials[:, skyfibers, :]
log.debug("shape of skyfibers_monomials = {}".format(
skyfibers_monomials.shape))
sqrtw = np.sqrt(current_ivar)
sqrtwflux = sqrtw * flux
chi2 = np.zeros(flux.shape)
Pol = np.ones(flux.shape, dtype=float)
coef[0] = 1.
nout_tot = 0
previous_chi2 = -10.
for iteration in range(max_iterations):
# the matrix A is 1/2 of the second derivative of the chi2 with respect to the parameters
# A_ij = 1/2 d2(chi2)/di/dj
# A_ij = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * d(model)/dj[fiber,w]
# the vector B is 1/2 of the first derivative of the chi2 with respect to the parameters
# B_i = 1/2 d(chi2)/di
# B_i = sum_fiber sum_wave_w ivar[fiber,w] d(model)/di[fiber,w] * (flux[fiber,w]-model[fiber,w])
# the model is model[fiber]=R[fiber]*Pol(x,y,wave)*sky
# the parameters are the unconvolved sky flux at the wavelength i
# and the polynomial coefficients
A = np.zeros((nwave, nwave), dtype=float)
B = np.zeros((nwave), dtype=float)
D = scipy.sparse.lil_matrix((nwave, nwave))
D2 = scipy.sparse.lil_matrix((nwave, nwave))
Pol /= coef[0] # force constant term to 1.
# solving for the deconvolved mean sky spectrum
# loop on fiber to handle resolution
for fiber in range(nfibers):
if fiber % 10 == 0:
log.info("iter %d sky fiber (1st fit) %d/%d" %
(iteration, fiber, nfibers))
D.setdiag(sqrtw[fiber])
D2.setdiag(Pol[fiber])
sqrtwRP = D.dot(Rsky[fiber]).dot(
D2) # each row r of R is multiplied by sqrtw[r]
A += (sqrtwRP.T * sqrtwRP).todense()
B += sqrtwRP.T * sqrtwflux[fiber]
log.info("iter %d solving" % iteration)
w = A.diagonal() > 0
A_pos_def = A[w, :]
A_pos_def = A_pos_def[:, w]
parameters = B * 0
try:
parameters[w] = cholesky_solve(A_pos_def, B[w])
except:
log.info("cholesky failed, trying svd in iteration {}".format(
iteration))
parameters[w] = np.linalg.lstsq(A_pos_def, B[w])[0]
# parameters = the deconvolved mean sky spectrum
# now evaluate the polynomial coefficients
Ap = np.zeros((ncoef, ncoef), dtype=float)
Bp = np.zeros((ncoef), dtype=float)
D2.setdiag(parameters)
for fiber in range(nfibers):
if fiber % 10 == 0:
log.info("iter %d sky fiber (2nd fit) %d/%d" %
(iteration, fiber, nfibers))
D.setdiag(sqrtw[fiber])
sqrtwRSM = D.dot(Rsky[fiber]).dot(D2).dot(
skyfibers_monomials[:, fiber, :].T)
Ap += sqrtwRSM.T.dot(sqrtwRSM)
Bp += sqrtwRSM.T.dot(sqrtwflux[fiber])
# Add huge prior on zeroth angular order terms to converge faster
# (because those terms are degenerate with the mean deconvolved spectrum)
weight = 1e24
Ap[0, 0] += weight
Bp[0] += weight # force 0th term to 1
for i in range(1, chromatic_variation_deg + 1):
Ap[i, i] += weight # force other wavelength terms to 0
coef = cholesky_solve(Ap, Bp)
log.info("pol coef = {}".format(coef))
# recompute the polynomial values
Pol = skyfibers_monomials.T.dot(coef).T
# chi2 and outlier rejection
log.info("iter %d compute chi2" % iteration)
for fiber in range(nfibers):
chi2[fiber] = current_ivar[fiber] * (
flux[fiber] - Rsky[fiber].dot(Pol[fiber] * parameters))**2
log.info("rejecting")
nout_iter = 0
if iteration < 1:
# only remove worst outlier per wave
# apply rejection iteratively, only one entry per wave among fibers
# find waves with outlier (fastest way)
nout_per_wave = np.sum(chi2 > nsig_clipping**2, axis=0)
selection = np.where(nout_per_wave > 0)[0]
for i in selection:
worst_entry = np.argmax(chi2[:, i])
current_ivar[worst_entry, i] = 0
sqrtw[worst_entry, i] = 0
sqrtwflux[worst_entry, i] = 0
nout_iter += 1
else:
# remove all of them at once
bad = (chi2 > nsig_clipping**2)
current_ivar *= (bad == 0)
sqrtw *= (bad == 0)
sqrtwflux *= (bad == 0)
nout_iter += np.sum(bad)
nout_tot += nout_iter
sum_chi2 = float(np.sum(chi2))
ndf = int(np.sum(chi2 > 0) - nwave)
chi2pdf = 0.
if ndf > 0:
chi2pdf = sum_chi2 / ndf
log.info("iter #%d chi2=%g ndf=%d chi2pdf=%f delta=%f nout=%d" %
(iteration, sum_chi2, ndf, chi2pdf,
abs(sum_chi2 - previous_chi2), nout_iter))
if nout_iter == 0 and abs(sum_chi2 - previous_chi2) < 0.2:
break
previous_chi2 = sum_chi2 + 0.
log.info("nout tot=%d" % nout_tot)
# we know have to compute the sky model for all fibers
# and propagate the uncertainties
# no need to restore the original ivar to compute the model errors when modeling ivar
# the sky inverse variances are very similar
# we ignore here the fact that we have fit a angular variation,
# so the sky model uncertainties are inaccurate
log.info("compute the parameter covariance")
try:
parameter_covar = cholesky_invert(A)
except np.linalg.linalg.LinAlgError:
log.warning(
"cholesky_solve_and_invert failed, switching to np.linalg.lstsq and np.linalg.pinv"
)
parameter_covar = np.linalg.pinv(A)
log.info("compute mean resolution")
# we make an approximation for the variance to save CPU time
# we use the average resolution of all fibers in the frame:
mean_res_data = np.mean(frame.resolution_data, axis=0)
Rmean = Resolution(mean_res_data)
log.info("compute convolved sky and ivar")
# The parameters are directly the unconvolved sky
# First convolve with average resolution :
convolved_sky_covar = Rmean.dot(parameter_covar).dot(Rmean.T.todense())
# and keep only the diagonal
convolved_sky_var = np.diagonal(convolved_sky_covar)
# inverse
convolved_sky_ivar = (convolved_sky_var > 0) / (convolved_sky_var +
(convolved_sky_var == 0))
# and simply consider it's the same for all spectra
cskyivar = np.tile(convolved_sky_ivar,
frame.nspec).reshape(frame.nspec, nwave)
# The sky model for each fiber (simple convolution with resolution of each fiber)
cskyflux = np.zeros(frame.flux.shape)
Pol = allfibers_monomials.T.dot(coef).T
for fiber in range(frame.nspec):
cskyflux[fiber] = frame.R[fiber].dot(Pol[fiber] * parameters)
# look at chi2 per wavelength and increase sky variance to reach chi2/ndf=1
if skyfibers.size > 1 and add_variance:
modified_cskyivar = _model_variance(frame, cskyflux, cskyivar,
skyfibers)
else:
modified_cskyivar = cskyivar.copy()
# need to do better here
mask = (cskyivar == 0).astype(np.uint32)
return SkyModel(
frame.wave.copy(),
cskyflux,
modified_cskyivar,
mask,
nrej=nout_tot,
stat_ivar=cskyivar) # keep a record of the statistical ivar for QA
def main(args=None):
log = get_logger()
if args is None:
args = parse()
if args.lin_step is not None and args.log10_step is not None:
log.critical(
"cannot have both linear and logarthmic bins :-), choose either --lin-step or --log10-step"
)
return 12
if args.coadd_cameras and (args.lin_step is not None
or args.log10_step is not None):
log.critical(
"cannot specify a new wavelength binning along with --coadd-cameras option"
)
return 12
if len(args.infile) == 0:
log.critical("You must specify input files")
return 12
log.info("reading input ...")
# inspect headers
input_is_frames = False
input_is_spectra = False
for filename in args.infile:
ifile = fitsio.FITS(filename)
head = ifile[0].read_header()
identified = False
if "EXTNAME" in head and head["EXTNAME"] == "FLUX":
print(filename, "is a frame")
input_is_frames = True
identified = True
ifile.close()
continue
for hdu in ifile:
head = hdu.read_header()
if "EXTNAME" in head and head["EXTNAME"].find("_FLUX") >= 0:
print(filename, "is a spectra")
input_is_spectra = True
identified = True
break
ifile.close()
if not identified:
log.error(
"{} not identified as frame of spectra file".format(filename))
sys.exit(1)
if input_is_frames and input_is_spectra:
log.error("cannot combine input spectra and frames")
sys.exit(1)
if input_is_spectra:
spectra = read_spectra(args.infile[0])
for filename in args.infile[1:]:
log.info("append {}".format(filename))
spectra.update(read_spectra(filename))
else: # frames
frames = dict()
cameras = {}
for filename in args.infile:
frame = read_frame(filename)
night = frame.meta['NIGHT']
expid = frame.meta['EXPID']
camera = frame.meta['CAMERA']
frames[(night, expid, camera)] = frame
if args.coadd_cameras:
cam, spec = camera[0], camera[1]
# Keep a list of cameras (b,r,z) for each exposure + spec
if (night, expid) not in cameras.keys():
cameras[(night, expid)] = {spec: [cam]}
elif spec not in cameras[(night, expid)].keys():
cameras[(night, expid)][spec] = [cam]
else:
cameras[(night, expid)][spec].append(cam)
if args.coadd_cameras:
# If not all 3 cameras are available, remove the incomplete sets
for (night, expid), camdict in cameras.items():
for spec, camlist in camdict.items():
log.info("Found {} for SP{} on NIGHT {} EXP {}".format(
camlist, spec, night, expid))
if len(camlist) != 3 or np.any(
np.sort(camlist) != np.array(['b', 'r', 'z'])):
for cam in camlist:
frames.pop((night, expid, cam + spec))
log.warning(
"Removing {}{} from Night {} EXP {}".format(
cam, spec, night, expid))
#import pdb
#pdb.set_trace()
spectra = frames2spectra(frames)
#- hacks to make SpectraLite like a Spectra
spectra.fibermap = Table(spectra.fibermap)
del frames #- maybe free some memory
if args.coadd_cameras:
log.info("coadding cameras ...")
spectra = coadd_cameras(spectra, cosmics_nsig=args.nsig)
else:
log.info("coadding ...")
coadd(spectra, cosmics_nsig=args.nsig)
if args.lin_step is not None:
log.info("resampling ...")
spectra = resample_spectra_lin_or_log(spectra,
linear_step=args.lin_step,
wave_min=args.wave_min,
wave_max=args.wave_max,
fast=args.fast,
nproc=args.nproc)
if args.log10_step is not None:
log.info("resampling ...")
spectra = resample_spectra_lin_or_log(spectra,
log10_step=args.log10_step,
wave_min=args.wave_min,
wave_max=args.wave_max,
fast=args.fast,
nproc=args.nproc)
#- Add input files to header
if spectra.meta is None:
spectra.meta = dict()
for i, filename in enumerate(args.infile):
spectra.meta['INFIL{:03d}'.format(i)] = os.path.basename(filename)
log.info("writing {} ...".format(args.outfile))
write_spectra(args.outfile, spectra)
log.info("done")
def read_raw(filename, camera, **kwargs):
'''
Returns preprocessed raw data from `camera` extension of `filename`
Args:
filename : input fits filename with DESI raw data
camera : camera name (B0,R1, .. Z9) or FITS extension name or number
Options:
Other keyword arguments are passed to desispec.preproc.preproc(),
e.g. bias, pixflat, mask. See preproc() documentation for details.
Returns Image object with member variables pix, ivar, mask, readnoise
'''
log = get_logger()
fx = fits.open(filename, memmap=False)
if camera.upper() not in fx:
raise IOError('Camera {} not in {}'.format(camera, filename))
rawimage = fx[camera.upper()].data
header = fx[camera.upper()].header
primary_header = fx[0].header
blacklist = [
"EXTEND", "SIMPLE", "NAXIS1", "NAXIS2", "CHECKSUM", "DATASUM",
"XTENSION", "EXTNAME", "COMMENT"
]
if 'INHERIT' in header and header['INHERIT']:
h0 = fx[0].header
for key in h0:
if (key not in blacklist) and (key not in header):
header[key] = h0[key]
if "fill_header" in kwargs:
hdus = kwargs["fill_header"]
if hdus is not None:
log.info("will add header keywords from hdus %s" % str(hdus))
for hdu in hdus:
try:
ihdu = int(hdu)
hdu = ihdu
except ValueError:
pass
if hdu in fx:
hdu_header = fx[hdu].header
for key in hdu_header:
if (key not in blacklist) and (key not in header):
log.debug("adding {} = {}".format(
key, hdu_header[key]))
header[key] = hdu_header[key]
else:
log.debug(
"key %s already in header or blacklisted" %
key)
else:
log.warning("warning HDU %s not in fits file" % str(hdu))
kwargs.pop("fill_header")
fx.close()
img = desispec.preproc.preproc(rawimage, header, primary_header, **kwargs)
return img
def make_mtl(targets, zcat=None, trim=False):
"""Adds NUMOBS, PRIORITY, and OBSCONDITIONS columns to a targets table.
Parameters
----------
targets : :class:`~numpy.array` or `~astropy.table.Table`
A numpy rec array or astropy Table with at least the columns
``TARGETID``, ``DESI_TARGET``, ``NUMOBS_INIT``, ``PRIORITY_INIT``.
or the corresponding columns for SV or commissioning.
zcat : :class:`~astropy.table.Table`, optional
Redshift catalog table with columns ``TARGETID``, ``NUMOBS``, ``Z``,
``ZWARN``.
trim : :class:`bool`, optional
If ``True`` (default), don't include targets that don't need
any more observations. If ``False``, include every input target.
Returns
-------
:class:`~astropy.table.Table`
MTL Table with targets columns plus:
* NUMOBS_MORE - number of additional observations requested
* PRIORITY - target priority (larger number = higher priority)
* OBSCONDITIONS - replaces old GRAYLAYER
"""
# ADM set up the default logger.
from desiutil.log import get_logger
log = get_logger()
# ADM determine whether the input targets are main survey, cmx or SV.
colnames, masks, survey = main_cmx_or_sv(targets)
# ADM set the first column to be the "desitarget" column
desi_target, desi_mask = colnames[0], masks[0]
# Trim targets from zcat that aren't in original targets table
if zcat is not None:
ok = np.in1d(zcat['TARGETID'], targets['TARGETID'])
num_extra = np.count_nonzero(~ok)
if num_extra > 0:
log.warning("Ignoring {} zcat entries that aren't "
"in the input target list".format(num_extra))
zcat = zcat[ok]
n = len(targets)
# ADM if the input target columns were incorrectly called NUMOBS or PRIORITY
# ADM rename them to NUMOBS_INIT or PRIORITY_INIT.
# ADM Note that the syntax is slightly different for a Table.
for name in ['NUMOBS', 'PRIORITY']:
if isinstance(targets, Table):
try:
targets.rename_column(name, name + '_INIT')
except KeyError:
pass
else:
targets.dtype.names = [
name + '_INIT' if col == name else col
for col in targets.dtype.names
]
# ADM if a redshift catalog was passed, order it to match the input targets
# ADM catalog on 'TARGETID'.
if zcat is not None:
# ADM there might be a quicker way to do this?
# ADM set up a dictionary of the indexes of each target id.
d = dict(tuple(zip(targets["TARGETID"], np.arange(n))))
# ADM loop through the zcat and look-up the index in the dictionary.
zmatcher = np.array([d[tid] for tid in zcat["TARGETID"]])
ztargets = zcat
if ztargets.masked:
unobs = ztargets['NUMOBS'].mask
ztargets['NUMOBS'][unobs] = 0
unobsz = ztargets['Z'].mask
ztargets['Z'][unobsz] = -1
unobszw = ztargets['ZWARN'].mask
ztargets['ZWARN'][unobszw] = -1
else:
ztargets = Table()
ztargets['TARGETID'] = targets['TARGETID']
ztargets['NUMOBS'] = np.zeros(n, dtype=np.int32)
ztargets['Z'] = -1 * np.ones(n, dtype=np.float32)
ztargets['ZWARN'] = -1 * np.ones(n, dtype=np.int32)
# ADM if zcat wasn't passed, there is a one-to-one correspondence
# ADM between the targets and the zcat.
zmatcher = np.arange(n)
# ADM extract just the targets that match the input zcat.
targets_zmatcher = targets[zmatcher]
# ADM use passed value of NUMOBS_INIT instead of calling the memory-heavy calc_numobs.
# ztargets['NUMOBS_MORE'] = np.maximum(0, calc_numobs(ztargets) - ztargets['NUMOBS'])
ztargets['NUMOBS_MORE'] = np.maximum(
0, targets_zmatcher['NUMOBS_INIT'] - ztargets['NUMOBS'])
# ADM we need a minor hack to ensure that BGS targets are observed once (and only once)
# ADM every time, regardless of how many times they've previously been observed.
# ADM I've turned this off for commissioning. Not sure if we'll keep it in general.
if survey != 'cmx':
ii = targets_zmatcher[desi_target] & desi_mask.BGS_ANY > 0
ztargets['NUMOBS_MORE'][ii] = 1
# ADM assign priorities, note that only things in the zcat can have changed priorities.
# ADM anything else will be assigned PRIORITY_INIT, below.
priority = calc_priority(targets_zmatcher, ztargets)
# If priority went to 0==DONOTOBSERVE or 1==OBS or 2==DONE, then NUMOBS_MORE should also be 0.
# ## mtl['NUMOBS_MORE'] = ztargets['NUMOBS_MORE']
ii = (priority <= 2)
log.info(
'{:d} of {:d} targets have priority zero, setting N_obs=0.'.format(
np.sum(ii), n))
ztargets['NUMOBS_MORE'][ii] = 0
# - Set the OBSCONDITIONS mask for each target bit.
obscon = set_obsconditions(targets)
# ADM set up the output mtl table.
mtl = Table(targets)
mtl.meta['EXTNAME'] = 'MTL'
# ADM any target that wasn't matched to the ZCAT should retain its
# ADM original (INIT) value of PRIORITY and NUMOBS.
mtl['NUMOBS_MORE'] = mtl['NUMOBS_INIT']
mtl['PRIORITY'] = mtl['PRIORITY_INIT']
# ADM now populate the new mtl columns with the updated information.
mtl['OBSCONDITIONS'] = obscon
mtl['PRIORITY'][zmatcher] = priority
mtl['NUMOBS_MORE'][zmatcher] = ztargets['NUMOBS_MORE']
# Filter out any targets marked as done.
if trim:
notdone = mtl['NUMOBS_MORE'] > 0
log.info('{:d} of {:d} targets are done, trimming these'.format(
len(mtl) - np.sum(notdone), len(mtl)))
mtl = mtl[notdone]
# Filtering can reset the fill_value, which is just wrong wrong wrong
# See https://github.com/astropy/astropy/issues/4707
# and https://github.com/astropy/astropy/issues/4708
mtl['NUMOBS_MORE'].fill_value = -1
return mtl
