def test_step_by_step(master_dir):
# Masters
spectrograph = load_spectrograph('shane_kast_blue')
msarc, tslits_dict, mstrace = load_kast_blue_masters(aimg=True, tslits=True)
# Instantiate
master_key = 'A_1_01'
parset = spectrograph.default_pypeit_par()
par = parset['calibrations']['tilts']
wavepar = parset['calibrations']['wavelengths']
waveTilts = wavetilts.WaveTilts(msarc, tslits_dict, spectrograph, par, wavepar,
det=1, master_key=master_key,
master_dir=master_dir,reuse_masters=True)
# Extract arcs
arccen, maskslits = waveTilts.extract_arcs(waveTilts.slitcen, waveTilts.slitmask, msarc, waveTilts.inmask)
assert arccen.shape == (2048,1)
# Tilts in the slit
slit = 0
waveTilts.slitmask = pixels.tslits2mask(waveTilts.tslits_dict)
thismask = waveTilts.slitmask == slit
waveTilts.lines_spec, waveTilts.lines_spat = waveTilts.find_lines(arccen[:, slit], waveTilts.slitcen[:, slit], slit)
trcdict = waveTilts.trace_tilts(waveTilts.msarc, waveTilts.lines_spec, waveTilts.lines_spat, thismask, slit)
assert isinstance(trcdict, dict)
# 2D Fit
spat_order = waveTilts._parse_param(waveTilts.par, 'spat_order', slit)
spec_order = waveTilts._parse_param(waveTilts.par, 'spec_order', slit)
coeffs = waveTilts.fit_tilts(trcdict, thismask, waveTilts.slitcen[:, slit], spat_order, spec_order,slit, doqa=False)
tilts = tracewave.fit2tilts(waveTilts.slitmask_science.shape, coeffs, waveTilts.par['func2d'])
assert np.max(tilts) < 1.01
def __init__(self,
tslits_dict,
tilts,
wv_calib,
spectrograph,
maskslits,
master_key=None,
master_dir=None,
reuse_masters=False):
# MasterFrame
masterframe.MasterFrame.__init__(self,
self.frametype,
master_key,
master_dir,
reuse_masters=reuse_masters)
# Required parameters (but can be None)
self.tslits_dict = tslits_dict
self.tilts = tilts
self.wv_calib = wv_calib
self.spectrograph = spectrograph
self.slitmask = pixels.tslits2mask(
self.tslits_dict) if tslits_dict is not None else None
self.par = wv_calib['par'] if wv_calib is not None else None
# Optional parameters
self.maskslits = maskslits
# Attributes
self.steps = [
] # List to hold ouptut from inspect about what module create the image?
# Main output
self.wave = None
def __init__(self, msarc, tslits_dict, spectrograph, par, binspectral=None, det=1,
master_key=None, master_dir=None, reuse_masters=False, redux_path=None, msbpm=None):
# MasterFrame
masterframe.MasterFrame.__init__(self, self.frametype, master_key,
master_dir, reuse_masters=reuse_masters)
# Required parameters (but can be None)
self.msarc = msarc
self.tslits_dict = tslits_dict
self.spectrograph = spectrograph
self.par = par
# Optional parameters
self.bpm = msbpm
self.binspectral = binspectral
self.redux_path = redux_path
self.det = det
self.master_key = master_key
# Attributes
self.steps = [] # steps executed
self.wv_calib = {} # main output
self.arccen = None # central arc spectrum
# TODO this code is duplicated verbatim in wavetilts. Should it be a function
if self.spectrograph is not None:
if self.spectrograph.detector is not None:
self.nonlinear_counts = self.spectrograph.detector[self.det-1]['saturation']*self.spectrograph.detector[self.det-1]['nonlinear']
else:
self.nonlinear_counts=1e10
# Set the slitmask and slit boundary related attributes that the code needs for execution. This also deals with
# arcimages that have a different binning then the trace images used to defined the slits
if self.tslits_dict is not None and self.msarc is not None:
self.slitmask_science = pixels.tslits2mask(self.tslits_dict)
inmask = (self.bpm == 0) if self.bpm is not None else np.ones_like(self.slitmask_science, dtype=bool)
self.shape_science = self.slitmask_science.shape
self.shape_arc = self.msarc.shape
self.nslits = self.tslits_dict['slit_left'].shape[1]
self.slit_left = arc.resize_slits2arc(self.shape_arc, self.shape_science, self.tslits_dict['slit_left'])
self.slit_righ = arc.resize_slits2arc(self.shape_arc, self.shape_science, self.tslits_dict['slit_righ'])
self.slitcen = arc.resize_slits2arc(self.shape_arc, self.shape_science, self.tslits_dict['slitcen'])
self.slitmask = arc.resize_mask2arc(self.shape_arc, self.slitmask_science)
self.inmask = arc.resize_mask2arc(self.shape_arc,inmask)
# TODO -- Remove the following two lines if deemed ok
if self.par['method'] != 'full_template':
self.inmask &= self.msarc < self.nonlinear_counts
else:
self.slitmask_science = None
self.shape_science = None
self.shape_arc = None
self.nslits = 0
self.slit_left = None
self.slit_righ = None
self.slitcen = None
self.slitmask = None
self.inmask = None
def __init__(self, msarc, tslits_dict, spectrograph, par, wavepar, det=1,
master_key=None, master_dir=None, reuse_masters=False, redux_path=None, bpm=None):
self.spectrograph = spectrograph
self.par = par # pypeitpar.WaveTiltsPar() if par is None else par
self.wavepar = wavepar # pypeitpar.WavelengthSolutionPar() if wavepar is None else wavepar
# MasterFrame
masterframe.MasterFrame.__init__(self, self.frametype, master_key,
master_dir, reuse_masters=reuse_masters)
# Parameters (but can be None)
self.msarc = msarc
self.tslits_dict = tslits_dict
self.bpm = bpm
# Optional parameters
self.det = det
self.redux_path = redux_path
#
if self.spectrograph is not None:
self.nonlinear_counts = self.spectrograph.detector[self.det-1]['saturation']*self.spectrograph.detector[self.det-1]['nonlinear']
else:
self.nonlinear_counts=1e10
# Set the slitmask and slit boundary related attributes that the code needs for execution. This also deals with
# arcimages that have a different binning then the trace images used to defined the slits
if self.tslits_dict is not None and self.msarc is not None:
self.slitmask_science = pixels.tslits2mask(self.tslits_dict)
inmask = (self.bpm == 0) if self.bpm is not None else np.ones_like(self.slitmask_science, dtype=bool)
self.shape_science = self.slitmask_science.shape
self.shape_arc = self.msarc.shape
self.nslits = self.tslits_dict['slit_left'].shape[1]
self.slit_left = arc.resize_slits2arc(self.shape_arc, self.shape_science, self.tslits_dict['slit_left'])
self.slit_righ = arc.resize_slits2arc(self.shape_arc, self.shape_science, self.tslits_dict['slit_righ'])
self.slitcen = arc.resize_slits2arc(self.shape_arc, self.shape_science, self.tslits_dict['slitcen'])
self.slitmask = arc.resize_mask2arc(self.shape_arc, self.slitmask_science)
self.inmask = (arc.resize_mask2arc(self.shape_arc, inmask)) & (self.msarc < self.nonlinear_counts)
else:
self.slitmask_science = None
self.shape_science = None
self.shape_arc = None
self.nslits = 0
self.slit_left = None
self.slit_righ = None
self.slitcen = None
self.slitmask = None
self.inmask = None
# Key Internals
self.mask = None
self.all_trace_dict = [None]*self.nslits
self.tilts = None
# 2D fits are stored as a dictionary rather than list because we will jsonify the dict
self.all_fit_dict = [None]*self.nslits
self.steps = []
# Main outputs
self.final_tilts = None
self.fit_dict = None
self.trace_dict = None
def __init__(self,
sciImg,
spectrograph,
tslits_dict,
par,
tilts,
ir_redux=False,
det=1,
objtype='science',
binning=None,
setup=None,
maskslits=None):
# Setup the parameters sets for this object. NOTE: This uses objtype, not frametype!
self.objtype = objtype
self.par = par
# TODO -- I find it very confusing to break apart the main parset
self.proc_par = self.par['scienceframe']['process']
# TODO Rename the scienceimage arset to reduce.
self.redux_par = self.par['scienceimage']
self.wave_par = self.par['calibrations']['wavelengths']
self.flex_par = self.par['flexure']
# Instantiation attributes for this object
self.sciImg = sciImg
self.spectrograph = spectrograph
self.tslits_dict = tslits_dict
self.slitmask = pixels.tslits2mask(self.tslits_dict)
# Now add the slitmask to the mask (i.e. post CR rejection in proc)
self.sciImg.update_mask_slitmask(self.slitmask)
self.maskslits = self._get_goodslits(maskslits)
self.ir_redux = ir_redux
self.det = det
self.tilts = tilts
self.binning = binning
self.setup = setup
self.pypeline = spectrograph.pypeline
self.steps = []
# Other attributes that will be set later during object finding,
# sky-subtraction, and extraction
self.waveimage = None # used by extract
# Key outputs images for extraction
self.ivarmodel = None
self.objimage = None
self.skyimage = None
self.global_sky = None
self.skymask = None
self.outmask = None
self.extractmask = None
# SpecObjs object
self.sobjs_obj = None # Only object finding but no extraction
self.sobjs = None # Final extracted object list with trace corrections applied
def __init__(self,
sciImg,
spectrograph,
par,
caliBrate,
ir_redux=False,
det=1,
std_redux=False,
show=False,
objtype='science',
binning=None,
setup=None,
maskslits=None):
# Setup the parameters sets for this object. NOTE: This uses objtype, not frametype!
# Instantiation attributes for this object
self.sciImg = sciImg
self.spectrograph = spectrograph
self.objtype = objtype
self.par = par
#self.extraction_par = self.par['scienceimage']['extraction']
#self.wave_par = self.par['calibrations']['wavelengths']
#self.flex_par = self.par['flexure']
# Parse
self.caliBrate = caliBrate
self.tslits_dict = self.caliBrate.tslits_dict
self.tilts = self.caliBrate.tilts_dict['tilts']
# Now add the slitmask to the mask (i.e. post CR rejection in proc)
self.slitmask = pixels.tslits2mask(self.tslits_dict)
self.sciImg.update_mask_slitmask(self.slitmask)
self.maskslits = self._get_goodslits(maskslits)
# Load up other input items
self.ir_redux = ir_redux
self.std_redux = std_redux
self.det = det
self.binning = binning
self.setup = setup
self.pypeline = spectrograph.pypeline
self.reduce_show = show
self.steps = []
# Key outputs images for extraction
self.ivarmodel = None
self.objimage = None
self.skyimage = None
self.initial_sky = None
self.global_sky = None
self.skymask = None
self.outmask = None
self.extractmask = None
# SpecObjs object
self.sobjs_obj = None # Only object finding but no extraction
self.sobjs = None # Final extracted object list with trace corrections applied
def _qa(self, use_slitid=True):
"""
QA
Wrapper to trace_slits.slit_trace_qa()
Returns
-------
"""
slitmask = pixels.tslits2mask(self.tslits_dict)
trace_slits.slit_trace_qa(self.mstrace, self.slit_left,
self.slit_righ, slitmask, self.extrapord, self.master_key,
desc="Trace of the slit edges D{:02d}".format(self.det),
use_slitid=use_slitid, out_dir=self.redux_path)
def __init__(self,
tslits_dict,
tilts,
wv_calib,
spectrograph,
det,
maskslits,
master_key=None,
master_dir=None,
reuse_masters=False):
# Image
pypeitimage.PypeItImage.__init__(self, spectrograph, det)
# MasterFrame
MasterFrame.__init__(self,
self.master_type,
master_dir=master_dir,
master_key=master_key,
reuse_masters=reuse_masters)
# Required parameters
self.spectrograph = spectrograph
self.tslits_dict = tslits_dict
self.tilts = tilts
self.wv_calib = wv_calib
if tslits_dict is not None:
self.slitmask = pixels.tslits2mask(self.tslits_dict)
self.slit_spat_pos = trace_slits.slit_spat_pos(self.tslits_dict)
else:
self.slitmask = None
self.slit_spat_pos = None
# TODO: only echelle is ever used. Do we need to keep the whole
# thing?
self.par = wv_calib['par'] if wv_calib is not None else None
self.maskslits = maskslits
# For echelle order, primarily
# List to hold ouptut from inspect about what module create the image?
self.steps = []
# Main output
self.image = None
def __init__(self,
tslits_dict,
tilts,
wv_calib,
spectrograph,
det,
maskslits,
master_key=None,
master_dir=None,
reuse_masters=False):
# MasterFrame
masterframe.MasterFrame.__init__(self,
self.master_type,
master_dir=master_dir,
master_key=master_key,
reuse_masters=reuse_masters)
# Required parameters
self.spectrograph = spectrograph
self.det = det
self.tslits_dict = tslits_dict
self.tilts = tilts
self.wv_calib = wv_calib
if tslits_dict is not None:
self.slitmask = pixels.tslits2mask(self.tslits_dict)
self.slit_spat_pos = edgetrace.slit_spat_pos(self.tslits_dict)
else:
self.slitmask = None
self.slit_spat_pos = None
self.maskslits = maskslits
# For echelle order, primarily
# TODO: only echelle is ever used. Do we need to keep the whole
# thing?
self.par = wv_calib['par'] if wv_calib is not None else None
# Main output
self.image = None
self.steps = []
def __init__(self,
tslits_dict,
tilts,
wv_calib,
spectrograph,
maskslits,
master_key=None,
master_dir=None,
reuse_masters=False):
# def __init__(self, spectrograph, tslits_dict, tilts, wv_calib, maskslits,
# master_key=None, master_dir=None, reuse_masters=False):
# MasterFrame
MasterFrame.__init__(self,
self.master_type,
master_dir=master_dir,
master_key=master_key,
reuse_masters=reuse_masters)
# Required parameters
self.spectrograph = spectrograph
self.tslits_dict = tslits_dict
self.tilts = tilts
self.wv_calib = wv_calib
self.slitmask = pixels.tslits2mask(
self.tslits_dict) if tslits_dict is not None else None
# TODO: only echelle is ever used. Do we need to keep the whole
# thing?
self.par = wv_calib['par'] if wv_calib is not None else None
self.maskslits = maskslits
# List to hold ouptut from inspect about what module create the image?
self.steps = []
# Main output
self.mswave = None
def generate_mask(self):
"""Generate the mask of sky regions
Returns:
ndarray : Boolean mask containing sky regions
"""
nreg = 0
left_edg, righ_edg = np.zeros((self.nspec, 0)), np.zeros(
(self.nspec, 0))
spec_min, spec_max = np.array([]), np.array([])
for sl in range(self._nslits):
diff = self.tslits_dict['slit_righ'][:, sl] - self.tslits_dict[
'slit_left'][:, sl]
tmp = np.zeros(self._resolution + 2)
tmp[1:-1] = self._skyreg[sl]
wl = np.where(tmp[1:] > tmp[:-1])[0]
wr = np.where(tmp[1:] < tmp[:-1])[0]
for rr in range(wl.size):
left = self.tslits_dict['slit_left'][:, sl] + wl[rr] * diff / (
self._resolution - 1.0)
righ = self.tslits_dict['slit_left'][:, sl] + wr[rr] * diff / (
self._resolution - 1.0)
left_edg = np.append(left_edg, left[:, np.newaxis], axis=1)
righ_edg = np.append(righ_edg, righ[:, np.newaxis], axis=1)
nreg += 1
spec_min = np.append(spec_min,
self.tslits_dict['spec_min'][sl])
spec_max = np.append(spec_max,
self.tslits_dict['spec_max'][sl])
reg_dict = dict(pad=0,
slit_left=left_edg,
slit_righ=righ_edg,
nslits=nreg,
nspec=self.nspec,
nspat=self.nspat,
spec_min=spec_min,
spec_max=spec_max)
return (pixels.tslits2mask(reg_dict) >= 0).astype(np.int)
def __init__(self,
msarc,
tslits_dict,
spectrograph,
par,
binspectral=None,
det=1,
master_key=None,
master_dir=None,
reuse_masters=False,
qa_path=None,
msbpm=None):
# MasterFrame
masterframe.MasterFrame.__init__(self,
self.master_type,
master_dir=master_dir,
master_key=master_key,
file_format='json',
reuse_masters=reuse_masters)
# Required parameters (but can be None)
self.msarc = msarc
self.tslits_dict = tslits_dict
self.spectrograph = spectrograph
self.par = par
# Optional parameters
self.bpm = msbpm
self.binspectral = binspectral
self.qa_path = qa_path
self.det = det
self.master_key = master_key
# Attributes
self.steps = [] # steps executed
self.wv_calib = {} # main output
self.arccen = None # central arc spectrum
# --------------------------------------------------------------
# TODO: Build another base class that does these things for both
# WaveTilts and WaveCalib?
# Get the non-linear count level
self.nonlinear_counts = 1e10 if self.spectrograph is None \
else self.spectrograph.nonlinear_counts(det=self.det)
# Set the slitmask and slit boundary related attributes that the
# code needs for execution. This also deals with arcimages that
# have a different binning then the trace images used to defined
# the slits
if self.tslits_dict is not None and self.msarc is not None:
self.slitmask_science = pixels.tslits2mask(self.tslits_dict)
inmask = self.bpm == 0 if self.bpm is not None \
else np.ones_like(self.slitmask_science, dtype=bool)
self.shape_science = self.slitmask_science.shape
self.shape_arc = self.msarc.shape
self.nslits = self.tslits_dict['slit_left'].shape[1]
self.slit_left = arc.resize_slits2arc(
self.shape_arc, self.shape_science,
self.tslits_dict['slit_left'])
self.slit_righ = arc.resize_slits2arc(
self.shape_arc, self.shape_science,
self.tslits_dict['slit_righ'])
self.slitcen = arc.resize_slits2arc(self.shape_arc,
self.shape_science,
self.tslits_dict['slitcen'])
self.slitmask = arc.resize_mask2arc(self.shape_arc,
self.slitmask_science)
self.inmask = arc.resize_mask2arc(self.shape_arc, inmask)
# TODO: Remove the following two lines if deemed ok
if self.par['method'] != 'full_template':
self.inmask &= self.msarc < self.nonlinear_counts
self.slit_spat_pos = trace_slits.slit_spat_pos(self.tslits_dict)
else:
self.slitmask_science = None
self.shape_science = None
self.shape_arc = None
self.nslits = 0
self.slit_left = None
self.slit_righ = None
self.slitcen = None
self.slitmask = None
self.inmask = None
def __init__(self,
msarc,
tslits_dict,
spectrograph,
par,
wavepar,
det=1,
master_key=None,
master_dir=None,
reuse_masters=False,
qa_path=None,
msbpm=None):
self.spectrograph = spectrograph
self.par = par
self.wavepar = wavepar
# MasterFrame
masterframe.MasterFrame.__init__(self,
self.master_type,
master_dir=master_dir,
master_key=master_key,
reuse_masters=reuse_masters)
self.msarc = msarc
self.tslits_dict = tslits_dict
self.msbpm = msbpm
self.det = det
self.qa_path = qa_path
# --------------------------------------------------------------
# TODO: Build another base class that does these things for both
# WaveTilts and WaveCalib?
# Get the non-linear count level
self.nonlinear_counts = 1e10 if self.spectrograph is None \
else self.spectrograph.nonlinear_counts(det=self.det)
# Set the slitmask and slit boundary related attributes that the
# code needs for execution. This also deals with arcimages that
# have a different binning then the trace images used to defined
# the slits
if self.tslits_dict is not None and self.msarc is not None:
self.slitmask_science = pixels.tslits2mask(self.tslits_dict)
inmask = (self.msbpm == 0) if self.msbpm is not None \
else np.ones_like(self.slitmask_science, dtype=bool)
self.shape_science = self.slitmask_science.shape
self.shape_arc = self.msarc.shape
self.nslits = self.tslits_dict['slit_left'].shape[1]
self.slit_left = arc.resize_slits2arc(
self.shape_arc, self.shape_science,
self.tslits_dict['slit_left'])
self.slit_righ = arc.resize_slits2arc(
self.shape_arc, self.shape_science,
self.tslits_dict['slit_righ'])
self.slitcen = arc.resize_slits2arc(self.shape_arc,
self.shape_science,
self.tslits_dict['slitcen'])
self.slitmask = arc.resize_mask2arc(self.shape_arc,
self.slitmask_science)
self.inmask = (arc.resize_mask2arc(
self.shape_arc, inmask)) & (self.msarc < self.nonlinear_counts)
else:
self.slitmask_science = None
self.shape_science = None
self.shape_arc = None
self.nslits = 0
self.slit_left = None
self.slit_righ = None
self.slitcen = None
self.slitmask = None
self.inmask = None
# --------------------------------------------------------------
# Key Internals
self.mask = None
self.all_trace_dict = [None] * self.nslits
self.tilts = None
# 2D fits are stored as a dictionary rather than list because we will jsonify the dict
self.all_fit_dict = [None] * self.nslits
self.steps = []
# Main outputs
self.final_tilts = None
self.fit_dict = None
self.trace_dict = None
def run(self, debug=False, show=False, maskslits=None):
"""
Generate normalized pixel and illumination flats
Code flow::
1. Generate the pixelflat image (if necessary)
2. Prepare b-spline knot spacing
3. Loop on slits/orders
a. Calculate the slit profile
b. Normalize
c. Save
Args:
debug (:obj:`bool`, optional):
Run in debug mode.
show (:obj:`bool`, optional):
Show the results in the ginga viewer.
maskslits (np.ndarray, optional):
Array specifying whether a slit is good.
True = bad
Returns:
`numpy.ndarray`_: Two arrays are returned, the normalized
pixel flat data and the slit illumination correction data.
"""
# Mask
if maskslits is None:
maskslits = np.zeros(self.nslits, dtype=bool)
# Build the pixel flat (as needed)
self.build_pixflat()
# Prep tck (sets self.ntckx, self.ntcky)
#self._prep_tck()
# Setup
self.mspixelflat = np.ones_like(self.rawflatimg.image)
self.msillumflat = np.ones_like(self.rawflatimg.image)
self.flat_model = np.zeros_like(self.rawflatimg.image)
self.slitmask = pixels.tslits2mask(self.tslits_dict)
final_tilts = np.zeros_like(self.rawflatimg.image)
# If we are tweaking slits allocate the new aray to hold tweaked slit boundaries
if self.flatpar['tweak_slits']:
self.tslits_dict['slit_left_tweak'] = np.zeros_like(
self.tslits_dict['slit_left'])
self.tslits_dict['slit_righ_tweak'] = np.zeros_like(
self.tslits_dict['slit_righ'])
# Loop on slits
for slit in range(self.nslits):
# Is this a good slit??
if maskslits[slit]:
msgs.info('Skipping bad slit: {}'.format(slit))
continue
#
msgs.info('Computing flat field image for slit: {:d}/{:d}'.format(
slit, self.nslits - 1))
if self.msbpm is not None:
inmask = np.invert(self.msbpm)
else:
inmask = np.ones_like(self.rawflatimg.image, dtype=bool)
# Fit flats for a single slit
this_tilts_dict = {
'tilts': self.tilts_dict['tilts'],
'coeffs': self.tilts_dict['coeffs'][:, :, slit].copy(),
'slitcen': self.tilts_dict['slitcen'][:, slit].copy(),
'func2d': self.tilts_dict['func2d']
}
nonlinear_counts = self.spectrograph.nonlinear_counts(det=self.det)
pixelflat, illumflat, flat_model, tilts_out, thismask_out, slit_left_out, \
slit_righ_out \
= flat.fit_flat(self.rawflatimg.image, this_tilts_dict, self.tslits_dict,
slit, inmask=inmask, nonlinear_counts=nonlinear_counts,
spec_samp_fine=self.flatpar['spec_samp_fine'],
spec_samp_coarse=self.flatpar['spec_samp_coarse'],
spat_samp=self.flatpar['spat_samp'],
tweak_slits=self.flatpar['tweak_slits'],
tweak_slits_thresh=self.flatpar['tweak_slits_thresh'],
tweak_slits_maxfrac=self.flatpar['tweak_slits_maxfrac'],
debug=debug)
self.mspixelflat[thismask_out] = pixelflat[thismask_out]
self.msillumflat[thismask_out] = illumflat[thismask_out]
self.flat_model[thismask_out] = flat_model[thismask_out]
# Did we tweak slit boundaries? If so, update the tslits_dict and the tilts_dict
if self.flatpar['tweak_slits']:
self.tslits_dict['slit_left'][:, slit] = slit_left_out
self.tslits_dict['slit_righ'][:, slit] = slit_righ_out
self.tslits_dict['slit_left_tweak'][:, slit] = slit_left_out
self.tslits_dict['slit_righ_tweak'][:, slit] = slit_righ_out
final_tilts[thismask_out] = tilts_out[thismask_out]
# If we tweaked the slits update the tilts_dict
if self.flatpar['tweak_slits']:
self.tilts_dict['tilts'] = final_tilts
if show:
# Global skysub is the first step in a new extraction so clear the channels here
self.show(slits=True, wcs_match=True)
# If illumination flat fielding is turned off, set the illumflat to be None.
if not self.flatpar['illumflatten']:
msgs.warn(
'No illumination flat will be applied to your data (illumflatten=False).'
)
self.msillumflat = None
# Return
return self.mspixelflat, self.msillumflat
def run(self, debug=False, show=False):
"""
Main driver to generate normalized flat field and illumination flats
Code flow --
1. Generate the pixelflat image (if necessary)
2. Prepare b-spline knot spacing
3. Loop on slits/orders
a. Calculate the slit profile
b. Normalize
c. Save
Args:
debug (bool, optional):
show (bool, optional):
Returns:
ndarray, ndarray: self.mspixelflatnrm and self.slit_profiles
"""
# Build the pixel flat (as needed)
if self.rawflatimg is None:
self.rawflatimg = self.build_pixflat()
# Prep tck (sets self.ntckx, self.ntcky)
#self._prep_tck()
# Setup
self.mspixelflat = np.ones_like(self.rawflatimg)
self.msillumflat = np.ones_like(self.rawflatimg)
self.flat_model = np.zeros_like(self.rawflatimg)
self.slitmask = pixels.tslits2mask(self.tslits_dict)
final_tilts = np.zeros_like(self.rawflatimg)
# If we are tweaking slits allocate the new aray to hold tweaked slit boundaries
if self.flatpar['tweak_slits']:
self.tslits_dict['slit_left_tweak'] = np.zeros_like(
self.tslits_dict['slit_left'])
self.tslits_dict['slit_righ_tweak'] = np.zeros_like(
self.tslits_dict['slit_righ'])
# Loop on slits
for slit in range(self.nslits):
msgs.info('Computing flat field image for slit: {:d}/{:d}'.format(
slit, self.nslits - 1))
if self.msbpm is not None:
inmask = np.invert(self.msbpm)
else:
inmask = np.ones_like(self.rawflatimg, dtype=bool)
# Fit flats for a single slit
this_tilts_dict = {
'tilts': self.tilts_dict['tilts'],
'coeffs': self.tilts_dict['coeffs'][:, :, slit].copy(),
'slitcen': self.tilts_dict['slitcen'][:, slit].copy(),
'func2d': self.tilts_dict['func2d']
}
nonlinear_counts = self.spectrograph.detector[self.det - 1]['nonlinear']*\
self.spectrograph.detector[self.det - 1]['saturation']
pixelflat, illumflat, flat_model, tilts_out, thismask_out, slit_left_out, slit_righ_out = \
flat.fit_flat(self.rawflatimg, this_tilts_dict, self.tslits_dict, slit,
inmask=inmask,
nonlinear_counts=nonlinear_counts,
spec_samp_fine=self.flatpar['spec_samp_fine'], spec_samp_coarse=self.flatpar['spec_samp_coarse'],
spat_samp=self.flatpar['spat_samp'], tweak_slits=self.flatpar['tweak_slits'],
tweak_slits_thresh=self.flatpar['tweak_slits_thresh'],
tweak_slits_maxfrac=self.flatpar['tweak_slits_maxfrac'],debug=debug)
self.mspixelflat[thismask_out] = pixelflat[thismask_out]
self.msillumflat[thismask_out] = illumflat[thismask_out]
self.flat_model[thismask_out] = flat_model[thismask_out]
# Did we tweak slit boundaries? If so, update the tslits_dict and the tilts_dict
if self.flatpar['tweak_slits']:
self.tslits_dict['slit_left'][:, slit] = slit_left_out
self.tslits_dict['slit_righ'][:, slit] = slit_righ_out
self.tslits_dict['slit_left_tweak'][:, slit] = slit_left_out
self.tslits_dict['slit_righ_tweak'][:, slit] = slit_righ_out
final_tilts[thismask_out] = tilts_out[thismask_out]
# If we tweaked the slits update the tilts_dict
if self.flatpar['tweak_slits']:
self.tilts_dict['tilts'] = final_tilts
if show:
# Global skysub is the first step in a new extraction so clear the channels here
self.show(slits=True, wcs_match=True)
# If illumination flat fielding is turned off, set the illumflat to be None.
if not self.flatpar['illumflatten']:
msgs.warn(
'You have set illumflatten=False. No illumination flat will be applied to your data.'
)
self.msillumflat = None
# Return
return self.mspixelflat, self.msillumflat
def main(args):
# List only?
hdu = fits.open(args.file)
head0 = hdu[0].header
if args.list:
hdu.info()
return
# Setup for PYPIT imports
msgs.reset(verbosity=2)
# Init
sdet = get_dnum(args.det, prefix=False)
# One detector, sky sub for now
names = [hdu[i].name for i in range(len(hdu))]
try:
exten = names.index('DET{:s}-PROCESSED'.format(sdet))
except: # Backwards compatability
msgs.error(
'Requested detector {:s} was not processed.\n'
'Maybe you chose the wrong one to view?\n'
'Set with --det= or check file contents with --list'.format(sdet))
sciimg = hdu[exten].data
try:
exten = names.index('DET{:s}-SKY'.format(sdet))
except: # Backwards compatability
msgs.error(
'Requested detector {:s} has no sky model.\n'
'Maybe you chose the wrong one to view?\n'
'Set with --det= or check file contents with --list'.format(sdet))
skymodel = hdu[exten].data
try:
exten = names.index('DET{:s}-MASK'.format(sdet))
except ValueError: # Backwards compatability
msgs.error(
'Requested detector {:s} has no bit mask.\n'
'Maybe you chose the wrong one to view?\n'
'Set with --det= or check file contents with --list'.format(sdet))
mask = hdu[exten].data
try:
exten = names.index('DET{:s}-IVARMODEL'.format(sdet))
except ValueError: # Backwards compatability
msgs.error(
'Requested detector {:s} has no IVARMODEL.\n'
'Maybe you chose the wrong one to view?\n' +
'Set with --det= or check file contents with --list'.format(sdet))
ivarmodel = hdu[exten].data
# Read in the object model for residual map
try:
exten = names.index('DET{:s}-OBJ'.format(sdet))
except ValueError: # Backwards compatability
msgs.error(
'Requested detector {:s} has no object model.\n'
'Maybe you chose the wrong one to view?\n' +
'Set with --det= or check file contents with --list'.format(sdet))
objmodel = hdu[exten].data
# Get waveimg
mdir = head0['PYPMFDIR'] + '/'
if not os.path.exists(mdir):
mdir_base = os.path.basename(os.path.dirname(mdir)) + '/'
msgs.warn('Master file dir: {0} does not exist. Using ./{1}'.format(
mdir, mdir_base))
mdir = mdir_base
trace_key = '{0}_{1:02d}'.format(head0['TRACMKEY'], args.det)
trc_file = os.path.join(
mdir, MasterFrame.construct_file_name('Trace', trace_key))
wave_key = '{0}_{1:02d}'.format(head0['ARCMKEY'], args.det)
waveimg = os.path.join(mdir,
MasterFrame.construct_file_name('Wave', wave_key))
tslits_dict = TraceSlits.load_from_file(trc_file)[0]
slitmask = pixels.tslits2mask(tslits_dict)
shape = (tslits_dict['nspec'], tslits_dict['nspat'])
slit_ids = [
trace_slits.get_slitid(shape, tslits_dict['slit_left'],
tslits_dict['slit_righ'], ii)[0]
for ii in range(tslits_dict['slit_left'].shape[1])
]
# Show the bitmask?
mask_in = mask if args.showmask else None
# Object traces
spec1d_file = args.file.replace('spec2d', 'spec1d')
det_nm = 'DET{:s}'.format(sdet)
if os.path.isfile(spec1d_file):
hdulist_1d = fits.open(spec1d_file)
else:
hdulist_1d = []
msgs.warn('Could not find spec1d file: {:s}'.format(spec1d_file) +
msgs.newline() +
' No objects were extracted.')
# Unpack the bitmask
bitMask = bitmask()
bpm, crmask, satmask, minmask, offslitmask, nanmask, ivar0mask, ivarnanmask, extractmask \
= bitMask.unpack(mask)
# Now show each image to a separate channel
# SCIIMG
image = sciimg # Raw science image
(mean, med, sigma) = sigma_clipped_stats(image[mask == 0],
sigma_lower=5.0,
sigma_upper=5.0)
cut_min = mean - 1.0 * sigma
cut_max = mean + 4.0 * sigma
chname_skysub = 'sciimg-det{:s}'.format(sdet)
# Clear all channels at the beginning
viewer, ch = ginga.show_image(image,
chname=chname_skysub,
waveimg=waveimg,
bitmask=mask_in,
clear=True)
#, cuts=(cut_min, cut_max), wcs_match=True)
ginga.show_slits(viewer, ch, tslits_dict['slit_left'],
tslits_dict['slit_righ'], slit_ids)
#, args.det)
# SKYSUB
image = (sciimg - skymodel) * (mask == 0) # sky subtracted image
(mean, med, sigma) = sigma_clipped_stats(image[mask == 0],
sigma_lower=5.0,
sigma_upper=5.0)
cut_min = mean - 1.0 * sigma
cut_max = mean + 4.0 * sigma
chname_skysub = 'skysub-det{:s}'.format(sdet)
# Clear all channels at the beginning
# TODO: JFH For some reason Ginga crashes when I try to put cuts in here.
viewer, ch = ginga.show_image(
image, chname=chname_skysub, waveimg=waveimg,
bitmask=mask_in) #, cuts=(cut_min, cut_max),wcs_match=True)
show_trace(hdulist_1d, det_nm, viewer, ch)
ginga.show_slits(viewer, ch, tslits_dict['slit_left'],
tslits_dict['slit_righ'], slit_ids)
#, args.det)
# SKRESIDS
chname_skyresids = 'sky_resid-det{:s}'.format(sdet)
image = (sciimg - skymodel) * np.sqrt(ivarmodel) * (mask == 0
) # sky residual map
viewer, ch = ginga.show_image(image,
chname_skyresids,
waveimg=waveimg,
cuts=(-5.0, 5.0),
bitmask=mask_in) #,wcs_match=True)
show_trace(hdulist_1d, det_nm, viewer, ch)
ginga.show_slits(viewer, ch, tslits_dict['slit_left'],
tslits_dict['slit_righ'], slit_ids)
#, args.det)
# RESIDS
chname_resids = 'resid-det{:s}'.format(sdet)
# full model residual map
image = (sciimg - skymodel - objmodel) * np.sqrt(ivarmodel) * (mask == 0)
viewer, ch = ginga.show_image(image,
chname=chname_resids,
waveimg=waveimg,
cuts=(-5.0, 5.0),
bitmask=mask_in) #,wcs_match=True)
show_trace(hdulist_1d, det_nm, viewer, ch)
ginga.show_slits(viewer, ch, tslits_dict['slit_left'],
tslits_dict['slit_righ'], slit_ids)
#, args.det)
# After displaying all the images sync up the images with WCS_MATCH
shell = viewer.shell()
out = shell.start_global_plugin('WCSMatch')
out = shell.call_global_plugin_method('WCSMatch', 'set_reference_channel',
[chname_resids], {})
if args.embed:
IPython.embed()
def extract_coadd2d(stack_dict, master_dir, samp_fact = 1.0,ir_redux=False, par=None, std=False, show=False, show_peaks=False):
"""
Main routine to run the extraction for 2d coadds.
Algorithm steps are as follows:
- Fill this in.
This performs 2d coadd specific tasks, and then also performs some
of the tasks analogous to the pypeit.extract_one method. Docs coming
soon....
Args:
stack_dict:
master_dir:
samp_fact: float
sampling factor to make the wavelength grid finer or coarser. samp_fact > 1.0 oversamples (finer),
samp_fact < 1.0 undersamples (coarser)
ir_redux:
par:
show:
show_peaks:
Returns:
"""
# Find the objid of the brighest object, and the average snr across all orders
nslits = stack_dict['tslits_dict']['slit_left'].shape[1]
objid, snr_bar = get_brightest_obj(stack_dict['specobjs_list'], echelle=True)
# TODO Print out a report here on the image stack, i.e. S/N of each image
spectrograph = util.load_spectrograph(stack_dict['spectrograph'])
par = spectrograph.default_pypeit_par() if par is None else par
binning = np.array([stack_dict['tslits_dict']['binspectral'],stack_dict['tslits_dict']['binspatial']])
# Grab the wavelength grid that we will rectify onto
wave_grid = spectrograph.wavegrid(binning=binning,samp_fact=samp_fact)
wave_grid_mid = spectrograph.wavegrid(midpoint=True,binning=binning,samp_fact=samp_fact)
coadd_list = []
nspec_vec = np.zeros(nslits,dtype=int)
nspat_vec = np.zeros(nslits,dtype=int)
# TODO: Generalize this to be a loop over detectors, such tha the
# coadd_list is an ordered dict (perhaps) with all the slits on all
# detectors
for islit in range(nslits):
msgs.info('Performing 2d coadd for slit: {:d}/{:d}'.format(islit,nslits-1))
# Determine the wavelength dependent optimal weights and grab the reference trace
rms_sn, weights, trace_stack, wave_stack = optimal_weights(stack_dict['specobjs_list'],
islit, objid)
thismask_stack = stack_dict['slitmask_stack'] == islit
# Perform the 2d coadd
coadd_dict = coadd2d(trace_stack, stack_dict['sciimg_stack'], stack_dict['sciivar_stack'],
stack_dict['skymodel_stack'], stack_dict['mask_stack'] == 0,
stack_dict['tilts_stack'], stack_dict['waveimg_stack'],
thismask_stack, weights=weights, wave_grid=wave_grid)
coadd_list.append(coadd_dict)
nspec_vec[islit]=coadd_dict['nspec']
nspat_vec[islit]=coadd_dict['nspat']
# Determine the size of the psuedo image
nspat_pad = 10
nspec_psuedo = nspec_vec.max()
nspat_psuedo = np.sum(nspat_vec) + (nslits + 1)*nspat_pad
spec_vec_psuedo = np.arange(nspec_psuedo)
shape_psuedo = (nspec_psuedo, nspat_psuedo)
imgminsky_psuedo = np.zeros(shape_psuedo)
sciivar_psuedo = np.zeros(shape_psuedo)
waveimg_psuedo = np.zeros(shape_psuedo)
tilts_psuedo = np.zeros(shape_psuedo)
spat_psuedo = np.zeros(shape_psuedo)
nused_psuedo = np.zeros(shape_psuedo, dtype=int)
inmask_psuedo = np.zeros(shape_psuedo, dtype=bool)
wave_mid = np.zeros((nspec_psuedo, nslits))
wave_mask = np.zeros((nspec_psuedo, nslits),dtype=bool)
wave_min = np.zeros((nspec_psuedo, nslits))
wave_max = np.zeros((nspec_psuedo, nslits))
dspat_mid = np.zeros((nspat_psuedo, nslits))
spat_left = nspat_pad
slit_left = np.zeros((nspec_psuedo, nslits))
slit_righ = np.zeros((nspec_psuedo, nslits))
spec_min1 = np.zeros(nslits)
spec_max1 = np.zeros(nslits)
for islit, coadd_dict in enumerate(coadd_list):
spat_righ = spat_left + nspat_vec[islit]
ispec = slice(0,nspec_vec[islit])
ispat = slice(spat_left,spat_righ)
imgminsky_psuedo[ispec, ispat] = coadd_dict['imgminsky']
sciivar_psuedo[ispec, ispat] = coadd_dict['sciivar']
waveimg_psuedo[ispec, ispat] = coadd_dict['waveimg']
tilts_psuedo[ispec, ispat] = coadd_dict['tilts']
# spat_psuedo is the sub-pixel image position on the rebinned psuedo image
inmask_psuedo[ispec, ispat] = coadd_dict['outmask']
image_temp = (coadd_dict['dspat'] - coadd_dict['dspat_mid'][0] + spat_left)*coadd_dict['outmask']
spat_psuedo[ispec, ispat] = image_temp
nused_psuedo[ispec, ispat] = coadd_dict['nused']
wave_min[ispec, islit] = coadd_dict['wave_min']
wave_max[ispec, islit] = coadd_dict['wave_max']
wave_mid[ispec, islit] = coadd_dict['wave_mid']
wave_mask[ispec, islit] = True
# Fill in the rest of the wave_mid with the corresponding points in the wave_grid
wave_this = wave_mid[wave_mask[:,islit], islit]
ind_upper = np.argmin(np.abs(wave_grid_mid - np.max(wave_this.max()))) + 1
if nspec_vec[islit] != nspec_psuedo:
wave_mid[nspec_vec[islit]:, islit] = wave_grid_mid[ind_upper:ind_upper + (nspec_psuedo-nspec_vec[islit])]
dspat_mid[ispat, islit] = coadd_dict['dspat_mid']
slit_left[:,islit] = np.full(nspec_psuedo, spat_left)
slit_righ[:,islit] = np.full(nspec_psuedo, spat_righ)
spec_max1[islit] = nspec_vec[islit]-1
spat_left = spat_righ + nspat_pad
slitcen = (slit_left + slit_righ)/2.0
tslits_dict_psuedo = dict(slit_left=slit_left, slit_righ=slit_righ, slitcen=slitcen,
nspec=nspec_psuedo, nspat=nspat_psuedo, pad=0,
nslits = nslits, binspectral=1, binspatial=1, spectrograph=spectrograph.spectrograph,
spec_min=spec_min1, spec_max=spec_max1)
slitmask_psuedo = pixels.tslits2mask(tslits_dict_psuedo)
# This is a kludge to deal with cases where bad wavelengths result in large regions where the slit is poorly sampled,
# which wreaks havoc on the local sky-subtraction
min_slit_frac = 0.70
spec_min = np.zeros(nslits)
spec_max = np.zeros(nslits)
for islit in range(nslits):
slit_width = np.sum(inmask_psuedo*(slitmask_psuedo == islit),axis=1)
slit_width_img = np.outer(slit_width, np.ones(nspat_psuedo))
med_slit_width = np.median(slit_width_img[slitmask_psuedo==islit])
nspec_eff = np.sum(slit_width > min_slit_frac*med_slit_width)
nsmooth = int(np.fmax(np.ceil(nspec_eff*0.02),10))
slit_width_sm = scipy.ndimage.filters.median_filter(slit_width, size=nsmooth, mode='reflect')
igood = (slit_width_sm > min_slit_frac*med_slit_width)
spec_min[islit] = spec_vec_psuedo[igood].min()
spec_max[islit] = spec_vec_psuedo[igood].max()
bad_pix = (slit_width_img < min_slit_frac*med_slit_width) & (slitmask_psuedo == islit)
inmask_psuedo[bad_pix] = False
# Update with tslits_dict_psuedo
tslits_dict_psuedo['spec_min'] = spec_min
tslits_dict_psuedo['spec_max'] = spec_max
slitmask_psuedo = pixels.tslits2mask(tslits_dict_psuedo)
# Make a fake bitmask from the outmask. We are kludging the crmask to be the outmask_psuedo here, and setting the bpm to
# be good everywhere
mask = processimages.ProcessImages.build_mask(imgminsky_psuedo, sciivar_psuedo, np.invert(inmask_psuedo),
np.zeros_like(inmask_psuedo), slitmask=slitmask_psuedo)
redux = reduce.instantiate_me(spectrograph, tslits_dict_psuedo, mask, ir_redux=ir_redux, par=par,
objtype = 'science', binning=binning)
if show:
redux.show('image', image=imgminsky_psuedo*(mask == 0), chname = 'imgminsky', slits=True, clear=True)
# Object finding
sobjs_obj, nobj, skymask_init = redux.find_objects(imgminsky_psuedo, sciivar_psuedo, ir_redux=ir_redux, std=std,
show_peaks=show_peaks, show=show)
# Local sky-subtraction
global_sky_psuedo = np.zeros_like(imgminsky_psuedo) # No global sky for co-adds since we go straight to local
rn2img_psuedo = global_sky_psuedo # No rn2img for co-adds since we go do not model noise
skymodel_psuedo, objmodel_psuedo, ivarmodel_psuedo, outmask_psuedo, sobjs = \
redux.local_skysub_extract(imgminsky_psuedo, sciivar_psuedo, tilts_psuedo, waveimg_psuedo, global_sky_psuedo,
rn2img_psuedo, sobjs_obj, spat_pix=spat_psuedo, std=std,
model_noise=False, show_profile=show, show=show)
if ir_redux:
sobjs.purge_neg()
# Add the information about the fixed wavelength grid to the sobjs
for spec in sobjs:
spec.boxcar['WAVE_GRID_MASK'] = wave_mask[:,spec.slitid]
spec.boxcar['WAVE_GRID'] = wave_mid[:,spec.slitid]
spec.boxcar['WAVE_GRID_MIN'] = wave_min[:,spec.slitid]
spec.boxcar['WAVE_GRID_MAX'] = wave_max[:,spec.slitid]
spec.optimal['WAVE_GRID_MASK'] = wave_mask[:,spec.slitid]
spec.optimal['WAVE_GRID'] = wave_mid[:,spec.slitid]
spec.optimal['WAVE_GRID_MIN'] = wave_min[:,spec.slitid]
spec.optimal['WAVE_GRID_MAX'] = wave_max[:,spec.slitid]
# TODO Implement flexure and heliocentric corrections on the single exposure 1d reductions and apply them to the
# waveimage. Change the data model to accomodate a wavelength model for each image.
# Using the same implementation as in core/pypeit
# Write out the psuedo master files to disk
master_key_dict = stack_dict['master_key_dict']
# TODO: These saving operations are a temporary kludge
waveImage = WaveImage(None, None, None, None, None, master_key=master_key_dict['arc'],
master_dir=master_dir)
waveImage.save(mswave=waveimg_psuedo)
traceSlits = TraceSlits(None, None, master_key=master_key_dict['trace'], master_dir=master_dir)
traceSlits.save(tslits_dict=tslits_dict_psuedo)
return imgminsky_psuedo, sciivar_psuedo, skymodel_psuedo, objmodel_psuedo, ivarmodel_psuedo, outmask_psuedo, sobjs
def load_coadd2d_stacks(spec2d_files, det):
"""
Args:
spec2d_files: list
List of spec2d filenames
det: int
detector in question
Returns:
stack_dict: dict
Dictionary containing all the images and keys required for perfomring 2d coadds.
"""
# Get the detector string
sdet = parse.get_dnum(det, prefix=False)
# Get the master dir
head0 = fits.getheader(spec2d_files[0])
master_dir = os.path.basename(head0['PYPMFDIR'])
redux_path = os.getcwd()
master_path = os.path.join(redux_path, master_dir)
# Grab the files
head2d_list=[]
tracefiles = []
waveimgfiles = []
tiltfiles = []
spec1d_files = []
for f in spec2d_files:
head = fits.getheader(f)
trace_key = '{0}_{1:02d}'.format(head['TRACMKEY'], det)
wave_key = '{0}_{1:02d}'.format(head['ARCMKEY'], det)
head2d_list.append(head)
spec1d_files.append(f.replace('spec2d', 'spec1d'))
tracefiles.append(os.path.join(master_path,
MasterFrame.construct_file_name('Trace', trace_key)))
waveimgfiles.append(os.path.join(master_path,
MasterFrame.construct_file_name('Wave', wave_key)))
tiltfiles.append(os.path.join(master_path,
MasterFrame.construct_file_name('Tilts', wave_key)))
nfiles = len(spec2d_files)
specobjs_list = []
head1d_list=[]
# TODO Sort this out with the correct detector extensions etc.
# Read in the image stacks
for ifile in range(nfiles):
waveimg = WaveImage.load_from_file(waveimgfiles[ifile])
tilts = WaveTilts.load_from_file(tiltfiles[ifile])
hdu = fits.open(spec2d_files[ifile])
# One detector, sky sub for now
names = [hdu[i].name for i in range(len(hdu))]
# science image
try:
exten = names.index('DET{:s}-PROCESSED'.format(sdet))
except: # Backwards compatability
det_error_msg(exten, sdet)
sciimg = hdu[exten].data
# skymodel
try:
exten = names.index('DET{:s}-SKY'.format(sdet))
except: # Backwards compatability
det_error_msg(exten, sdet)
skymodel = hdu[exten].data
# Inverse variance model
try:
exten = names.index('DET{:s}-IVARMODEL'.format(sdet))
except ValueError: # Backwards compatability
det_error_msg(exten, sdet)
sciivar = hdu[exten].data
# Mask
try:
exten = names.index('DET{:s}-MASK'.format(sdet))
except ValueError: # Backwards compatability
det_error_msg(exten, sdet)
mask = hdu[exten].data
if ifile == 0:
# the two shapes accomodate the possibility that waveimg and tilts are binned differently
shape_wave = (nfiles,waveimg.shape[0],waveimg.shape[1])
shape_sci = (nfiles,sciimg.shape[0],sciimg.shape[1])
waveimg_stack = np.zeros(shape_wave,dtype=float)
tilts_stack = np.zeros(shape_wave,dtype=float)
sciimg_stack = np.zeros(shape_sci,dtype=float)
skymodel_stack = np.zeros(shape_sci,dtype=float)
sciivar_stack = np.zeros(shape_sci,dtype=float)
mask_stack = np.zeros(shape_sci,dtype=float)
waveimg_stack[ifile,:,:] = waveimg
tilts_stack[ifile,:,:] = tilts['tilts']
sciimg_stack[ifile,:,:] = sciimg
sciivar_stack[ifile,:,:] = sciivar
mask_stack[ifile,:,:] = mask
skymodel_stack[ifile,:,:] = skymodel
sobjs, head = load.load_specobjs(spec1d_files[ifile])
head1d_list.append(head)
specobjs_list.append(sobjs)
# Right now we assume there is a single tslits_dict for all images and read in the first one
# TODO this needs to become a tslits_dict for each file to accomodate slits defined by flats taken on different
# nights
tslits_dict, _ = TraceSlits.load_from_file(tracefiles[0])
spectrograph = util.load_spectrograph(tslits_dict['spectrograph'])
slitmask = pixels.tslits2mask(tslits_dict)
slitmask_stack = np.einsum('i,jk->ijk', np.ones(nfiles), slitmask)
# Fill the master key dict
head2d = head2d_list[0]
master_key_dict = {}
master_key_dict['frame'] = head2d['FRAMMKEY'] + '_{:02d}'.format(det)
master_key_dict['bpm'] = head2d['BPMMKEY'] + '_{:02d}'.format(det)
master_key_dict['bias'] = head2d['BIASMKEY'] + '_{:02d}'.format(det)
master_key_dict['arc'] = head2d['ARCMKEY'] + '_{:02d}'.format(det)
master_key_dict['trace'] = head2d['TRACMKEY'] + '_{:02d}'.format(det)
master_key_dict['flat'] = head2d['FLATMKEY'] + '_{:02d}'.format(det)
stack_dict = dict(specobjs_list=specobjs_list, tslits_dict=tslits_dict,
slitmask_stack=slitmask_stack,
sciimg_stack=sciimg_stack, sciivar_stack=sciivar_stack,
skymodel_stack=skymodel_stack, mask_stack=mask_stack,
tilts_stack=tilts_stack, waveimg_stack=waveimg_stack,
head1d_list = head1d_list, head2d_list=head2d_list,
redux_path=redux_path, master_path=master_path, master_dir=master_dir,
master_key_dict=master_key_dict,
spectrograph = tslits_dict['spectrograph'])
return stack_dict
def __init__(self,
msalign,
tslits_dict,
spectrograph,
par,
det=1,
binning=None,
qa_path=None,
msbpm=None):
# MasterFrame
#masterframe.MasterFrame.__init__(self, self.master_type, master_dir=master_dir,
# master_key=master_key, file_format='fits',
# reuse_masters=reuse_masters)
# Required parameters (but can be None)
self.msalign = msalign
self.tslits_dict = tslits_dict
self.spectrograph = spectrograph
self.par = par
self.binning = binning
# Optional parameters
self.bpm = msalign.mask if msbpm is None else msbpm
self.qa_path = qa_path
self.det = det
#self.master_key = master_key
# Attributes
self.viewer, self.channel = None, None
self.steps = [] # steps executed
# Get the non-linear count level
self.nonlinear_counts = 1e10 if self.spectrograph is None \
else self.spectrograph.nonlinear_counts(self.det)
# --------------------------------------------------------------
# Set the slitmask and slit boundary related attributes that the
# code needs for execution. This also deals with alignframes that
# have a different binning then the images used to defined
# the slits
if self.tslits_dict is not None and self.msalign is not None:
self.slitmask_science = pixels.tslits2mask(self.tslits_dict)
gpm = self.bpm == 0 if self.bpm is not None \
else np.ones_like(self.slitmask_science, dtype=bool)
self.shape_science = self.slitmask_science.shape
self.shape_align = self.msalign.image.shape
self.nslits = self.tslits_dict['slit_left'].shape[1]
self.slit_left = arc.resize_slits2arc(
self.shape_align, self.shape_science,
self.tslits_dict['slit_left'])
self.slit_righ = arc.resize_slits2arc(
self.shape_align, self.shape_science,
self.tslits_dict['slit_righ'])
self.slitcen = arc.resize_slits2arc(self.shape_align,
self.shape_science,
self.tslits_dict['slitcen'])
self.slitmask = arc.resize_mask2arc(self.shape_align,
self.slitmask_science)
self.gpm = arc.resize_mask2arc(self.shape_align, gpm)
self.gpm &= self.msalign.image < self.nonlinear_counts
self.slit_spat_pos = edgetrace.slit_spat_pos(
self.tslits_dict['slit_left'], self.tslits_dict['slit_righ'],
self.tslits_dict['nspec'], self.tslits_dict['nspat'])
else:
self.slitmask_science = None
self.shape_science = None
self.shape_align = None
self.nslits = 0
self.slit_left = None
self.slit_righ = None
self.slitcen = None
self.slitmask = None
self.gpm = None
self.nonlinear_counts = None
def extract_one(self, frames, det, bg_frames, std_outfile=None):
"""
Extract a single exposure/detector pair
sci_ID and det need to have been set internally prior to calling this method
Args:
frames (list):
List of frames to extract; stacked if more than one is provided
det (int):
bg_frames (list):
List of frames to use as the background
Can be empty
std_outfile (str, optional):
Returns:
seven objects are returned::
- ndarray: Science image
- ndarray: Science inverse variance image
- ndarray: Model of the sky
- ndarray: Model of the object
- ndarray: Model of inverse variance
- ndarray: Mask
- :obj:`pypeit.specobjs.SpecObjs`: spectra
"""
# Grab some meta-data needed for the reduction from the fitstbl
self.objtype, self.setup, self.obstime, self.basename, self.binning = self.get_sci_metadata(
frames[0], det)
# Is this a standard star?
self.std_redux = 'standard' in self.objtype
# Get the standard trace if need be
std_trace = self.get_std_trace(self.std_redux, det, std_outfile)
# Build Science image
sci_files = self.fitstbl.frame_paths(frames)
self.sciImg = scienceimage.build_from_file_list(
self.spectrograph,
det,
self.par['scienceframe']['process'],
self.caliBrate.msbpm,
sci_files,
self.caliBrate.msbias,
self.caliBrate.mspixelflat,
illum_flat=self.caliBrate.msillumflat)
# Background Image?
if len(bg_frames) > 0:
bg_file_list = self.fitstbl.frame_paths(bg_frames)
self.sciImg = self.sciImg - scienceimage.build_from_file_list(
self.spectrograph,
det,
self.par['scienceframe']['process'],
self.caliBrate.msbpm,
bg_file_list,
self.caliBrate.msbias,
self.caliBrate.mspixelflat,
illum_flat=self.caliBrate.msillumflat)
# Update mask for slitmask
slitmask = pixels.tslits2mask(self.caliBrate.tslits_dict)
self.sciImg.update_mask_slitmask(slitmask)
# For QA on crash
msgs.sciexp = self.sciImg
# Instantiate Reduce object
self.maskslits = self.caliBrate.tslits_dict['maskslits'].copy()
# Required for pypeline specific object
# TODO -- caliBrate should be replaced by the ~3 primary Objects needed
# once we have the data models in place.
self.redux = reduce.instantiate_me(self.sciImg,
self.spectrograph,
self.par,
self.caliBrate,
maskslits=self.maskslits,
ir_redux=self.ir_redux,
std_redux=self.std_redux,
objtype=self.objtype,
setup=self.setup,
show=self.show,
det=det,
binning=self.binning)
# Show?
if self.show:
self.redux.show('image',
image=self.sciImg.image,
chname='processed',
slits=True,
clear=True)
# Prep for manual extraction (if requested)
manual_extract_dict = self.fitstbl.get_manual_extract(frames, det)
self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs = self.redux.run(
std_trace=std_trace,
manual_extract_dict=manual_extract_dict,
show_peaks=self.show,
basename=self.basename,
ra=self.fitstbl["ra"][frames[0]],
dec=self.fitstbl["dec"][frames[0]],
obstime=self.obstime)
# Return
return self.sciImg.image, self.sciImg.ivar, self.skymodel, self.objmodel, self.ivarmodel, self.outmask, self.sobjs
