def test_step_by_step(master_dir):
# Masters
spectrograph = load_spectrograph('shane_kast_blue')
msarc, tslits_dict, mstrace = load_kast_blue_masters(aimg=True, tslits=True)
# Instantiate
master_key = 'A_1_01'
parset = spectrograph.default_pypeit_par()
par = parset['calibrations']['tilts']
wavepar = parset['calibrations']['wavelengths']
waveTilts = wavetilts.WaveTilts(msarc, tslits_dict, spectrograph, par, wavepar,
det=1, master_key=master_key,
master_dir=master_dir,reuse_masters=True)
# Extract arcs
arccen, maskslits = waveTilts.extract_arcs(waveTilts.slitcen, waveTilts.slitmask, msarc, waveTilts.inmask)
assert arccen.shape == (2048,1)
# Tilts in the slit
slit = 0
waveTilts.slitmask = pixels.tslits2mask(waveTilts.tslits_dict)
thismask = waveTilts.slitmask == slit
waveTilts.lines_spec, waveTilts.lines_spat = waveTilts.find_lines(arccen[:, slit], waveTilts.slitcen[:, slit], slit)
trcdict = waveTilts.trace_tilts(waveTilts.msarc, waveTilts.lines_spec, waveTilts.lines_spat, thismask, slit)
assert isinstance(trcdict, dict)
# 2D Fit
spat_order = waveTilts._parse_param(waveTilts.par, 'spat_order', slit)
spec_order = waveTilts._parse_param(waveTilts.par, 'spec_order', slit)
coeffs = waveTilts.fit_tilts(trcdict, thismask, waveTilts.slitcen[:, slit], spat_order, spec_order,slit, doqa=False)
tilts = tracewave.fit2tilts(waveTilts.slitmask_science.shape, coeffs, waveTilts.par['func2d'])
assert np.max(tilts) < 1.01
def fit2tiltimg(self, slitmask, flexure=None):
"""
Generate a tilt image from the fit parameters
Mainly to allow for flexure
Args:
slitmask (`numpy.ndarray`_):
flexure (float, optional):
Spatial shift of the tilt image onto the desired frame
(typically a science image)
Returns:
`numpy.ndarray`_: New tilt image
"""
_flexure = 0. if flexure is None else flexure
final_tilts = np.zeros_like(slitmask).astype(float)
gdslit_spat = np.unique(slitmask[slitmask >= 0]).astype(int)
# Loop
for slit_spat in gdslit_spat:
slit_idx = self.spatid_to_zero(slit_spat)
# Calculate
coeff_out = self.coeffs[:self.spec_order[slit_idx]+1,:self.spat_order[slit_idx]+1,slit_idx]
_tilts = tracewave.fit2tilts(final_tilts.shape, coeff_out, self.func2d, spat_shift=-1*_flexure)
# Fill
thismask_science = slitmask == slit_spat
final_tilts[thismask_science] = _tilts[thismask_science]
# Return
return final_tilts
def run(self, maskslits=None, doqa=True, debug=False, show=False):
"""
Main driver for tracing arc lines
Code flow::
1. Extract an arc spectrum down the center of each slit/order
2. Loop on slits/orders
i. Trace and fit the arc lines (This is done twice, once
with trace_crude as the tracing crutch, then again
with a PCA model fit as the crutch).
ii. Repeat trace.
iii. 2D Fit to the offset from slitcen
iv. Save
Keyword Args:
maskslits (`numpy.ndarray`_, optional):
Boolean array to ignore slits.
doqa (bool):
debug (bool):
show (bool):
Returns:
dict, ndarray: Tilts dict and maskslits array
"""
if maskslits is None:
maskslits = np.zeros(self.nslits, dtype=bool)
# Extract the arc spectra for all slits
self.arccen, self.arc_maskslit = self.extract_arcs(
self.slitcen, self.slitmask, self.msarc, self.inmask)
# maskslit
self.mask = maskslits & (self.arc_maskslit == 1)
gdslits = np.where(np.invert(self.mask))[0]
# Final tilts image
self.final_tilts = np.zeros(self.shape_science, dtype=float)
max_spat_dim = (np.asarray(self.par['spat_order']) + 1).max()
max_spec_dim = (np.asarray(self.par['spec_order']) + 1).max()
self.coeffs = np.zeros((max_spec_dim, max_spat_dim, self.nslits))
self.spat_order = np.zeros(self.nslits, dtype=int)
self.spec_order = np.zeros(self.nslits, dtype=int)
# TODO sort out show methods for debugging
#if show:
# viewer,ch = ginga.show_image(self.msarc*(self.slitmask > -1),chname='tilts')
# Loop on all slits
for slit in gdslits:
msgs.info('Computing tilts for slit {:d}/{:d}'.format(
slit, self.nslits - 1))
# Identify lines for tracing tilts
msgs.info('Finding lines for tilt analysis')
self.lines_spec, self.lines_spat = self.find_lines(
self.arccen[:, slit], self.slitcen[:, slit], slit, debug=debug)
if self.lines_spec is None:
self.mask[slit] = True
maskslits[slit] = True
continue
thismask = self.slitmask == slit
# Trace
msgs.info('Trace the tilts')
self.trace_dict = self.trace_tilts(self.msarc, self.lines_spec,
self.lines_spat, thismask,
self.slitcen[:, slit])
#if show:
# ginga.show_tilts(viewer, ch, self.trace_dict)
self.spat_order[slit] = self._parse_param(self.par, 'spat_order',
slit)
self.spec_order[slit] = self._parse_param(self.par, 'spec_order',
slit)
# 2D model of the tilts, includes construction of QA
coeff_out = self.fit_tilts(self.trace_dict,
thismask,
self.slitcen[:, slit],
self.spat_order[slit],
self.spec_order[slit],
slit,
doqa=doqa,
show_QA=show,
debug=show)
self.coeffs[0:self.spec_order[slit] + 1,
0:self.spat_order[slit] + 1, slit] = coeff_out
# Tilts are created with the size of the original slitmask,
# which corresonds to the same binning as the science
# images, trace images, and pixelflats etc.
self.tilts = tracewave.fit2tilts(self.slitmask_science.shape,
coeff_out, self.par['func2d'])
# Save to final image
thismask_science = self.slitmask_science == slit
self.final_tilts[thismask_science] = self.tilts[thismask_science]
self.tilts_dict = {
'tilts': self.final_tilts,
'coeffs': self.coeffs,
'slitcen': self.slitcen,
'func2d': self.par['func2d'],
'nslit': self.nslits,
'spat_order': self.spat_order,
'spec_order': self.spec_order
}
return self.tilts_dict, maskslits
def run(self, maskslits=None, doqa=True, debug=False, show=False):
"""
Main driver for tracing arc lines
Code flow:
1. Extract an arc spectrum down the center of each slit/order
2. Loop on slits/orders
i. Trace and fit the arc lines (This is done twice, once
with trace_crude as the tracing crutch, then again
with a PCA model fit as the crutch).
ii. Repeat trace.
iii. 2D Fit to the offset from slitcen
iv. Save
Args:
maskslits (`numpy.ndarray`_, optional):
Boolean array to ignore slits.
doqa (bool):
debug (bool):
show (bool):
Returns:
dict, ndarray: Tilts dict and maskslits array
"""
if maskslits is None:
maskslits = np.zeros(self.nslits, dtype=bool)
# Extract the arc spectra for all slits
self.arccen, self.arccen_bpm, self.arc_maskslit = self.extract_arcs()
# TODO: Leave for now. Used for debugging
# self.par['rm_continuum'] = True
# debug = True
# show = True
# Subtract arc continuum
_msarc = self.msarc.image.copy()
if self.par['rm_continuum']:
continuum = self.model_arc_continuum(debug=debug)
_msarc -= continuum
if debug:
# TODO: Put this into a function
vmin, vmax = visualization.ZScaleInterval().get_limits(_msarc)
w, h = plt.figaspect(1)
fig = plt.figure(figsize=(3 * w, h))
ax = fig.add_axes([0.15 / 3, 0.1, 0.8 / 3, 0.8])
ax.imshow(self.msarc.image,
origin='lower',
interpolation='nearest',
aspect='auto',
vmin=vmin,
vmax=vmax)
ax.set_title('MasterArc')
ax = fig.add_axes([1.15 / 3, 0.1, 0.8 / 3, 0.8])
ax.imshow(continuum,
origin='lower',
interpolation='nearest',
aspect='auto',
vmin=vmin,
vmax=vmax)
ax.set_title('Continuum')
ax = fig.add_axes([2.15 / 3, 0.1, 0.8 / 3, 0.8])
ax.imshow(_msarc,
origin='lower',
interpolation='nearest',
aspect='auto',
vmin=vmin,
vmax=vmax)
ax.set_title('MasterArc - Continuum')
plt.show()
# maskslit
self.mask = np.any([maskslits, self.arc_maskslit == 1], axis=0)
gdslits = np.where(np.invert(self.mask))[0]
# Final tilts image
self.final_tilts = np.zeros(self.shape_science, dtype=float)
max_spat_dim = (np.asarray(self.par['spat_order']) + 1).max()
max_spec_dim = (np.asarray(self.par['spec_order']) + 1).max()
self.coeffs = np.zeros((max_spec_dim, max_spat_dim, self.nslits))
self.spat_order = np.zeros(self.nslits, dtype=int)
self.spec_order = np.zeros(self.nslits, dtype=int)
# TODO sort out show methods for debugging
#if show:
# viewer,ch = ginga.show_image(self.msarc*(self.slitmask > -1),chname='tilts')
# Loop on all slits
for slit in gdslits:
msgs.info('Computing tilts for slit {0}/{1}'.format(
slit, self.nslits - 1))
# Identify lines for tracing tilts
msgs.info('Finding lines for tilt analysis')
self.lines_spec, self.lines_spat \
= self.find_lines(self.arccen[:,slit], self.slitcen[:,slit], slit,
bpm=self.arccen_bpm[:,slit], debug=False) #debug)
if self.lines_spec is None:
self.mask[slit] = True
maskslits[slit] = True
continue
thismask = self.slitmask == slit
# Performs the initial tracing of the line centroids as a
# function of spatial position resulting in 1D traces for
# each line.
msgs.info('Trace the tilts')
self.trace_dict = self.trace_tilts(_msarc, self.lines_spec,
self.lines_spat, thismask,
self.slitcen[:, slit])
# TODO: Show the traces before running the 2D fit
#if show:
# ginga.show_tilts(viewer, ch, self.trace_dict)
self.spat_order[slit] = self._parse_param(self.par, 'spat_order',
slit)
self.spec_order[slit] = self._parse_param(self.par, 'spec_order',
slit)
# 2D model of the tilts, includes construction of QA
# NOTE: This also fills in self.all_fit_dict and self.all_trace_dict
coeff_out = self.fit_tilts(self.trace_dict,
thismask,
self.slitcen[:, slit],
self.spat_order[slit],
self.spec_order[slit],
slit,
doqa=doqa,
show_QA=show,
debug=show)
self.coeffs[:self.spec_order[slit] + 1, :self.spat_order[slit] + 1,
slit] = coeff_out
# Tilts are created with the size of the original slitmask,
# which corresonds to the same binning as the science
# images, trace images, and pixelflats etc.
self.tilts = tracewave.fit2tilts(self.slitmask_science.shape,
coeff_out, self.par['func2d'])
# Save to final image
thismask_science = self.slitmask_science == slit
self.final_tilts[thismask_science] = self.tilts[thismask_science]
if debug:
# TODO: Add this to the show method?
vmin, vmax = visualization.ZScaleInterval().get_limits(_msarc)
plt.imshow(_msarc,
origin='lower',
interpolation='nearest',
aspect='auto',
vmin=vmin,
vmax=vmax)
for slit in gdslits:
spat = self.all_trace_dict[slit]['tilts_spat']
spec = self.all_trace_dict[slit]['tilts']
spec_fit = self.all_trace_dict[slit]['tilts_fit']
in_fit = self.all_trace_dict[slit]['tot_mask']
not_fit = np.invert(in_fit) & (spec > 0)
fit_rej = in_fit & np.invert(
self.all_trace_dict[slit]['fit_mask'])
fit_keep = in_fit & self.all_trace_dict[slit]['fit_mask']
plt.scatter(spat[not_fit],
spec[not_fit],
color='C1',
marker='.',
s=30,
lw=0)
plt.scatter(spat[fit_rej],
spec[fit_rej],
color='C3',
marker='.',
s=30,
lw=0)
plt.scatter(spat[fit_keep],
spec[fit_keep],
color='k',
marker='.',
s=30,
lw=0)
with_fit = np.invert(np.all(np.invert(fit_keep), axis=0))
for t in range(in_fit.shape[1]):
if not with_fit[t]:
continue
l, r = np.nonzero(in_fit[:, t])[0][[0, -1]]
plt.plot(spat[l:r + 1, t], spec_fit[l:r + 1, t], color='k')
plt.show()
self.tilts_dict = {
'tilts': self.final_tilts,
'coeffs': self.coeffs,
'slitcen': self.slitcen,
'func2d': self.par['func2d'],
'nslit': self.nslits,
'spat_order': self.spat_order,
'spec_order': self.spec_order
}
return self.tilts_dict, maskslits
def run(self, doqa=True, debug=False, show=False):
"""
Main driver for tracing arc lines
Code flow:
#. Extract an arc spectrum down the center of each slit/order
#. Loop on slits/orders
#. Trace and fit the arc lines (This is done twice, once
with trace_crude as the tracing crutch, then again
with a PCA model fit as the crutch).
#. Repeat trace.
#. 2D Fit to the offset from slitcen
#. Save
Args:
doqa (bool):
debug (bool):
show (bool):
Returns:
:class:`WaveTilts`:
"""
# Extract the arc spectra for all slits
self.arccen, self.arccen_bpm = self.extract_arcs()
# TODO: Leave for now. Used for debugging
# self.par['rm_continuum'] = True
# debug = True
# show = True
# Subtract arc continuum
_mstilt = self.mstilt.image.copy()
if self.par['rm_continuum']:
continuum = self.model_arc_continuum(debug=debug)
_mstilt -= continuum
if debug:
# TODO: Put this into a function
vmin, vmax = visualization.ZScaleInterval().get_limits(_mstilt)
w,h = plt.figaspect(1)
fig = plt.figure(figsize=(3*w,h))
ax = fig.add_axes([0.15/3, 0.1, 0.8/3, 0.8])
ax.imshow(self.mstilt.image, origin='lower', interpolation='nearest',
aspect='auto', vmin=vmin, vmax=vmax)
ax.set_title('MasterArc')
ax = fig.add_axes([1.15/3, 0.1, 0.8/3, 0.8])
ax.imshow(continuum, origin='lower', interpolation='nearest',
aspect='auto', vmin=vmin, vmax=vmax)
ax.set_title('Continuum')
ax = fig.add_axes([2.15/3, 0.1, 0.8/3, 0.8])
ax.imshow(_mstilt, origin='lower', interpolation='nearest',
aspect='auto', vmin=vmin, vmax=vmax)
ax.set_title('MasterArc - Continuum')
plt.show()
# Final tilts image
self.final_tilts = np.zeros(self.shape_science,dtype=float)
max_spat_dim = (np.asarray(self.par['spat_order']) + 1).max()
max_spec_dim = (np.asarray(self.par['spec_order']) + 1).max()
self.coeffs = np.zeros((max_spec_dim, max_spat_dim,self.slits.nslits))
self.spat_order = np.zeros(self.slits.nslits, dtype=int)
self.spec_order = np.zeros(self.slits.nslits, dtype=int)
# TODO sort out show methods for debugging
if show:
viewer,ch = ginga.show_image(self.mstilt.image*(self.slitmask > -1),chname='tilts')
# Loop on all slits
for slit_idx, slit_spat in enumerate(self.slits.spat_id):
if self.tilt_bpm[slit_idx]:
continue
#msgs.info('Computing tilts for slit {0}/{1}'.format(slit, self.slits.nslits-1))
msgs.info('Computing tilts for slit {0}/{1}'.format(slit_idx, self.slits.nslits))
# Identify lines for tracing tilts
msgs.info('Finding lines for tilt analysis')
self.lines_spec, self.lines_spat \
= self.find_lines(self.arccen[:,slit_idx], self.slitcen[:,slit_idx],
slit_idx,
bpm=self.arccen_bpm[:,slit_idx], debug=debug)
if self.lines_spec is None:
self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADTILTCALIB')
continue
thismask = self.slitmask == slit_spat
# Performs the initial tracing of the line centroids as a
# function of spatial position resulting in 1D traces for
# each line.
msgs.info('Trace the tilts')
self.trace_dict = self.trace_tilts(_mstilt, self.lines_spec, self.lines_spat,
thismask, self.slitcen[:, slit_idx])
# TODO: Show the traces before running the 2D fit
if show:
ginga.show_tilts(viewer, ch, self.trace_dict)
self.spat_order[slit_idx] = self._parse_param(self.par, 'spat_order', slit_idx)
self.spec_order[slit_idx] = self._parse_param(self.par, 'spec_order', slit_idx)
# 2D model of the tilts, includes construction of QA
# NOTE: This also fills in self.all_fit_dict and self.all_trace_dict
coeff_out = self.fit_tilts(self.trace_dict, thismask, self.slitcen[:,slit_idx],
self.spat_order[slit_idx], self.spec_order[slit_idx],
slit_idx,
doqa=doqa, show_QA=show, debug=show)
self.coeffs[:self.spec_order[slit_idx]+1,:self.spat_order[slit_idx]+1,slit_idx] = coeff_out
# TODO: Need a way to assess the success of fit_tilts and
# flag the slit if it fails
# Tilts are created with the size of the original slitmask,
# which corresonds to the same binning as the science
# images, trace images, and pixelflats etc.
self.tilts = tracewave.fit2tilts(self.slitmask_science.shape, coeff_out,
self.par['func2d'])
# Save to final image
thismask_science = self.slitmask_science == slit_spat
self.final_tilts[thismask_science] = self.tilts[thismask_science]
if debug:
# TODO: Add this to the show method?
vmin, vmax = visualization.ZScaleInterval().get_limits(_mstilt)
plt.imshow(_mstilt, origin='lower', interpolation='nearest', aspect='auto',
vmin=vmin, vmax=vmax)
for slit in self.slit_idx:
spat = self.all_trace_dict[slit]['tilts_spat']
spec = self.all_trace_dict[slit]['tilts']
spec_fit = self.all_trace_dict[slit]['tilts_fit']
in_fit = self.all_trace_dict[slit]['tot_mask']
not_fit = np.invert(in_fit) & (spec > 0)
fit_rej = in_fit & np.invert(self.all_trace_dict[slit]['fit_mask'])
fit_keep = in_fit & self.all_trace_dict[slit]['fit_mask']
plt.scatter(spat[not_fit], spec[not_fit], color='C1', marker='.', s=30, lw=0)
plt.scatter(spat[fit_rej], spec[fit_rej], color='C3', marker='.', s=30, lw=0)
plt.scatter(spat[fit_keep], spec[fit_keep], color='k', marker='.', s=30, lw=0)
with_fit = np.invert(np.all(np.invert(fit_keep), axis=0))
for t in range(in_fit.shape[1]):
if not with_fit[t]:
continue
l, r = np.nonzero(in_fit[:,t])[0][[0,-1]]
plt.plot(spat[l:r+1,t], spec_fit[l:r+1,t], color='k')
plt.show()
# Record the Mask
bpmtilts = np.zeros_like(self.slits.mask, dtype=self.slits.bitmask.minimum_dtype())
for flag in ['BADTILTCALIB']:
bpm = self.slits.bitmask.flagged(self.slits.mask, flag)
if np.any(bpm):
bpmtilts[bpm] = self.slits.bitmask.turn_on(bpmtilts[bpm], flag)
# Build and return DataContainer
tilts_dict = {'coeffs':self.coeffs,
'func2d':self.par['func2d'], 'nslit':self.slits.nslits,
'spat_order':self.spat_order, 'spec_order':self.spec_order,
'spat_id':self.slits.spat_id, 'bpmtilts': bpmtilts,
'spat_flexure': self.spat_flexure,
'PYP_SPEC': self.spectrograph.spectrograph}
return WaveTilts(**tilts_dict)
def fit(self, debug=False):
"""
Construct a model of the flat-field image.
For this method to work, :attr:`rawflatimg` must have been
previously constructed; see :func:`build_pixflat`.
The method loops through all slits provided by the :attr:`slits`
object, except those that have been masked (i.e., slits with
``self.slits.mask == True`` are skipped). For each slit:
- Collapse the flat-field data spatially using the
wavelength coordinates provided by the fit to the arc-line
traces (:class:`pypeit.wavetilts.WaveTilts`), and fit the
result with a bspline. This provides the
spatially-averaged spectral response of the instrument.
The data used in the fit is trimmed toward the slit
spatial center via the ``slit_trim`` parameter in
:attr:`flatpar`.
- Use the bspline fit to construct and normalize out the
spectral response.
- Collapse the normalized flat-field data spatially using a
coordinate system defined by the left slit edge. The data
included in the spatial (illumination) profile calculation
is expanded beyond the nominal slit edges using the
``slit_illum_pad`` parameter in :attr:`flatpar`. The raw,
collapsed data is then median filtered (see ``spat_samp``
in :attr:`flatpar`) and Gaussian filtered; see
:func:`pypeit.core.flat.illum_filter`. This creates an
empirical, highly smoothed representation of the
illumination profile that is fit with a bspline using
the :func:`spatial_fit` method. The
construction of the empirical illumination profile (i.e.,
before the bspline fitting) can be done iteratively, where
each iteration sigma-clips outliers; see the
``illum_iter`` and ``illum_rej`` parameters in
:attr:`flatpar` and
:func:`pypeit.core.flat.construct_illum_profile`.
- If requested, the 1D illumination profile is used to
"tweak" the slit edges by offsetting them to a threshold
of the illumination peak to either side of the slit center
(see ``tweak_slits_thresh`` in :attr:`flatpar`), up to a
maximum allowed shift from the existing slit edge (see
``tweak_slits_maxfrac`` in :attr:`flatpar`). See
:func:`pypeit.core.tweak_slit_edges`. If tweaked, the
:func:`spatial_fit` is repeated to place it on the tweaked
slits reference frame.
- Use the bspline fit to construct the 2D illumination image
(:attr:`msillumflat`) and normalize out the spatial
response.
- Fit the residuals of the flat-field data that has been
independently normalized for its spectral and spatial
response with a 2D bspline-polynomial fit. The order of
the polynomial has been optimized via experimentation; it
can be changed but you should use extreme caution when
doing so (see ``twod_fit_npoly``). The multiplication of
the 2D spectral response, 2D spatial response, and joint
2D fit to the high-order residuals define the final flat
model (:attr:`flat_model`).
- Finally, the pixel-to-pixel response of the instrument is
defined as the ratio of the raw flat data to the
best-fitting flat-field model (:attr:`mspixelflat`)
This method is the primary method that builds the
:class:`FlatField` instance, constructing :attr:`mspixelflat`,
:attr:`msillumflat`, and :attr:`flat_model`. All of these
attributes are altered internally. If the slit edges are to be
tweaked using the 1D illumination profile (``tweak_slits`` in
:attr:`flatpar`), the tweaked slit edge arrays in the internal
:class:`pypeit.edgetrace.SlitTraceSet` object, :attr:`slits`,
are also altered.
Used parameters from :attr:`flatpar`
(:class:`pypeit.par.pypeitpar.FlatFieldPar`) are
``spec_samp_fine``, ``spec_samp_coarse``, ``spat_samp``,
``tweak_slits``, ``tweak_slits_thresh``,
``tweak_slits_maxfrac``, ``rej_sticky``, ``slit_trim``,
``slit_illum_pad``, ``illum_iter``, ``illum_rej``, and
``twod_fit_npoly``, ``saturated_slits``.
**Revision History**:
- 11-Mar-2005 First version written by Scott Burles.
- 2005-2018 Improved by J. F. Hennawi and J. X. Prochaska
- 3-Sep-2018 Ported to python by J. F. Hennawi and significantly improved
Args:
debug (:obj:`bool`, optional):
Show plots useful for debugging. This will block
further execution of the code until the plot windows
are closed.
"""
# TODO: break up this function! Can it be partitioned into a series of "core" methods?
# TODO: JFH I wrote all this code and will have to maintain it and I don't want to see it broken up.
# TODO: JXP This definitely needs breaking up..
# Init
self.list_of_spat_bsplines = []
# Set parameters (for convenience;
spec_samp_fine = self.flatpar['spec_samp_fine']
spec_samp_coarse = self.flatpar['spec_samp_coarse']
tweak_slits = self.flatpar['tweak_slits']
tweak_slits_thresh = self.flatpar['tweak_slits_thresh']
tweak_slits_maxfrac = self.flatpar['tweak_slits_maxfrac']
# If sticky, points rejected at each stage (spec, spat, 2d) are
# propagated to the next stage
sticky = self.flatpar['rej_sticky']
trim = self.flatpar['slit_trim']
pad = self.flatpar['slit_illum_pad']
# Iteratively construct the illumination profile by rejecting outliers
npoly = self.flatpar['twod_fit_npoly']
saturated_slits = self.flatpar['saturated_slits']
# Setup images
nspec, nspat = self.rawflatimg.image.shape
rawflat = self.rawflatimg.image
# Good pixel mask
gpm = np.ones_like(rawflat, dtype=bool) if self.rawflatimg.bpm is None else (
1-self.rawflatimg.bpm).astype(bool)
# Flat-field modeling is done in the log of the counts
flat_log = np.log(np.fmax(rawflat, 1.0))
gpm_log = (rawflat > 1.0) & gpm
# set errors to just be 0.5 in the log
ivar_log = gpm_log.astype(float)/0.5**2
# Other setup
nonlinear_counts = self.spectrograph.nonlinear_counts(self.rawflatimg.detector)
# TODO -- JFH -- CONFIRM THIS SHOULD BE ON INIT
# It does need to be *all* of the slits
median_slit_widths = np.median(self.slits.right_init - self.slits.left_init, axis=0)
if tweak_slits:
# NOTE: This copies the input slit edges to a set that can be tweaked.
self.slits.init_tweaked()
# TODO: This needs to include a padding check
# Construct three versions of the slit ID image, all of unmasked slits!
# - an image that uses the padding defined by self.slits
slitid_img_init = self.slits.slit_img(initial=True)
# - an image that uses the extra padding defined by
# self.flatpar. This was always 5 pixels in the previous
# version.
padded_slitid_img = self.slits.slit_img(initial=True, pad=pad)
# - and an image that trims the width of the slit using the
# parameter in self.flatpar. This was always 3 pixels in
# the previous version.
# TODO: Fix this for when trim is a tuple
trimmed_slitid_img = self.slits.slit_img(pad=-trim, initial=True)
# Prep for results
self.mspixelflat = np.ones_like(rawflat)
self.msillumflat = np.ones_like(rawflat)
self.flat_model = np.zeros_like(rawflat)
# Allocate work arrays only once
spec_model = np.ones_like(rawflat)
norm_spec = np.ones_like(rawflat)
norm_spec_spat = np.ones_like(rawflat)
twod_model = np.ones_like(rawflat)
# #################################################
# Model each slit independently
for slit_idx, slit_spat in enumerate(self.slits.spat_id):
# Is this a good slit??
if self.slits.mask[slit_idx] != 0:
msgs.info('Skipping bad slit: {}'.format(slit_spat))
self.list_of_spat_bsplines.append(bspline.bspline(None))
continue
msgs.info('Modeling the flat-field response for slit spat_id={}: {}/{}'.format(
slit_spat, slit_idx+1, self.slits.nslits))
# Find the pixels on the initial slit
onslit_init = slitid_img_init == slit_spat
# Check for saturation of the flat. If there are not enough
# pixels do not attempt a fit, and continue to the next
# slit.
# TODO: set the threshold to a parameter?
good_frac = np.sum(onslit_init & (rawflat < nonlinear_counts))/np.sum(onslit_init)
if good_frac < 0.5:
common_message = 'To change the behavior, use the \'saturated_slits\' parameter ' \
'in the \'flatfield\' parameter group; see here:\n\n' \
'https://pypeit.readthedocs.io/en/latest/pypeit_par.html \n\n' \
'You could also choose to use a different flat-field image ' \
'for this calibration group.'
if saturated_slits == 'crash':
msgs.error('Only {:4.2f}'.format(100*good_frac)
+ '% of the pixels on slit {0} are not saturated. '.format(slit_spat)
+ 'Selected behavior was to crash if this occurred. '
+ common_message)
elif saturated_slits == 'mask':
self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADFLATCALIB')
msgs.warn('Only {:4.2f}'.format(100*good_frac)
+ '% of the pixels on slit {0} are not saturated. '.format(slit_spat)
+ 'Selected behavior was to mask this slit and continue with the '
+ 'remainder of the reduction, meaning no science data will be '
+ 'extracted from this slit. ' + common_message)
elif saturated_slits == 'continue':
self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'SKIPFLATCALIB')
msgs.warn('Only {:4.2f}'.format(100*good_frac)
+ '% of the pixels on slit {0} are not saturated. '.format(slit_spat)
+ 'Selected behavior was to simply continue, meaning no '
+ 'field-flatting correction will be applied to this slit but '
+ 'pypeit will attempt to extract any objects found on this slit. '
+ common_message)
else:
# Should never get here
raise NotImplementedError('Unknown behavior for saturated slits: {0}'.format(
saturated_slits))
self.list_of_spat_bsplines.append(bspline.bspline(None))
continue
# Demand at least 10 pixels per row (on average) per degree
# of the polynomial.
# NOTE: This is not used until the 2D fit. Defined here to
# be close to the definition of ``onslit``.
if npoly is None:
# Approximate number of pixels sampling each spatial pixel
# for this (original) slit.
npercol = np.fmax(np.floor(np.sum(onslit_init)/nspec),1.0)
npoly = np.clip(7, 1, int(np.ceil(npercol/10.)))
# TODO: Always calculate the optimized `npoly` and warn the
# user if npoly is provided but higher than the nominal
# calculation?
# Create an image with the spatial coordinates relative to the left edge of this slit
spat_coo_init = self.slits.spatial_coordinate_image(slitidx=slit_idx, full=True, initial=True)
# Find pixels on the padded and trimmed slit coordinates
onslit_padded = padded_slitid_img == slit_spat
onslit_trimmed = trimmed_slitid_img == slit_spat
# ----------------------------------------------------------
# Collapse the slit spatially and fit the spectral function
# TODO: Put this stuff in a self.spectral_fit method?
# Create the tilts image for this slit
# TODO -- JFH Confirm the sign of this shift is correct!
_flexure = 0. if self.wavetilts.spat_flexure is None else self.wavetilts.spat_flexure
tilts = tracewave.fit2tilts(rawflat.shape, self.wavetilts['coeffs'][:,:,slit_idx],
self.wavetilts['func2d'], spat_shift=-1*_flexure)
# Convert the tilt image to an image with the spectral pixel index
spec_coo = tilts * (nspec-1)
# Only include the trimmed set of pixels in the flat-field
# fit along the spectral direction.
spec_gpm = onslit_trimmed & gpm_log # & (rawflat < nonlinear_counts)
spec_nfit = np.sum(spec_gpm)
spec_ntot = np.sum(onslit_init)
msgs.info('Spectral fit of flatfield for {0}/{1} '.format(spec_nfit, spec_ntot)
+ ' pixels in the slit.')
# Set this to a parameter?
if spec_nfit/spec_ntot < 0.5:
# TODO: Shouldn't this raise an exception or continue to the next slit instead?
msgs.warn('Spectral fit includes only {:.1f}'.format(100*spec_nfit/spec_ntot)
+ '% of the pixels on this slit.' + msgs.newline()
+ ' Either the slit has many bad pixels or the number of '
'trimmed pixels is too large.')
# Sort the pixels by their spectral coordinate.
# TODO: Include ivar and sorted gpm in outputs?
spec_gpm, spec_srt, spec_coo_data, spec_flat_data \
= flat.sorted_flat_data(flat_log, spec_coo, gpm=spec_gpm)
# NOTE: By default np.argsort sorts the data over the last
# axis. Just to avoid the possibility (however unlikely) of
# spec_coo[spec_gpm] returning an array, all the arrays are
# explicitly flattened.
spec_ivar_data = ivar_log[spec_gpm].ravel()[spec_srt]
spec_gpm_data = gpm_log[spec_gpm].ravel()[spec_srt]
# Rejection threshold for spectral fit in log(image)
# TODO: Make this a parameter?
logrej = 0.5
# Fit the spectral direction of the blaze.
# TODO: Figure out how to deal with the fits going crazy at
# the edges of the chip in spec direction
# TODO: Can we add defaults to bspline_profile so that we
# don't have to instantiate invvar and profile_basis
spec_bspl, spec_gpm_fit, spec_flat_fit, _, exit_status \
= utils.bspline_profile(spec_coo_data, spec_flat_data, spec_ivar_data,
np.ones_like(spec_coo_data), ingpm=spec_gpm_data,
nord=4, upper=logrej, lower=logrej,
kwargs_bspline={'bkspace': spec_samp_fine},
kwargs_reject={'groupbadpix': True, 'maxrej': 5})
if exit_status > 1:
# TODO -- MAKE A FUNCTION
msgs.warn('Flat-field spectral response bspline fit failed! Not flat-fielding '
'slit {0} and continuing!'.format(slit_spat))
self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADFLATCALIB')
self.list_of_spat_bsplines.append(bspline.bspline(None))
continue
# Debugging/checking spectral fit
if debug:
utils.bspline_qa(spec_coo_data, spec_flat_data, spec_bspl, spec_gpm_fit,
spec_flat_fit, xlabel='Spectral Pixel', ylabel='log(flat counts)',
title='Spectral Fit for slit={:d}'.format(slit_spat))
if sticky:
# Add rejected pixels to gpm
gpm[spec_gpm] = (spec_gpm_fit & spec_gpm_data)[np.argsort(spec_srt)]
# Construct the model of the flat-field spectral shape
# including padding on either side of the slit.
spec_model[...] = 1.
spec_model[onslit_padded] = np.exp(spec_bspl.value(spec_coo[onslit_padded])[0])
# ----------------------------------------------------------
# ----------------------------------------------------------
# To fit the spatial response, first normalize out the
# spectral response, and then collapse the slit spectrally.
# Normalize out the spectral shape of the flat
norm_spec[...] = 1.
norm_spec[onslit_padded] = rawflat[onslit_padded] \
/ np.fmax(spec_model[onslit_padded],1.0)
# Find pixels fot fit in the spatial direction:
# - Fit pixels in the padded slit that haven't been masked
# by the BPM
spat_gpm = onslit_padded & gpm #& (rawflat < nonlinear_counts)
# - Fit pixels with non-zero flux and less than 70% above
# the average spectral profile.
spat_gpm &= (norm_spec > 0.0) & (norm_spec < 1.7)
# - Determine maximum counts in median filtered flat
# spectrum model.
spec_interp = interpolate.interp1d(spec_coo_data, spec_flat_fit, kind='linear',
assume_sorted=True, bounds_error=False,
fill_value=-np.inf)
spec_sm = utils.fast_running_median(np.exp(spec_interp(np.arange(nspec))),
np.fmax(np.ceil(0.10*nspec).astype(int),10))
# - Only fit pixels with at least values > 10% of this maximum and no less than 1.
spat_gpm &= (spec_model > 0.1*np.amax(spec_sm)) & (spec_model > 1.0)
# Report
spat_nfit = np.sum(spat_gpm)
spat_ntot = np.sum(onslit_padded)
msgs.info('Spatial fit of flatfield for {0}/{1} '.format(spat_nfit, spat_ntot)
+ ' pixels in the slit.')
if spat_nfit/spat_ntot < 0.5:
# TODO: Shouldn't this raise an exception or continue to the next slit instead?
msgs.warn('Spatial fit includes only {:.1f}'.format(100*spat_nfit/spat_ntot)
+ '% of the pixels on this slit.' + msgs.newline()
+ ' Either the slit has many bad pixels, the model of the '
'spectral shape is poor, or the illumination profile is very irregular.')
# First fit -- With initial slits
exit_status, spat_coo_data, spat_flat_data, spat_bspl, spat_gpm_fit, \
spat_flat_fit, spat_flat_data_raw \
= self.spatial_fit(norm_spec, spat_coo_init, median_slit_widths[slit_idx],
spat_gpm, gpm, debug=debug)
if tweak_slits:
# TODO: Should the tweak be based on the bspline fit?
# TODO: Will this break if
left_thresh, left_shift, self.slits.left_tweak[:,slit_idx], right_thresh, \
right_shift, self.slits.right_tweak[:,slit_idx] \
= flat.tweak_slit_edges(self.slits.left_init[:,slit_idx],
self.slits.right_init[:,slit_idx],
spat_coo_data, spat_flat_data,
thresh=tweak_slits_thresh,
maxfrac=tweak_slits_maxfrac, debug=debug)
# TODO: Because the padding doesn't consider adjacent
# slits, calling slit_img for individual slits can be
# different from the result when you construct the
# image for all slits. Fix this...
# Update the onslit mask
_slitid_img = self.slits.slit_img(slitidx=slit_idx, initial=False)
onslit_tweak = _slitid_img == slit_spat
spat_coo_tweak = self.slits.spatial_coordinate_image(slitidx=slit_idx,
slitid_img=_slitid_img)
# Construct the empirical illumination profile
# TODO This is extremely inefficient, because we only need to re-fit the illumflat, but
# spatial_fit does both the reconstruction of the illumination function and the bspline fitting.
# Only the b-spline fitting needs be reddone with the new tweaked spatial coordinates, so that would
# save a ton of runtime. It is not a trivial change becauase the coords are sorted, etc.
exit_status, spat_coo_data, spat_flat_data, spat_bspl, spat_gpm_fit, \
spat_flat_fit, spat_flat_data_raw = self.spatial_fit(
norm_spec, spat_coo_tweak, median_slit_widths[slit_idx], spat_gpm, gpm, debug=False)
spat_coo_final = spat_coo_tweak
else:
_slitid_img = slitid_img_init
spat_coo_final = spat_coo_init
onslit_tweak = onslit_init
# Add an approximate pixel axis at the top
if debug:
# TODO: Move this into a qa plot that gets saved
ax = utils.bspline_qa(spat_coo_data, spat_flat_data, spat_bspl, spat_gpm_fit,
spat_flat_fit, show=False)
ax.scatter(spat_coo_data, spat_flat_data_raw, marker='.', s=1, zorder=0, color='k',
label='raw data')
# Force the center of the slit to be at the center of the plot for the hline
ax.set_xlim(-0.1,1.1)
ax.axvline(0.0, color='lightgreen', linestyle=':', linewidth=2.0,
label='original left edge', zorder=8)
ax.axvline(1.0, color='red', linestyle=':', linewidth=2.0,
label='original right edge', zorder=8)
if tweak_slits and left_shift > 0:
label = 'threshold = {:5.2f}'.format(tweak_slits_thresh) \
+ ' % of max of left illumprofile'
ax.axhline(left_thresh, xmax=0.5, color='lightgreen', linewidth=3.0,
label=label, zorder=10)
ax.axvline(left_shift, color='lightgreen', linestyle='--', linewidth=3.0,
label='tweaked left edge', zorder=11)
if tweak_slits and right_shift > 0:
label = 'threshold = {:5.2f}'.format(tweak_slits_thresh) \
+ ' % of max of right illumprofile'
ax.axhline(right_thresh, xmin=0.5, color='red', linewidth=3.0, label=label,
zorder=10)
ax.axvline(1-right_shift, color='red', linestyle='--', linewidth=3.0,
label='tweaked right edge', zorder=20)
ax.legend()
ax.set_xlabel('Normalized Slit Position')
ax.set_ylabel('Normflat Spatial Profile')
ax.set_title('Illumination Function Fit for slit={:d}'.format(slit_spat))
plt.show()
# ----------------------------------------------------------
# Construct the illumination profile with the tweaked edges
# of the slit
if exit_status <= 1:
# TODO -- JFH -- Check this is ok for flexure!!
self.msillumflat[onslit_tweak] = spat_bspl.value(spat_coo_final[onslit_tweak])[0]
self.list_of_spat_bsplines.append(spat_bspl)
else:
# Save the nada
msgs.warn('Slit illumination profile bspline fit failed! Spatial profile not '
'included in flat-field model for slit {0}!'.format(slit_spat))
self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADFLATCALIB')
self.list_of_spat_bsplines.append(bspline.bspline(None))
continue
# ----------------------------------------------------------
# Fit the 2D residuals of the 1D spectral and spatial fits.
msgs.info('Performing 2D illumination + scattered light flat field fit')
# Construct the spectrally and spatially normalized flat
norm_spec_spat[...] = 1.
norm_spec_spat[onslit_tweak] = rawflat[onslit_tweak] / np.fmax(spec_model[onslit_tweak], 1.0) \
/ np.fmax(self.msillumflat[onslit_tweak], 0.01)
# Sort the pixels by their spectral coordinate. The mask
# uses the nominal padding defined by the slits object.
twod_gpm, twod_srt, twod_spec_coo_data, twod_flat_data \
= flat.sorted_flat_data(norm_spec_spat, spec_coo, gpm=onslit_tweak)
# Also apply the sorting to the spatial coordinates
twod_spat_coo_data = spat_coo_final[twod_gpm].ravel()[twod_srt]
# TODO: Reset back to origin gpm if sticky is true?
twod_gpm_data = gpm[twod_gpm].ravel()[twod_srt]
# Only fit data with less than 30% variations
# TODO: Make 30% a parameter?
twod_gpm_data &= np.absolute(twod_flat_data - 1) < 0.3
# Here we ignore the formal photon counting errors and
# simply assume that a typical error per pixel. This guess
# is somewhat aribtrary. We then set the rejection
# threshold with sigrej_twod
# TODO: Make twod_sig and twod_sigrej parameters?
twod_sig = 0.01
twod_ivar_data = twod_gpm_data.astype(float)/(twod_sig**2)
twod_sigrej = 4.0
poly_basis = basis.fpoly(2.0*twod_spat_coo_data - 1.0, npoly)
# Perform the full 2d fit
twod_bspl, twod_gpm_fit, twod_flat_fit, _ , exit_status \
= utils.bspline_profile(twod_spec_coo_data, twod_flat_data, twod_ivar_data,
poly_basis, ingpm=twod_gpm_data, nord=4,
upper=twod_sigrej, lower=twod_sigrej,
kwargs_bspline={'bkspace': spec_samp_coarse},
kwargs_reject={'groupbadpix': True, 'maxrej': 10})
if debug:
# TODO: Make a plot that shows the residuals in the 2D
# image
resid = twod_flat_data - twod_flat_fit
goodpix = twod_gpm_fit & twod_gpm_data
badpix = np.invert(twod_gpm_fit) & twod_gpm_data
plt.clf()
ax = plt.gca()
ax.plot(twod_spec_coo_data[goodpix], resid[goodpix], color='k', marker='o',
markersize=0.2, mfc='k', fillstyle='full', linestyle='None',
label='good points')
ax.plot(twod_spec_coo_data[badpix], resid[badpix], color='red', marker='+',
markersize=0.5, mfc='red', fillstyle='full', linestyle='None',
label='masked')
ax.axhline(twod_sigrej*twod_sig, color='lawngreen', linestyle='--',
label='rejection thresholds', zorder=10, linewidth=2.0)
ax.axhline(-twod_sigrej*twod_sig, color='lawngreen', linestyle='--', zorder=10,
linewidth=2.0)
# ax.set_ylim(-0.05, 0.05)
ax.legend()
ax.set_xlabel('Spectral Pixel')
ax.set_ylabel('Residuals from pixelflat 2-d fit')
ax.set_title('Spectral Residuals for slit={:d}'.format(slit_spat))
plt.show()
plt.clf()
ax = plt.gca()
ax.plot(twod_spat_coo_data[goodpix], resid[goodpix], color='k', marker='o',
markersize=0.2, mfc='k', fillstyle='full', linestyle='None',
label='good points')
ax.plot(twod_spat_coo_data[badpix], resid[badpix], color='red', marker='+',
markersize=0.5, mfc='red', fillstyle='full', linestyle='None',
label='masked')
ax.axhline(twod_sigrej*twod_sig, color='lawngreen', linestyle='--',
label='rejection thresholds', zorder=10, linewidth=2.0)
ax.axhline(-twod_sigrej*twod_sig, color='lawngreen', linestyle='--', zorder=10,
linewidth=2.0)
# ax.set_ylim((-0.05, 0.05))
# ax.set_xlim(-0.02, 1.02)
ax.legend()
ax.set_xlabel('Normalized Slit Position')
ax.set_ylabel('Residuals from pixelflat 2-d fit')
ax.set_title('Spatial Residuals for slit={:d}'.format(slit_spat))
plt.show()
# Save the 2D residual model
twod_model[...] = 1.
if exit_status > 1:
msgs.warn('Two-dimensional fit to flat-field data failed! No higher order '
'flat-field corrections included in model of slit {0}!'.format(slit_spat))
else:
twod_model[twod_gpm] = twod_flat_fit[np.argsort(twod_srt)]
# Construct the full flat-field model
# TODO: Why is the 0.05 here for the illumflat compared to the 0.01 above?
self.flat_model[onslit_tweak] = twod_model[onslit_tweak] \
* np.fmax(self.msillumflat[onslit_tweak], 0.05) \
* np.fmax(spec_model[onslit_tweak], 1.0)
# Construct the pixel flat
#self.mspixelflat[onslit] = rawflat[onslit]/self.flat_model[onslit]
#self.mspixelflat[onslit_tweak] = 1.
#trimmed_slitid_img_anew = self.slits.slit_img(pad=-trim, slitidx=slit_idx)
#onslit_trimmed_anew = trimmed_slitid_img_anew == slit_spat
self.mspixelflat[onslit_tweak] = rawflat[onslit_tweak]/self.flat_model[onslit_tweak]
# TODO: Add some code here to treat the edges and places where fits
# go bad?
# Set the pixelflat to 1.0 wherever the flat was nonlinear
self.mspixelflat[rawflat >= nonlinear_counts] = 1.0
# Set the pixelflat to 1.0 within trim pixels of all the slit edges
trimmed_slitid_img_new = self.slits.slit_img(pad=-trim, initial=False)
tweaked_slitid_img = self.slits.slit_img(initial=False)
self.mspixelflat[(trimmed_slitid_img_new < 0) & (tweaked_slitid_img > 0)] = 1.0
# Do not apply pixelflat field corrections that are greater than
# 100% to avoid creating edge effects, etc.
self.mspixelflat = np.clip(self.mspixelflat, 0.5, 2.0)
