def illum_filter(spat_flat_data_raw, med_width):
"""
Filter the flat data to produce the empirical illumination
profile.
This is primarily a convenience method for
:func:`construct_illum_profile`. The method first median filters
with a window set by ``med_width`` and then Gaussian-filters the
result with a kernel sigma set to be the maximum of 0.5 or
``med_width``/20.
Args:
spat_flat_data_raw (`numpy.ndarray`_);
Raw flat data collapsed along the spectral direction.
med_width (:obj:`int`):
Width of the median filter window.
Returns:
`numpy.ndarray`_: Returns the filtered spatial profile of the
flat data.
"""
# Median filter the data
spat_flat_data = utils.fast_running_median(spat_flat_data_raw, med_width)
# Gaussian filter the data with a kernel that is 1/20th of the
# median-filter width (or at least 0.5 pixels where here a "pixel"
# is just the index of the data to fit)
return ndimage.filters.gaussian_filter1d(spat_flat_data, np.fmax(med_width/20.0, 0.5),
mode='nearest')
def get_delta_wave(wave, wave_gpm, frac_spec_med_filter=0.03):
r"""
Compute the change in wavelength per pixel.
Given an input wavelength vector and an input good pixel mask, the *raw*
change in wavelength is defined to be ``delta_wave[i] =
wave[i+1]-wave[i]``, with ``delta_wave[-1] = delta_wave[-2]`` to force
``wave`` and ``delta_wave`` to have the same length.
The method imposes a smoothness on the change in wavelength by (1)
running a median filter over the raw values and (2) smoothing
``delta_wave`` with a Gaussian kernel. The boxsize for the median filter
is set by ``frac_spec_med_filter``, and the :math:`\sigma` for the
Gaussian kernel is either a 10th of that boxsize or 3 pixels (whichever
is larger).
Parameters
----------
wave : float `numpy.ndarray`_, shape = (nspec,)
Array of input wavelengths. Must be 1D.
wave_gpm : bool `numpy.ndarray`_, shape = (nspec)
Boolean good-pixel mask defining where the ``wave`` values are
good.
frac_spec_med_filter : :obj:`float`, optional
Fraction of the length of the wavelength vector to use to median
filter the raw change in wavelength, used to impose a smoothness.
Default is 0.03, which means the boxsize for the running median
filter will be approximately ``0.03*nspec`` (forced to be an odd
number of pixels).
Returns
-------
delta_wave : `numpy.ndarray`_, float, shape = (nspec,)
A smooth estimate for the change in wavelength for each pixel in the
input wavelength vector.
"""
# Check input
if wave.ndim != 1:
msgs.error('Input wavelength array must be 1D.')
nspec = wave.size
# This needs to be an odd number
nspec_med_filter = 2 * int(np.round(
nspec * frac_spec_med_filter / 2.0)) + 1
delta_wave = np.zeros_like(wave)
wave_diff = np.diff(wave[wave_gpm])
wave_diff = np.append(wave_diff, wave_diff[-1])
wave_diff_filt = utils.fast_running_median(wave_diff, nspec_med_filter)
# Smooth with a Gaussian kernel
sig_res = np.fmax(nspec_med_filter / 10.0, 3.0)
gauss_kernel = convolution.Gaussian1DKernel(sig_res)
wave_diff_smooth = convolution.convolve(wave_diff_filt,
gauss_kernel,
boundary='extend')
delta_wave[wave_gpm] = wave_diff_smooth
return delta_wave
def fit(self, debug=False):
"""
Construct a model of the flat-field image.
For this method to work, :attr:`rawflatimg` must have been
previously constructed; see :func:`build_pixflat`.
The method loops through all slits provided by the :attr:`slits`
object, except those that have been masked (i.e., slits with
``self.slits.mask == True`` are skipped). For each slit:
- Collapse the flat-field data spatially using the
wavelength coordinates provided by the fit to the arc-line
traces (:class:`pypeit.wavetilts.WaveTilts`), and fit the
result with a bspline. This provides the
spatially-averaged spectral response of the instrument.
The data used in the fit is trimmed toward the slit
spatial center via the ``slit_trim`` parameter in
:attr:`flatpar`.
- Use the bspline fit to construct and normalize out the
spectral response.
- Collapse the normalized flat-field data spatially using a
coordinate system defined by the left slit edge. The data
included in the spatial (illumination) profile calculation
is expanded beyond the nominal slit edges using the
``slit_illum_pad`` parameter in :attr:`flatpar`. The raw,
collapsed data is then median filtered (see ``spat_samp``
in :attr:`flatpar`) and Gaussian filtered; see
:func:`pypeit.core.flat.illum_filter`. This creates an
empirical, highly smoothed representation of the
illumination profile that is fit with a bspline using
the :func:`spatial_fit` method. The
construction of the empirical illumination profile (i.e.,
before the bspline fitting) can be done iteratively, where
each iteration sigma-clips outliers; see the
``illum_iter`` and ``illum_rej`` parameters in
:attr:`flatpar` and
:func:`pypeit.core.flat.construct_illum_profile`.
- If requested, the 1D illumination profile is used to
"tweak" the slit edges by offsetting them to a threshold
of the illumination peak to either side of the slit center
(see ``tweak_slits_thresh`` in :attr:`flatpar`), up to a
maximum allowed shift from the existing slit edge (see
``tweak_slits_maxfrac`` in :attr:`flatpar`). See
:func:`pypeit.core.tweak_slit_edges`. If tweaked, the
:func:`spatial_fit` is repeated to place it on the tweaked
slits reference frame.
- Use the bspline fit to construct the 2D illumination image
(:attr:`msillumflat`) and normalize out the spatial
response.
- Fit the residuals of the flat-field data that has been
independently normalized for its spectral and spatial
response with a 2D bspline-polynomial fit. The order of
the polynomial has been optimized via experimentation; it
can be changed but you should use extreme caution when
doing so (see ``twod_fit_npoly``). The multiplication of
the 2D spectral response, 2D spatial response, and joint
2D fit to the high-order residuals define the final flat
model (:attr:`flat_model`).
- Finally, the pixel-to-pixel response of the instrument is
defined as the ratio of the raw flat data to the
best-fitting flat-field model (:attr:`mspixelflat`)
This method is the primary method that builds the
:class:`FlatField` instance, constructing :attr:`mspixelflat`,
:attr:`msillumflat`, and :attr:`flat_model`. All of these
attributes are altered internally. If the slit edges are to be
tweaked using the 1D illumination profile (``tweak_slits`` in
:attr:`flatpar`), the tweaked slit edge arrays in the internal
:class:`pypeit.edgetrace.SlitTraceSet` object, :attr:`slits`,
are also altered.
Used parameters from :attr:`flatpar`
(:class:`pypeit.par.pypeitpar.FlatFieldPar`) are
``spec_samp_fine``, ``spec_samp_coarse``, ``spat_samp``,
``tweak_slits``, ``tweak_slits_thresh``,
``tweak_slits_maxfrac``, ``rej_sticky``, ``slit_trim``,
``slit_illum_pad``, ``illum_iter``, ``illum_rej``, and
``twod_fit_npoly``, ``saturated_slits``.
**Revision History**:
- 11-Mar-2005 First version written by Scott Burles.
- 2005-2018 Improved by J. F. Hennawi and J. X. Prochaska
- 3-Sep-2018 Ported to python by J. F. Hennawi and significantly improved
Args:
debug (:obj:`bool`, optional):
Show plots useful for debugging. This will block
further execution of the code until the plot windows
are closed.
"""
# TODO: break up this function! Can it be partitioned into a series of "core" methods?
# TODO: JFH I wrote all this code and will have to maintain it and I don't want to see it broken up.
# TODO: JXP This definitely needs breaking up..
# Init
self.list_of_spat_bsplines = []
# Set parameters (for convenience;
spec_samp_fine = self.flatpar['spec_samp_fine']
spec_samp_coarse = self.flatpar['spec_samp_coarse']
tweak_slits = self.flatpar['tweak_slits']
tweak_slits_thresh = self.flatpar['tweak_slits_thresh']
tweak_slits_maxfrac = self.flatpar['tweak_slits_maxfrac']
# If sticky, points rejected at each stage (spec, spat, 2d) are
# propagated to the next stage
sticky = self.flatpar['rej_sticky']
trim = self.flatpar['slit_trim']
pad = self.flatpar['slit_illum_pad']
# Iteratively construct the illumination profile by rejecting outliers
npoly = self.flatpar['twod_fit_npoly']
saturated_slits = self.flatpar['saturated_slits']
# Setup images
nspec, nspat = self.rawflatimg.image.shape
rawflat = self.rawflatimg.image
# Good pixel mask
gpm = np.ones_like(rawflat, dtype=bool) if self.rawflatimg.bpm is None else (
1-self.rawflatimg.bpm).astype(bool)
# Flat-field modeling is done in the log of the counts
flat_log = np.log(np.fmax(rawflat, 1.0))
gpm_log = (rawflat > 1.0) & gpm
# set errors to just be 0.5 in the log
ivar_log = gpm_log.astype(float)/0.5**2
# Other setup
nonlinear_counts = self.spectrograph.nonlinear_counts(self.rawflatimg.detector)
# TODO -- JFH -- CONFIRM THIS SHOULD BE ON INIT
# It does need to be *all* of the slits
median_slit_widths = np.median(self.slits.right_init - self.slits.left_init, axis=0)
if tweak_slits:
# NOTE: This copies the input slit edges to a set that can be tweaked.
self.slits.init_tweaked()
# TODO: This needs to include a padding check
# Construct three versions of the slit ID image, all of unmasked slits!
# - an image that uses the padding defined by self.slits
slitid_img_init = self.slits.slit_img(initial=True)
# - an image that uses the extra padding defined by
# self.flatpar. This was always 5 pixels in the previous
# version.
padded_slitid_img = self.slits.slit_img(initial=True, pad=pad)
# - and an image that trims the width of the slit using the
# parameter in self.flatpar. This was always 3 pixels in
# the previous version.
# TODO: Fix this for when trim is a tuple
trimmed_slitid_img = self.slits.slit_img(pad=-trim, initial=True)
# Prep for results
self.mspixelflat = np.ones_like(rawflat)
self.msillumflat = np.ones_like(rawflat)
self.flat_model = np.zeros_like(rawflat)
# Allocate work arrays only once
spec_model = np.ones_like(rawflat)
norm_spec = np.ones_like(rawflat)
norm_spec_spat = np.ones_like(rawflat)
twod_model = np.ones_like(rawflat)
# #################################################
# Model each slit independently
for slit_idx, slit_spat in enumerate(self.slits.spat_id):
# Is this a good slit??
if self.slits.mask[slit_idx] != 0:
msgs.info('Skipping bad slit: {}'.format(slit_spat))
self.list_of_spat_bsplines.append(bspline.bspline(None))
continue
msgs.info('Modeling the flat-field response for slit spat_id={}: {}/{}'.format(
slit_spat, slit_idx+1, self.slits.nslits))
# Find the pixels on the initial slit
onslit_init = slitid_img_init == slit_spat
# Check for saturation of the flat. If there are not enough
# pixels do not attempt a fit, and continue to the next
# slit.
# TODO: set the threshold to a parameter?
good_frac = np.sum(onslit_init & (rawflat < nonlinear_counts))/np.sum(onslit_init)
if good_frac < 0.5:
common_message = 'To change the behavior, use the \'saturated_slits\' parameter ' \
'in the \'flatfield\' parameter group; see here:\n\n' \
'https://pypeit.readthedocs.io/en/latest/pypeit_par.html \n\n' \
'You could also choose to use a different flat-field image ' \
'for this calibration group.'
if saturated_slits == 'crash':
msgs.error('Only {:4.2f}'.format(100*good_frac)
+ '% of the pixels on slit {0} are not saturated. '.format(slit_spat)
+ 'Selected behavior was to crash if this occurred. '
+ common_message)
elif saturated_slits == 'mask':
self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADFLATCALIB')
msgs.warn('Only {:4.2f}'.format(100*good_frac)
+ '% of the pixels on slit {0} are not saturated. '.format(slit_spat)
+ 'Selected behavior was to mask this slit and continue with the '
+ 'remainder of the reduction, meaning no science data will be '
+ 'extracted from this slit. ' + common_message)
elif saturated_slits == 'continue':
self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'SKIPFLATCALIB')
msgs.warn('Only {:4.2f}'.format(100*good_frac)
+ '% of the pixels on slit {0} are not saturated. '.format(slit_spat)
+ 'Selected behavior was to simply continue, meaning no '
+ 'field-flatting correction will be applied to this slit but '
+ 'pypeit will attempt to extract any objects found on this slit. '
+ common_message)
else:
# Should never get here
raise NotImplementedError('Unknown behavior for saturated slits: {0}'.format(
saturated_slits))
self.list_of_spat_bsplines.append(bspline.bspline(None))
continue
# Demand at least 10 pixels per row (on average) per degree
# of the polynomial.
# NOTE: This is not used until the 2D fit. Defined here to
# be close to the definition of ``onslit``.
if npoly is None:
# Approximate number of pixels sampling each spatial pixel
# for this (original) slit.
npercol = np.fmax(np.floor(np.sum(onslit_init)/nspec),1.0)
npoly = np.clip(7, 1, int(np.ceil(npercol/10.)))
# TODO: Always calculate the optimized `npoly` and warn the
# user if npoly is provided but higher than the nominal
# calculation?
# Create an image with the spatial coordinates relative to the left edge of this slit
spat_coo_init = self.slits.spatial_coordinate_image(slitidx=slit_idx, full=True, initial=True)
# Find pixels on the padded and trimmed slit coordinates
onslit_padded = padded_slitid_img == slit_spat
onslit_trimmed = trimmed_slitid_img == slit_spat
# ----------------------------------------------------------
# Collapse the slit spatially and fit the spectral function
# TODO: Put this stuff in a self.spectral_fit method?
# Create the tilts image for this slit
# TODO -- JFH Confirm the sign of this shift is correct!
_flexure = 0. if self.wavetilts.spat_flexure is None else self.wavetilts.spat_flexure
tilts = tracewave.fit2tilts(rawflat.shape, self.wavetilts['coeffs'][:,:,slit_idx],
self.wavetilts['func2d'], spat_shift=-1*_flexure)
# Convert the tilt image to an image with the spectral pixel index
spec_coo = tilts * (nspec-1)
# Only include the trimmed set of pixels in the flat-field
# fit along the spectral direction.
spec_gpm = onslit_trimmed & gpm_log # & (rawflat < nonlinear_counts)
spec_nfit = np.sum(spec_gpm)
spec_ntot = np.sum(onslit_init)
msgs.info('Spectral fit of flatfield for {0}/{1} '.format(spec_nfit, spec_ntot)
+ ' pixels in the slit.')
# Set this to a parameter?
if spec_nfit/spec_ntot < 0.5:
# TODO: Shouldn't this raise an exception or continue to the next slit instead?
msgs.warn('Spectral fit includes only {:.1f}'.format(100*spec_nfit/spec_ntot)
+ '% of the pixels on this slit.' + msgs.newline()
+ ' Either the slit has many bad pixels or the number of '
'trimmed pixels is too large.')
# Sort the pixels by their spectral coordinate.
# TODO: Include ivar and sorted gpm in outputs?
spec_gpm, spec_srt, spec_coo_data, spec_flat_data \
= flat.sorted_flat_data(flat_log, spec_coo, gpm=spec_gpm)
# NOTE: By default np.argsort sorts the data over the last
# axis. Just to avoid the possibility (however unlikely) of
# spec_coo[spec_gpm] returning an array, all the arrays are
# explicitly flattened.
spec_ivar_data = ivar_log[spec_gpm].ravel()[spec_srt]
spec_gpm_data = gpm_log[spec_gpm].ravel()[spec_srt]
# Rejection threshold for spectral fit in log(image)
# TODO: Make this a parameter?
logrej = 0.5
# Fit the spectral direction of the blaze.
# TODO: Figure out how to deal with the fits going crazy at
# the edges of the chip in spec direction
# TODO: Can we add defaults to bspline_profile so that we
# don't have to instantiate invvar and profile_basis
spec_bspl, spec_gpm_fit, spec_flat_fit, _, exit_status \
= utils.bspline_profile(spec_coo_data, spec_flat_data, spec_ivar_data,
np.ones_like(spec_coo_data), ingpm=spec_gpm_data,
nord=4, upper=logrej, lower=logrej,
kwargs_bspline={'bkspace': spec_samp_fine},
kwargs_reject={'groupbadpix': True, 'maxrej': 5})
if exit_status > 1:
# TODO -- MAKE A FUNCTION
msgs.warn('Flat-field spectral response bspline fit failed! Not flat-fielding '
'slit {0} and continuing!'.format(slit_spat))
self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADFLATCALIB')
self.list_of_spat_bsplines.append(bspline.bspline(None))
continue
# Debugging/checking spectral fit
if debug:
utils.bspline_qa(spec_coo_data, spec_flat_data, spec_bspl, spec_gpm_fit,
spec_flat_fit, xlabel='Spectral Pixel', ylabel='log(flat counts)',
title='Spectral Fit for slit={:d}'.format(slit_spat))
if sticky:
# Add rejected pixels to gpm
gpm[spec_gpm] = (spec_gpm_fit & spec_gpm_data)[np.argsort(spec_srt)]
# Construct the model of the flat-field spectral shape
# including padding on either side of the slit.
spec_model[...] = 1.
spec_model[onslit_padded] = np.exp(spec_bspl.value(spec_coo[onslit_padded])[0])
# ----------------------------------------------------------
# ----------------------------------------------------------
# To fit the spatial response, first normalize out the
# spectral response, and then collapse the slit spectrally.
# Normalize out the spectral shape of the flat
norm_spec[...] = 1.
norm_spec[onslit_padded] = rawflat[onslit_padded] \
/ np.fmax(spec_model[onslit_padded],1.0)
# Find pixels fot fit in the spatial direction:
# - Fit pixels in the padded slit that haven't been masked
# by the BPM
spat_gpm = onslit_padded & gpm #& (rawflat < nonlinear_counts)
# - Fit pixels with non-zero flux and less than 70% above
# the average spectral profile.
spat_gpm &= (norm_spec > 0.0) & (norm_spec < 1.7)
# - Determine maximum counts in median filtered flat
# spectrum model.
spec_interp = interpolate.interp1d(spec_coo_data, spec_flat_fit, kind='linear',
assume_sorted=True, bounds_error=False,
fill_value=-np.inf)
spec_sm = utils.fast_running_median(np.exp(spec_interp(np.arange(nspec))),
np.fmax(np.ceil(0.10*nspec).astype(int),10))
# - Only fit pixels with at least values > 10% of this maximum and no less than 1.
spat_gpm &= (spec_model > 0.1*np.amax(spec_sm)) & (spec_model > 1.0)
# Report
spat_nfit = np.sum(spat_gpm)
spat_ntot = np.sum(onslit_padded)
msgs.info('Spatial fit of flatfield for {0}/{1} '.format(spat_nfit, spat_ntot)
+ ' pixels in the slit.')
if spat_nfit/spat_ntot < 0.5:
# TODO: Shouldn't this raise an exception or continue to the next slit instead?
msgs.warn('Spatial fit includes only {:.1f}'.format(100*spat_nfit/spat_ntot)
+ '% of the pixels on this slit.' + msgs.newline()
+ ' Either the slit has many bad pixels, the model of the '
'spectral shape is poor, or the illumination profile is very irregular.')
# First fit -- With initial slits
exit_status, spat_coo_data, spat_flat_data, spat_bspl, spat_gpm_fit, \
spat_flat_fit, spat_flat_data_raw \
= self.spatial_fit(norm_spec, spat_coo_init, median_slit_widths[slit_idx],
spat_gpm, gpm, debug=debug)
if tweak_slits:
# TODO: Should the tweak be based on the bspline fit?
# TODO: Will this break if
left_thresh, left_shift, self.slits.left_tweak[:,slit_idx], right_thresh, \
right_shift, self.slits.right_tweak[:,slit_idx] \
= flat.tweak_slit_edges(self.slits.left_init[:,slit_idx],
self.slits.right_init[:,slit_idx],
spat_coo_data, spat_flat_data,
thresh=tweak_slits_thresh,
maxfrac=tweak_slits_maxfrac, debug=debug)
# TODO: Because the padding doesn't consider adjacent
# slits, calling slit_img for individual slits can be
# different from the result when you construct the
# image for all slits. Fix this...
# Update the onslit mask
_slitid_img = self.slits.slit_img(slitidx=slit_idx, initial=False)
onslit_tweak = _slitid_img == slit_spat
spat_coo_tweak = self.slits.spatial_coordinate_image(slitidx=slit_idx,
slitid_img=_slitid_img)
# Construct the empirical illumination profile
# TODO This is extremely inefficient, because we only need to re-fit the illumflat, but
# spatial_fit does both the reconstruction of the illumination function and the bspline fitting.
# Only the b-spline fitting needs be reddone with the new tweaked spatial coordinates, so that would
# save a ton of runtime. It is not a trivial change becauase the coords are sorted, etc.
exit_status, spat_coo_data, spat_flat_data, spat_bspl, spat_gpm_fit, \
spat_flat_fit, spat_flat_data_raw = self.spatial_fit(
norm_spec, spat_coo_tweak, median_slit_widths[slit_idx], spat_gpm, gpm, debug=False)
spat_coo_final = spat_coo_tweak
else:
_slitid_img = slitid_img_init
spat_coo_final = spat_coo_init
onslit_tweak = onslit_init
# Add an approximate pixel axis at the top
if debug:
# TODO: Move this into a qa plot that gets saved
ax = utils.bspline_qa(spat_coo_data, spat_flat_data, spat_bspl, spat_gpm_fit,
spat_flat_fit, show=False)
ax.scatter(spat_coo_data, spat_flat_data_raw, marker='.', s=1, zorder=0, color='k',
label='raw data')
# Force the center of the slit to be at the center of the plot for the hline
ax.set_xlim(-0.1,1.1)
ax.axvline(0.0, color='lightgreen', linestyle=':', linewidth=2.0,
label='original left edge', zorder=8)
ax.axvline(1.0, color='red', linestyle=':', linewidth=2.0,
label='original right edge', zorder=8)
if tweak_slits and left_shift > 0:
label = 'threshold = {:5.2f}'.format(tweak_slits_thresh) \
+ ' % of max of left illumprofile'
ax.axhline(left_thresh, xmax=0.5, color='lightgreen', linewidth=3.0,
label=label, zorder=10)
ax.axvline(left_shift, color='lightgreen', linestyle='--', linewidth=3.0,
label='tweaked left edge', zorder=11)
if tweak_slits and right_shift > 0:
label = 'threshold = {:5.2f}'.format(tweak_slits_thresh) \
+ ' % of max of right illumprofile'
ax.axhline(right_thresh, xmin=0.5, color='red', linewidth=3.0, label=label,
zorder=10)
ax.axvline(1-right_shift, color='red', linestyle='--', linewidth=3.0,
label='tweaked right edge', zorder=20)
ax.legend()
ax.set_xlabel('Normalized Slit Position')
ax.set_ylabel('Normflat Spatial Profile')
ax.set_title('Illumination Function Fit for slit={:d}'.format(slit_spat))
plt.show()
# ----------------------------------------------------------
# Construct the illumination profile with the tweaked edges
# of the slit
if exit_status <= 1:
# TODO -- JFH -- Check this is ok for flexure!!
self.msillumflat[onslit_tweak] = spat_bspl.value(spat_coo_final[onslit_tweak])[0]
self.list_of_spat_bsplines.append(spat_bspl)
else:
# Save the nada
msgs.warn('Slit illumination profile bspline fit failed! Spatial profile not '
'included in flat-field model for slit {0}!'.format(slit_spat))
self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADFLATCALIB')
self.list_of_spat_bsplines.append(bspline.bspline(None))
continue
# ----------------------------------------------------------
# Fit the 2D residuals of the 1D spectral and spatial fits.
msgs.info('Performing 2D illumination + scattered light flat field fit')
# Construct the spectrally and spatially normalized flat
norm_spec_spat[...] = 1.
norm_spec_spat[onslit_tweak] = rawflat[onslit_tweak] / np.fmax(spec_model[onslit_tweak], 1.0) \
/ np.fmax(self.msillumflat[onslit_tweak], 0.01)
# Sort the pixels by their spectral coordinate. The mask
# uses the nominal padding defined by the slits object.
twod_gpm, twod_srt, twod_spec_coo_data, twod_flat_data \
= flat.sorted_flat_data(norm_spec_spat, spec_coo, gpm=onslit_tweak)
# Also apply the sorting to the spatial coordinates
twod_spat_coo_data = spat_coo_final[twod_gpm].ravel()[twod_srt]
# TODO: Reset back to origin gpm if sticky is true?
twod_gpm_data = gpm[twod_gpm].ravel()[twod_srt]
# Only fit data with less than 30% variations
# TODO: Make 30% a parameter?
twod_gpm_data &= np.absolute(twod_flat_data - 1) < 0.3
# Here we ignore the formal photon counting errors and
# simply assume that a typical error per pixel. This guess
# is somewhat aribtrary. We then set the rejection
# threshold with sigrej_twod
# TODO: Make twod_sig and twod_sigrej parameters?
twod_sig = 0.01
twod_ivar_data = twod_gpm_data.astype(float)/(twod_sig**2)
twod_sigrej = 4.0
poly_basis = basis.fpoly(2.0*twod_spat_coo_data - 1.0, npoly)
# Perform the full 2d fit
twod_bspl, twod_gpm_fit, twod_flat_fit, _ , exit_status \
= utils.bspline_profile(twod_spec_coo_data, twod_flat_data, twod_ivar_data,
poly_basis, ingpm=twod_gpm_data, nord=4,
upper=twod_sigrej, lower=twod_sigrej,
kwargs_bspline={'bkspace': spec_samp_coarse},
kwargs_reject={'groupbadpix': True, 'maxrej': 10})
if debug:
# TODO: Make a plot that shows the residuals in the 2D
# image
resid = twod_flat_data - twod_flat_fit
goodpix = twod_gpm_fit & twod_gpm_data
badpix = np.invert(twod_gpm_fit) & twod_gpm_data
plt.clf()
ax = plt.gca()
ax.plot(twod_spec_coo_data[goodpix], resid[goodpix], color='k', marker='o',
markersize=0.2, mfc='k', fillstyle='full', linestyle='None',
label='good points')
ax.plot(twod_spec_coo_data[badpix], resid[badpix], color='red', marker='+',
markersize=0.5, mfc='red', fillstyle='full', linestyle='None',
label='masked')
ax.axhline(twod_sigrej*twod_sig, color='lawngreen', linestyle='--',
label='rejection thresholds', zorder=10, linewidth=2.0)
ax.axhline(-twod_sigrej*twod_sig, color='lawngreen', linestyle='--', zorder=10,
linewidth=2.0)
# ax.set_ylim(-0.05, 0.05)
ax.legend()
ax.set_xlabel('Spectral Pixel')
ax.set_ylabel('Residuals from pixelflat 2-d fit')
ax.set_title('Spectral Residuals for slit={:d}'.format(slit_spat))
plt.show()
plt.clf()
ax = plt.gca()
ax.plot(twod_spat_coo_data[goodpix], resid[goodpix], color='k', marker='o',
markersize=0.2, mfc='k', fillstyle='full', linestyle='None',
label='good points')
ax.plot(twod_spat_coo_data[badpix], resid[badpix], color='red', marker='+',
markersize=0.5, mfc='red', fillstyle='full', linestyle='None',
label='masked')
ax.axhline(twod_sigrej*twod_sig, color='lawngreen', linestyle='--',
label='rejection thresholds', zorder=10, linewidth=2.0)
ax.axhline(-twod_sigrej*twod_sig, color='lawngreen', linestyle='--', zorder=10,
linewidth=2.0)
# ax.set_ylim((-0.05, 0.05))
# ax.set_xlim(-0.02, 1.02)
ax.legend()
ax.set_xlabel('Normalized Slit Position')
ax.set_ylabel('Residuals from pixelflat 2-d fit')
ax.set_title('Spatial Residuals for slit={:d}'.format(slit_spat))
plt.show()
# Save the 2D residual model
twod_model[...] = 1.
if exit_status > 1:
msgs.warn('Two-dimensional fit to flat-field data failed! No higher order '
'flat-field corrections included in model of slit {0}!'.format(slit_spat))
else:
twod_model[twod_gpm] = twod_flat_fit[np.argsort(twod_srt)]
# Construct the full flat-field model
# TODO: Why is the 0.05 here for the illumflat compared to the 0.01 above?
self.flat_model[onslit_tweak] = twod_model[onslit_tweak] \
* np.fmax(self.msillumflat[onslit_tweak], 0.05) \
* np.fmax(spec_model[onslit_tweak], 1.0)
# Construct the pixel flat
#self.mspixelflat[onslit] = rawflat[onslit]/self.flat_model[onslit]
#self.mspixelflat[onslit_tweak] = 1.
#trimmed_slitid_img_anew = self.slits.slit_img(pad=-trim, slitidx=slit_idx)
#onslit_trimmed_anew = trimmed_slitid_img_anew == slit_spat
self.mspixelflat[onslit_tweak] = rawflat[onslit_tweak]/self.flat_model[onslit_tweak]
# TODO: Add some code here to treat the edges and places where fits
# go bad?
# Set the pixelflat to 1.0 wherever the flat was nonlinear
self.mspixelflat[rawflat >= nonlinear_counts] = 1.0
# Set the pixelflat to 1.0 within trim pixels of all the slit edges
trimmed_slitid_img_new = self.slits.slit_img(pad=-trim, initial=False)
tweaked_slitid_img = self.slits.slit_img(initial=False)
self.mspixelflat[(trimmed_slitid_img_new < 0) & (tweaked_slitid_img > 0)] = 1.0
# Do not apply pixelflat field corrections that are greater than
# 100% to avoid creating edge effects, etc.
self.mspixelflat = np.clip(self.mspixelflat, 0.5, 2.0)