def test_step_by_step(master_dir): # Masters spectrograph = load_spectrograph('shane_kast_blue') msarc, tslits_dict, mstrace = load_kast_blue_masters(aimg=True, tslits=True) # Instantiate master_key = 'A_1_01' parset = spectrograph.default_pypeit_par() par = parset['calibrations']['tilts'] wavepar = parset['calibrations']['wavelengths'] waveTilts = wavetilts.WaveTilts(msarc, tslits_dict, spectrograph, par, wavepar, det=1, master_key=master_key, master_dir=master_dir,reuse_masters=True) # Extract arcs arccen, maskslits = waveTilts.extract_arcs(waveTilts.slitcen, waveTilts.slitmask, msarc, waveTilts.inmask) assert arccen.shape == (2048,1) # Tilts in the slit slit = 0 waveTilts.slitmask = pixels.tslits2mask(waveTilts.tslits_dict) thismask = waveTilts.slitmask == slit waveTilts.lines_spec, waveTilts.lines_spat = waveTilts.find_lines(arccen[:, slit], waveTilts.slitcen[:, slit], slit) trcdict = waveTilts.trace_tilts(waveTilts.msarc, waveTilts.lines_spec, waveTilts.lines_spat, thismask, slit) assert isinstance(trcdict, dict) # 2D Fit spat_order = waveTilts._parse_param(waveTilts.par, 'spat_order', slit) spec_order = waveTilts._parse_param(waveTilts.par, 'spec_order', slit) coeffs = waveTilts.fit_tilts(trcdict, thismask, waveTilts.slitcen[:, slit], spat_order, spec_order,slit, doqa=False) tilts = tracewave.fit2tilts(waveTilts.slitmask_science.shape, coeffs, waveTilts.par['func2d']) assert np.max(tilts) < 1.01
def fit2tiltimg(self, slitmask, flexure=None): """ Generate a tilt image from the fit parameters Mainly to allow for flexure Args: slitmask (`numpy.ndarray`_): flexure (float, optional): Spatial shift of the tilt image onto the desired frame (typically a science image) Returns: `numpy.ndarray`_: New tilt image """ _flexure = 0. if flexure is None else flexure final_tilts = np.zeros_like(slitmask).astype(float) gdslit_spat = np.unique(slitmask[slitmask >= 0]).astype(int) # Loop for slit_spat in gdslit_spat: slit_idx = self.spatid_to_zero(slit_spat) # Calculate coeff_out = self.coeffs[:self.spec_order[slit_idx]+1,:self.spat_order[slit_idx]+1,slit_idx] _tilts = tracewave.fit2tilts(final_tilts.shape, coeff_out, self.func2d, spat_shift=-1*_flexure) # Fill thismask_science = slitmask == slit_spat final_tilts[thismask_science] = _tilts[thismask_science] # Return return final_tilts
def run(self, maskslits=None, doqa=True, debug=False, show=False): """ Main driver for tracing arc lines Code flow:: 1. Extract an arc spectrum down the center of each slit/order 2. Loop on slits/orders i. Trace and fit the arc lines (This is done twice, once with trace_crude as the tracing crutch, then again with a PCA model fit as the crutch). ii. Repeat trace. iii. 2D Fit to the offset from slitcen iv. Save Keyword Args: maskslits (`numpy.ndarray`_, optional): Boolean array to ignore slits. doqa (bool): debug (bool): show (bool): Returns: dict, ndarray: Tilts dict and maskslits array """ if maskslits is None: maskslits = np.zeros(self.nslits, dtype=bool) # Extract the arc spectra for all slits self.arccen, self.arc_maskslit = self.extract_arcs( self.slitcen, self.slitmask, self.msarc, self.inmask) # maskslit self.mask = maskslits & (self.arc_maskslit == 1) gdslits = np.where(np.invert(self.mask))[0] # Final tilts image self.final_tilts = np.zeros(self.shape_science, dtype=float) max_spat_dim = (np.asarray(self.par['spat_order']) + 1).max() max_spec_dim = (np.asarray(self.par['spec_order']) + 1).max() self.coeffs = np.zeros((max_spec_dim, max_spat_dim, self.nslits)) self.spat_order = np.zeros(self.nslits, dtype=int) self.spec_order = np.zeros(self.nslits, dtype=int) # TODO sort out show methods for debugging #if show: # viewer,ch = ginga.show_image(self.msarc*(self.slitmask > -1),chname='tilts') # Loop on all slits for slit in gdslits: msgs.info('Computing tilts for slit {:d}/{:d}'.format( slit, self.nslits - 1)) # Identify lines for tracing tilts msgs.info('Finding lines for tilt analysis') self.lines_spec, self.lines_spat = self.find_lines( self.arccen[:, slit], self.slitcen[:, slit], slit, debug=debug) if self.lines_spec is None: self.mask[slit] = True maskslits[slit] = True continue thismask = self.slitmask == slit # Trace msgs.info('Trace the tilts') self.trace_dict = self.trace_tilts(self.msarc, self.lines_spec, self.lines_spat, thismask, self.slitcen[:, slit]) #if show: # ginga.show_tilts(viewer, ch, self.trace_dict) self.spat_order[slit] = self._parse_param(self.par, 'spat_order', slit) self.spec_order[slit] = self._parse_param(self.par, 'spec_order', slit) # 2D model of the tilts, includes construction of QA coeff_out = self.fit_tilts(self.trace_dict, thismask, self.slitcen[:, slit], self.spat_order[slit], self.spec_order[slit], slit, doqa=doqa, show_QA=show, debug=show) self.coeffs[0:self.spec_order[slit] + 1, 0:self.spat_order[slit] + 1, slit] = coeff_out # Tilts are created with the size of the original slitmask, # which corresonds to the same binning as the science # images, trace images, and pixelflats etc. self.tilts = tracewave.fit2tilts(self.slitmask_science.shape, coeff_out, self.par['func2d']) # Save to final image thismask_science = self.slitmask_science == slit self.final_tilts[thismask_science] = self.tilts[thismask_science] self.tilts_dict = { 'tilts': self.final_tilts, 'coeffs': self.coeffs, 'slitcen': self.slitcen, 'func2d': self.par['func2d'], 'nslit': self.nslits, 'spat_order': self.spat_order, 'spec_order': self.spec_order } return self.tilts_dict, maskslits
def run(self, maskslits=None, doqa=True, debug=False, show=False): """ Main driver for tracing arc lines Code flow: 1. Extract an arc spectrum down the center of each slit/order 2. Loop on slits/orders i. Trace and fit the arc lines (This is done twice, once with trace_crude as the tracing crutch, then again with a PCA model fit as the crutch). ii. Repeat trace. iii. 2D Fit to the offset from slitcen iv. Save Args: maskslits (`numpy.ndarray`_, optional): Boolean array to ignore slits. doqa (bool): debug (bool): show (bool): Returns: dict, ndarray: Tilts dict and maskslits array """ if maskslits is None: maskslits = np.zeros(self.nslits, dtype=bool) # Extract the arc spectra for all slits self.arccen, self.arccen_bpm, self.arc_maskslit = self.extract_arcs() # TODO: Leave for now. Used for debugging # self.par['rm_continuum'] = True # debug = True # show = True # Subtract arc continuum _msarc = self.msarc.image.copy() if self.par['rm_continuum']: continuum = self.model_arc_continuum(debug=debug) _msarc -= continuum if debug: # TODO: Put this into a function vmin, vmax = visualization.ZScaleInterval().get_limits(_msarc) w, h = plt.figaspect(1) fig = plt.figure(figsize=(3 * w, h)) ax = fig.add_axes([0.15 / 3, 0.1, 0.8 / 3, 0.8]) ax.imshow(self.msarc.image, origin='lower', interpolation='nearest', aspect='auto', vmin=vmin, vmax=vmax) ax.set_title('MasterArc') ax = fig.add_axes([1.15 / 3, 0.1, 0.8 / 3, 0.8]) ax.imshow(continuum, origin='lower', interpolation='nearest', aspect='auto', vmin=vmin, vmax=vmax) ax.set_title('Continuum') ax = fig.add_axes([2.15 / 3, 0.1, 0.8 / 3, 0.8]) ax.imshow(_msarc, origin='lower', interpolation='nearest', aspect='auto', vmin=vmin, vmax=vmax) ax.set_title('MasterArc - Continuum') plt.show() # maskslit self.mask = np.any([maskslits, self.arc_maskslit == 1], axis=0) gdslits = np.where(np.invert(self.mask))[0] # Final tilts image self.final_tilts = np.zeros(self.shape_science, dtype=float) max_spat_dim = (np.asarray(self.par['spat_order']) + 1).max() max_spec_dim = (np.asarray(self.par['spec_order']) + 1).max() self.coeffs = np.zeros((max_spec_dim, max_spat_dim, self.nslits)) self.spat_order = np.zeros(self.nslits, dtype=int) self.spec_order = np.zeros(self.nslits, dtype=int) # TODO sort out show methods for debugging #if show: # viewer,ch = ginga.show_image(self.msarc*(self.slitmask > -1),chname='tilts') # Loop on all slits for slit in gdslits: msgs.info('Computing tilts for slit {0}/{1}'.format( slit, self.nslits - 1)) # Identify lines for tracing tilts msgs.info('Finding lines for tilt analysis') self.lines_spec, self.lines_spat \ = self.find_lines(self.arccen[:,slit], self.slitcen[:,slit], slit, bpm=self.arccen_bpm[:,slit], debug=False) #debug) if self.lines_spec is None: self.mask[slit] = True maskslits[slit] = True continue thismask = self.slitmask == slit # Performs the initial tracing of the line centroids as a # function of spatial position resulting in 1D traces for # each line. msgs.info('Trace the tilts') self.trace_dict = self.trace_tilts(_msarc, self.lines_spec, self.lines_spat, thismask, self.slitcen[:, slit]) # TODO: Show the traces before running the 2D fit #if show: # ginga.show_tilts(viewer, ch, self.trace_dict) self.spat_order[slit] = self._parse_param(self.par, 'spat_order', slit) self.spec_order[slit] = self._parse_param(self.par, 'spec_order', slit) # 2D model of the tilts, includes construction of QA # NOTE: This also fills in self.all_fit_dict and self.all_trace_dict coeff_out = self.fit_tilts(self.trace_dict, thismask, self.slitcen[:, slit], self.spat_order[slit], self.spec_order[slit], slit, doqa=doqa, show_QA=show, debug=show) self.coeffs[:self.spec_order[slit] + 1, :self.spat_order[slit] + 1, slit] = coeff_out # Tilts are created with the size of the original slitmask, # which corresonds to the same binning as the science # images, trace images, and pixelflats etc. self.tilts = tracewave.fit2tilts(self.slitmask_science.shape, coeff_out, self.par['func2d']) # Save to final image thismask_science = self.slitmask_science == slit self.final_tilts[thismask_science] = self.tilts[thismask_science] if debug: # TODO: Add this to the show method? vmin, vmax = visualization.ZScaleInterval().get_limits(_msarc) plt.imshow(_msarc, origin='lower', interpolation='nearest', aspect='auto', vmin=vmin, vmax=vmax) for slit in gdslits: spat = self.all_trace_dict[slit]['tilts_spat'] spec = self.all_trace_dict[slit]['tilts'] spec_fit = self.all_trace_dict[slit]['tilts_fit'] in_fit = self.all_trace_dict[slit]['tot_mask'] not_fit = np.invert(in_fit) & (spec > 0) fit_rej = in_fit & np.invert( self.all_trace_dict[slit]['fit_mask']) fit_keep = in_fit & self.all_trace_dict[slit]['fit_mask'] plt.scatter(spat[not_fit], spec[not_fit], color='C1', marker='.', s=30, lw=0) plt.scatter(spat[fit_rej], spec[fit_rej], color='C3', marker='.', s=30, lw=0) plt.scatter(spat[fit_keep], spec[fit_keep], color='k', marker='.', s=30, lw=0) with_fit = np.invert(np.all(np.invert(fit_keep), axis=0)) for t in range(in_fit.shape[1]): if not with_fit[t]: continue l, r = np.nonzero(in_fit[:, t])[0][[0, -1]] plt.plot(spat[l:r + 1, t], spec_fit[l:r + 1, t], color='k') plt.show() self.tilts_dict = { 'tilts': self.final_tilts, 'coeffs': self.coeffs, 'slitcen': self.slitcen, 'func2d': self.par['func2d'], 'nslit': self.nslits, 'spat_order': self.spat_order, 'spec_order': self.spec_order } return self.tilts_dict, maskslits
def run(self, doqa=True, debug=False, show=False): """ Main driver for tracing arc lines Code flow: #. Extract an arc spectrum down the center of each slit/order #. Loop on slits/orders #. Trace and fit the arc lines (This is done twice, once with trace_crude as the tracing crutch, then again with a PCA model fit as the crutch). #. Repeat trace. #. 2D Fit to the offset from slitcen #. Save Args: doqa (bool): debug (bool): show (bool): Returns: :class:`WaveTilts`: """ # Extract the arc spectra for all slits self.arccen, self.arccen_bpm = self.extract_arcs() # TODO: Leave for now. Used for debugging # self.par['rm_continuum'] = True # debug = True # show = True # Subtract arc continuum _mstilt = self.mstilt.image.copy() if self.par['rm_continuum']: continuum = self.model_arc_continuum(debug=debug) _mstilt -= continuum if debug: # TODO: Put this into a function vmin, vmax = visualization.ZScaleInterval().get_limits(_mstilt) w,h = plt.figaspect(1) fig = plt.figure(figsize=(3*w,h)) ax = fig.add_axes([0.15/3, 0.1, 0.8/3, 0.8]) ax.imshow(self.mstilt.image, origin='lower', interpolation='nearest', aspect='auto', vmin=vmin, vmax=vmax) ax.set_title('MasterArc') ax = fig.add_axes([1.15/3, 0.1, 0.8/3, 0.8]) ax.imshow(continuum, origin='lower', interpolation='nearest', aspect='auto', vmin=vmin, vmax=vmax) ax.set_title('Continuum') ax = fig.add_axes([2.15/3, 0.1, 0.8/3, 0.8]) ax.imshow(_mstilt, origin='lower', interpolation='nearest', aspect='auto', vmin=vmin, vmax=vmax) ax.set_title('MasterArc - Continuum') plt.show() # Final tilts image self.final_tilts = np.zeros(self.shape_science,dtype=float) max_spat_dim = (np.asarray(self.par['spat_order']) + 1).max() max_spec_dim = (np.asarray(self.par['spec_order']) + 1).max() self.coeffs = np.zeros((max_spec_dim, max_spat_dim,self.slits.nslits)) self.spat_order = np.zeros(self.slits.nslits, dtype=int) self.spec_order = np.zeros(self.slits.nslits, dtype=int) # TODO sort out show methods for debugging if show: viewer,ch = ginga.show_image(self.mstilt.image*(self.slitmask > -1),chname='tilts') # Loop on all slits for slit_idx, slit_spat in enumerate(self.slits.spat_id): if self.tilt_bpm[slit_idx]: continue #msgs.info('Computing tilts for slit {0}/{1}'.format(slit, self.slits.nslits-1)) msgs.info('Computing tilts for slit {0}/{1}'.format(slit_idx, self.slits.nslits)) # Identify lines for tracing tilts msgs.info('Finding lines for tilt analysis') self.lines_spec, self.lines_spat \ = self.find_lines(self.arccen[:,slit_idx], self.slitcen[:,slit_idx], slit_idx, bpm=self.arccen_bpm[:,slit_idx], debug=debug) if self.lines_spec is None: self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADTILTCALIB') continue thismask = self.slitmask == slit_spat # Performs the initial tracing of the line centroids as a # function of spatial position resulting in 1D traces for # each line. msgs.info('Trace the tilts') self.trace_dict = self.trace_tilts(_mstilt, self.lines_spec, self.lines_spat, thismask, self.slitcen[:, slit_idx]) # TODO: Show the traces before running the 2D fit if show: ginga.show_tilts(viewer, ch, self.trace_dict) self.spat_order[slit_idx] = self._parse_param(self.par, 'spat_order', slit_idx) self.spec_order[slit_idx] = self._parse_param(self.par, 'spec_order', slit_idx) # 2D model of the tilts, includes construction of QA # NOTE: This also fills in self.all_fit_dict and self.all_trace_dict coeff_out = self.fit_tilts(self.trace_dict, thismask, self.slitcen[:,slit_idx], self.spat_order[slit_idx], self.spec_order[slit_idx], slit_idx, doqa=doqa, show_QA=show, debug=show) self.coeffs[:self.spec_order[slit_idx]+1,:self.spat_order[slit_idx]+1,slit_idx] = coeff_out # TODO: Need a way to assess the success of fit_tilts and # flag the slit if it fails # Tilts are created with the size of the original slitmask, # which corresonds to the same binning as the science # images, trace images, and pixelflats etc. self.tilts = tracewave.fit2tilts(self.slitmask_science.shape, coeff_out, self.par['func2d']) # Save to final image thismask_science = self.slitmask_science == slit_spat self.final_tilts[thismask_science] = self.tilts[thismask_science] if debug: # TODO: Add this to the show method? vmin, vmax = visualization.ZScaleInterval().get_limits(_mstilt) plt.imshow(_mstilt, origin='lower', interpolation='nearest', aspect='auto', vmin=vmin, vmax=vmax) for slit in self.slit_idx: spat = self.all_trace_dict[slit]['tilts_spat'] spec = self.all_trace_dict[slit]['tilts'] spec_fit = self.all_trace_dict[slit]['tilts_fit'] in_fit = self.all_trace_dict[slit]['tot_mask'] not_fit = np.invert(in_fit) & (spec > 0) fit_rej = in_fit & np.invert(self.all_trace_dict[slit]['fit_mask']) fit_keep = in_fit & self.all_trace_dict[slit]['fit_mask'] plt.scatter(spat[not_fit], spec[not_fit], color='C1', marker='.', s=30, lw=0) plt.scatter(spat[fit_rej], spec[fit_rej], color='C3', marker='.', s=30, lw=0) plt.scatter(spat[fit_keep], spec[fit_keep], color='k', marker='.', s=30, lw=0) with_fit = np.invert(np.all(np.invert(fit_keep), axis=0)) for t in range(in_fit.shape[1]): if not with_fit[t]: continue l, r = np.nonzero(in_fit[:,t])[0][[0,-1]] plt.plot(spat[l:r+1,t], spec_fit[l:r+1,t], color='k') plt.show() # Record the Mask bpmtilts = np.zeros_like(self.slits.mask, dtype=self.slits.bitmask.minimum_dtype()) for flag in ['BADTILTCALIB']: bpm = self.slits.bitmask.flagged(self.slits.mask, flag) if np.any(bpm): bpmtilts[bpm] = self.slits.bitmask.turn_on(bpmtilts[bpm], flag) # Build and return DataContainer tilts_dict = {'coeffs':self.coeffs, 'func2d':self.par['func2d'], 'nslit':self.slits.nslits, 'spat_order':self.spat_order, 'spec_order':self.spec_order, 'spat_id':self.slits.spat_id, 'bpmtilts': bpmtilts, 'spat_flexure': self.spat_flexure, 'PYP_SPEC': self.spectrograph.spectrograph} return WaveTilts(**tilts_dict)
def fit(self, debug=False): """ Construct a model of the flat-field image. For this method to work, :attr:`rawflatimg` must have been previously constructed; see :func:`build_pixflat`. The method loops through all slits provided by the :attr:`slits` object, except those that have been masked (i.e., slits with ``self.slits.mask == True`` are skipped). For each slit: - Collapse the flat-field data spatially using the wavelength coordinates provided by the fit to the arc-line traces (:class:`pypeit.wavetilts.WaveTilts`), and fit the result with a bspline. This provides the spatially-averaged spectral response of the instrument. The data used in the fit is trimmed toward the slit spatial center via the ``slit_trim`` parameter in :attr:`flatpar`. - Use the bspline fit to construct and normalize out the spectral response. - Collapse the normalized flat-field data spatially using a coordinate system defined by the left slit edge. The data included in the spatial (illumination) profile calculation is expanded beyond the nominal slit edges using the ``slit_illum_pad`` parameter in :attr:`flatpar`. The raw, collapsed data is then median filtered (see ``spat_samp`` in :attr:`flatpar`) and Gaussian filtered; see :func:`pypeit.core.flat.illum_filter`. This creates an empirical, highly smoothed representation of the illumination profile that is fit with a bspline using the :func:`spatial_fit` method. The construction of the empirical illumination profile (i.e., before the bspline fitting) can be done iteratively, where each iteration sigma-clips outliers; see the ``illum_iter`` and ``illum_rej`` parameters in :attr:`flatpar` and :func:`pypeit.core.flat.construct_illum_profile`. - If requested, the 1D illumination profile is used to "tweak" the slit edges by offsetting them to a threshold of the illumination peak to either side of the slit center (see ``tweak_slits_thresh`` in :attr:`flatpar`), up to a maximum allowed shift from the existing slit edge (see ``tweak_slits_maxfrac`` in :attr:`flatpar`). See :func:`pypeit.core.tweak_slit_edges`. If tweaked, the :func:`spatial_fit` is repeated to place it on the tweaked slits reference frame. - Use the bspline fit to construct the 2D illumination image (:attr:`msillumflat`) and normalize out the spatial response. - Fit the residuals of the flat-field data that has been independently normalized for its spectral and spatial response with a 2D bspline-polynomial fit. The order of the polynomial has been optimized via experimentation; it can be changed but you should use extreme caution when doing so (see ``twod_fit_npoly``). The multiplication of the 2D spectral response, 2D spatial response, and joint 2D fit to the high-order residuals define the final flat model (:attr:`flat_model`). - Finally, the pixel-to-pixel response of the instrument is defined as the ratio of the raw flat data to the best-fitting flat-field model (:attr:`mspixelflat`) This method is the primary method that builds the :class:`FlatField` instance, constructing :attr:`mspixelflat`, :attr:`msillumflat`, and :attr:`flat_model`. All of these attributes are altered internally. If the slit edges are to be tweaked using the 1D illumination profile (``tweak_slits`` in :attr:`flatpar`), the tweaked slit edge arrays in the internal :class:`pypeit.edgetrace.SlitTraceSet` object, :attr:`slits`, are also altered. Used parameters from :attr:`flatpar` (:class:`pypeit.par.pypeitpar.FlatFieldPar`) are ``spec_samp_fine``, ``spec_samp_coarse``, ``spat_samp``, ``tweak_slits``, ``tweak_slits_thresh``, ``tweak_slits_maxfrac``, ``rej_sticky``, ``slit_trim``, ``slit_illum_pad``, ``illum_iter``, ``illum_rej``, and ``twod_fit_npoly``, ``saturated_slits``. **Revision History**: - 11-Mar-2005 First version written by Scott Burles. - 2005-2018 Improved by J. F. Hennawi and J. X. Prochaska - 3-Sep-2018 Ported to python by J. F. Hennawi and significantly improved Args: debug (:obj:`bool`, optional): Show plots useful for debugging. This will block further execution of the code until the plot windows are closed. """ # TODO: break up this function! Can it be partitioned into a series of "core" methods? # TODO: JFH I wrote all this code and will have to maintain it and I don't want to see it broken up. # TODO: JXP This definitely needs breaking up.. # Init self.list_of_spat_bsplines = [] # Set parameters (for convenience; spec_samp_fine = self.flatpar['spec_samp_fine'] spec_samp_coarse = self.flatpar['spec_samp_coarse'] tweak_slits = self.flatpar['tweak_slits'] tweak_slits_thresh = self.flatpar['tweak_slits_thresh'] tweak_slits_maxfrac = self.flatpar['tweak_slits_maxfrac'] # If sticky, points rejected at each stage (spec, spat, 2d) are # propagated to the next stage sticky = self.flatpar['rej_sticky'] trim = self.flatpar['slit_trim'] pad = self.flatpar['slit_illum_pad'] # Iteratively construct the illumination profile by rejecting outliers npoly = self.flatpar['twod_fit_npoly'] saturated_slits = self.flatpar['saturated_slits'] # Setup images nspec, nspat = self.rawflatimg.image.shape rawflat = self.rawflatimg.image # Good pixel mask gpm = np.ones_like(rawflat, dtype=bool) if self.rawflatimg.bpm is None else ( 1-self.rawflatimg.bpm).astype(bool) # Flat-field modeling is done in the log of the counts flat_log = np.log(np.fmax(rawflat, 1.0)) gpm_log = (rawflat > 1.0) & gpm # set errors to just be 0.5 in the log ivar_log = gpm_log.astype(float)/0.5**2 # Other setup nonlinear_counts = self.spectrograph.nonlinear_counts(self.rawflatimg.detector) # TODO -- JFH -- CONFIRM THIS SHOULD BE ON INIT # It does need to be *all* of the slits median_slit_widths = np.median(self.slits.right_init - self.slits.left_init, axis=0) if tweak_slits: # NOTE: This copies the input slit edges to a set that can be tweaked. self.slits.init_tweaked() # TODO: This needs to include a padding check # Construct three versions of the slit ID image, all of unmasked slits! # - an image that uses the padding defined by self.slits slitid_img_init = self.slits.slit_img(initial=True) # - an image that uses the extra padding defined by # self.flatpar. This was always 5 pixels in the previous # version. padded_slitid_img = self.slits.slit_img(initial=True, pad=pad) # - and an image that trims the width of the slit using the # parameter in self.flatpar. This was always 3 pixels in # the previous version. # TODO: Fix this for when trim is a tuple trimmed_slitid_img = self.slits.slit_img(pad=-trim, initial=True) # Prep for results self.mspixelflat = np.ones_like(rawflat) self.msillumflat = np.ones_like(rawflat) self.flat_model = np.zeros_like(rawflat) # Allocate work arrays only once spec_model = np.ones_like(rawflat) norm_spec = np.ones_like(rawflat) norm_spec_spat = np.ones_like(rawflat) twod_model = np.ones_like(rawflat) # ################################################# # Model each slit independently for slit_idx, slit_spat in enumerate(self.slits.spat_id): # Is this a good slit?? if self.slits.mask[slit_idx] != 0: msgs.info('Skipping bad slit: {}'.format(slit_spat)) self.list_of_spat_bsplines.append(bspline.bspline(None)) continue msgs.info('Modeling the flat-field response for slit spat_id={}: {}/{}'.format( slit_spat, slit_idx+1, self.slits.nslits)) # Find the pixels on the initial slit onslit_init = slitid_img_init == slit_spat # Check for saturation of the flat. If there are not enough # pixels do not attempt a fit, and continue to the next # slit. # TODO: set the threshold to a parameter? good_frac = np.sum(onslit_init & (rawflat < nonlinear_counts))/np.sum(onslit_init) if good_frac < 0.5: common_message = 'To change the behavior, use the \'saturated_slits\' parameter ' \ 'in the \'flatfield\' parameter group; see here:\n\n' \ 'https://pypeit.readthedocs.io/en/latest/pypeit_par.html \n\n' \ 'You could also choose to use a different flat-field image ' \ 'for this calibration group.' if saturated_slits == 'crash': msgs.error('Only {:4.2f}'.format(100*good_frac) + '% of the pixels on slit {0} are not saturated. '.format(slit_spat) + 'Selected behavior was to crash if this occurred. ' + common_message) elif saturated_slits == 'mask': self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADFLATCALIB') msgs.warn('Only {:4.2f}'.format(100*good_frac) + '% of the pixels on slit {0} are not saturated. '.format(slit_spat) + 'Selected behavior was to mask this slit and continue with the ' + 'remainder of the reduction, meaning no science data will be ' + 'extracted from this slit. ' + common_message) elif saturated_slits == 'continue': self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'SKIPFLATCALIB') msgs.warn('Only {:4.2f}'.format(100*good_frac) + '% of the pixels on slit {0} are not saturated. '.format(slit_spat) + 'Selected behavior was to simply continue, meaning no ' + 'field-flatting correction will be applied to this slit but ' + 'pypeit will attempt to extract any objects found on this slit. ' + common_message) else: # Should never get here raise NotImplementedError('Unknown behavior for saturated slits: {0}'.format( saturated_slits)) self.list_of_spat_bsplines.append(bspline.bspline(None)) continue # Demand at least 10 pixels per row (on average) per degree # of the polynomial. # NOTE: This is not used until the 2D fit. Defined here to # be close to the definition of ``onslit``. if npoly is None: # Approximate number of pixels sampling each spatial pixel # for this (original) slit. npercol = np.fmax(np.floor(np.sum(onslit_init)/nspec),1.0) npoly = np.clip(7, 1, int(np.ceil(npercol/10.))) # TODO: Always calculate the optimized `npoly` and warn the # user if npoly is provided but higher than the nominal # calculation? # Create an image with the spatial coordinates relative to the left edge of this slit spat_coo_init = self.slits.spatial_coordinate_image(slitidx=slit_idx, full=True, initial=True) # Find pixels on the padded and trimmed slit coordinates onslit_padded = padded_slitid_img == slit_spat onslit_trimmed = trimmed_slitid_img == slit_spat # ---------------------------------------------------------- # Collapse the slit spatially and fit the spectral function # TODO: Put this stuff in a self.spectral_fit method? # Create the tilts image for this slit # TODO -- JFH Confirm the sign of this shift is correct! _flexure = 0. if self.wavetilts.spat_flexure is None else self.wavetilts.spat_flexure tilts = tracewave.fit2tilts(rawflat.shape, self.wavetilts['coeffs'][:,:,slit_idx], self.wavetilts['func2d'], spat_shift=-1*_flexure) # Convert the tilt image to an image with the spectral pixel index spec_coo = tilts * (nspec-1) # Only include the trimmed set of pixels in the flat-field # fit along the spectral direction. spec_gpm = onslit_trimmed & gpm_log # & (rawflat < nonlinear_counts) spec_nfit = np.sum(spec_gpm) spec_ntot = np.sum(onslit_init) msgs.info('Spectral fit of flatfield for {0}/{1} '.format(spec_nfit, spec_ntot) + ' pixels in the slit.') # Set this to a parameter? if spec_nfit/spec_ntot < 0.5: # TODO: Shouldn't this raise an exception or continue to the next slit instead? msgs.warn('Spectral fit includes only {:.1f}'.format(100*spec_nfit/spec_ntot) + '% of the pixels on this slit.' + msgs.newline() + ' Either the slit has many bad pixels or the number of ' 'trimmed pixels is too large.') # Sort the pixels by their spectral coordinate. # TODO: Include ivar and sorted gpm in outputs? spec_gpm, spec_srt, spec_coo_data, spec_flat_data \ = flat.sorted_flat_data(flat_log, spec_coo, gpm=spec_gpm) # NOTE: By default np.argsort sorts the data over the last # axis. Just to avoid the possibility (however unlikely) of # spec_coo[spec_gpm] returning an array, all the arrays are # explicitly flattened. spec_ivar_data = ivar_log[spec_gpm].ravel()[spec_srt] spec_gpm_data = gpm_log[spec_gpm].ravel()[spec_srt] # Rejection threshold for spectral fit in log(image) # TODO: Make this a parameter? logrej = 0.5 # Fit the spectral direction of the blaze. # TODO: Figure out how to deal with the fits going crazy at # the edges of the chip in spec direction # TODO: Can we add defaults to bspline_profile so that we # don't have to instantiate invvar and profile_basis spec_bspl, spec_gpm_fit, spec_flat_fit, _, exit_status \ = utils.bspline_profile(spec_coo_data, spec_flat_data, spec_ivar_data, np.ones_like(spec_coo_data), ingpm=spec_gpm_data, nord=4, upper=logrej, lower=logrej, kwargs_bspline={'bkspace': spec_samp_fine}, kwargs_reject={'groupbadpix': True, 'maxrej': 5}) if exit_status > 1: # TODO -- MAKE A FUNCTION msgs.warn('Flat-field spectral response bspline fit failed! Not flat-fielding ' 'slit {0} and continuing!'.format(slit_spat)) self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADFLATCALIB') self.list_of_spat_bsplines.append(bspline.bspline(None)) continue # Debugging/checking spectral fit if debug: utils.bspline_qa(spec_coo_data, spec_flat_data, spec_bspl, spec_gpm_fit, spec_flat_fit, xlabel='Spectral Pixel', ylabel='log(flat counts)', title='Spectral Fit for slit={:d}'.format(slit_spat)) if sticky: # Add rejected pixels to gpm gpm[spec_gpm] = (spec_gpm_fit & spec_gpm_data)[np.argsort(spec_srt)] # Construct the model of the flat-field spectral shape # including padding on either side of the slit. spec_model[...] = 1. spec_model[onslit_padded] = np.exp(spec_bspl.value(spec_coo[onslit_padded])[0]) # ---------------------------------------------------------- # ---------------------------------------------------------- # To fit the spatial response, first normalize out the # spectral response, and then collapse the slit spectrally. # Normalize out the spectral shape of the flat norm_spec[...] = 1. norm_spec[onslit_padded] = rawflat[onslit_padded] \ / np.fmax(spec_model[onslit_padded],1.0) # Find pixels fot fit in the spatial direction: # - Fit pixels in the padded slit that haven't been masked # by the BPM spat_gpm = onslit_padded & gpm #& (rawflat < nonlinear_counts) # - Fit pixels with non-zero flux and less than 70% above # the average spectral profile. spat_gpm &= (norm_spec > 0.0) & (norm_spec < 1.7) # - Determine maximum counts in median filtered flat # spectrum model. spec_interp = interpolate.interp1d(spec_coo_data, spec_flat_fit, kind='linear', assume_sorted=True, bounds_error=False, fill_value=-np.inf) spec_sm = utils.fast_running_median(np.exp(spec_interp(np.arange(nspec))), np.fmax(np.ceil(0.10*nspec).astype(int),10)) # - Only fit pixels with at least values > 10% of this maximum and no less than 1. spat_gpm &= (spec_model > 0.1*np.amax(spec_sm)) & (spec_model > 1.0) # Report spat_nfit = np.sum(spat_gpm) spat_ntot = np.sum(onslit_padded) msgs.info('Spatial fit of flatfield for {0}/{1} '.format(spat_nfit, spat_ntot) + ' pixels in the slit.') if spat_nfit/spat_ntot < 0.5: # TODO: Shouldn't this raise an exception or continue to the next slit instead? msgs.warn('Spatial fit includes only {:.1f}'.format(100*spat_nfit/spat_ntot) + '% of the pixels on this slit.' + msgs.newline() + ' Either the slit has many bad pixels, the model of the ' 'spectral shape is poor, or the illumination profile is very irregular.') # First fit -- With initial slits exit_status, spat_coo_data, spat_flat_data, spat_bspl, spat_gpm_fit, \ spat_flat_fit, spat_flat_data_raw \ = self.spatial_fit(norm_spec, spat_coo_init, median_slit_widths[slit_idx], spat_gpm, gpm, debug=debug) if tweak_slits: # TODO: Should the tweak be based on the bspline fit? # TODO: Will this break if left_thresh, left_shift, self.slits.left_tweak[:,slit_idx], right_thresh, \ right_shift, self.slits.right_tweak[:,slit_idx] \ = flat.tweak_slit_edges(self.slits.left_init[:,slit_idx], self.slits.right_init[:,slit_idx], spat_coo_data, spat_flat_data, thresh=tweak_slits_thresh, maxfrac=tweak_slits_maxfrac, debug=debug) # TODO: Because the padding doesn't consider adjacent # slits, calling slit_img for individual slits can be # different from the result when you construct the # image for all slits. Fix this... # Update the onslit mask _slitid_img = self.slits.slit_img(slitidx=slit_idx, initial=False) onslit_tweak = _slitid_img == slit_spat spat_coo_tweak = self.slits.spatial_coordinate_image(slitidx=slit_idx, slitid_img=_slitid_img) # Construct the empirical illumination profile # TODO This is extremely inefficient, because we only need to re-fit the illumflat, but # spatial_fit does both the reconstruction of the illumination function and the bspline fitting. # Only the b-spline fitting needs be reddone with the new tweaked spatial coordinates, so that would # save a ton of runtime. It is not a trivial change becauase the coords are sorted, etc. exit_status, spat_coo_data, spat_flat_data, spat_bspl, spat_gpm_fit, \ spat_flat_fit, spat_flat_data_raw = self.spatial_fit( norm_spec, spat_coo_tweak, median_slit_widths[slit_idx], spat_gpm, gpm, debug=False) spat_coo_final = spat_coo_tweak else: _slitid_img = slitid_img_init spat_coo_final = spat_coo_init onslit_tweak = onslit_init # Add an approximate pixel axis at the top if debug: # TODO: Move this into a qa plot that gets saved ax = utils.bspline_qa(spat_coo_data, spat_flat_data, spat_bspl, spat_gpm_fit, spat_flat_fit, show=False) ax.scatter(spat_coo_data, spat_flat_data_raw, marker='.', s=1, zorder=0, color='k', label='raw data') # Force the center of the slit to be at the center of the plot for the hline ax.set_xlim(-0.1,1.1) ax.axvline(0.0, color='lightgreen', linestyle=':', linewidth=2.0, label='original left edge', zorder=8) ax.axvline(1.0, color='red', linestyle=':', linewidth=2.0, label='original right edge', zorder=8) if tweak_slits and left_shift > 0: label = 'threshold = {:5.2f}'.format(tweak_slits_thresh) \ + ' % of max of left illumprofile' ax.axhline(left_thresh, xmax=0.5, color='lightgreen', linewidth=3.0, label=label, zorder=10) ax.axvline(left_shift, color='lightgreen', linestyle='--', linewidth=3.0, label='tweaked left edge', zorder=11) if tweak_slits and right_shift > 0: label = 'threshold = {:5.2f}'.format(tweak_slits_thresh) \ + ' % of max of right illumprofile' ax.axhline(right_thresh, xmin=0.5, color='red', linewidth=3.0, label=label, zorder=10) ax.axvline(1-right_shift, color='red', linestyle='--', linewidth=3.0, label='tweaked right edge', zorder=20) ax.legend() ax.set_xlabel('Normalized Slit Position') ax.set_ylabel('Normflat Spatial Profile') ax.set_title('Illumination Function Fit for slit={:d}'.format(slit_spat)) plt.show() # ---------------------------------------------------------- # Construct the illumination profile with the tweaked edges # of the slit if exit_status <= 1: # TODO -- JFH -- Check this is ok for flexure!! self.msillumflat[onslit_tweak] = spat_bspl.value(spat_coo_final[onslit_tweak])[0] self.list_of_spat_bsplines.append(spat_bspl) else: # Save the nada msgs.warn('Slit illumination profile bspline fit failed! Spatial profile not ' 'included in flat-field model for slit {0}!'.format(slit_spat)) self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADFLATCALIB') self.list_of_spat_bsplines.append(bspline.bspline(None)) continue # ---------------------------------------------------------- # Fit the 2D residuals of the 1D spectral and spatial fits. msgs.info('Performing 2D illumination + scattered light flat field fit') # Construct the spectrally and spatially normalized flat norm_spec_spat[...] = 1. norm_spec_spat[onslit_tweak] = rawflat[onslit_tweak] / np.fmax(spec_model[onslit_tweak], 1.0) \ / np.fmax(self.msillumflat[onslit_tweak], 0.01) # Sort the pixels by their spectral coordinate. The mask # uses the nominal padding defined by the slits object. twod_gpm, twod_srt, twod_spec_coo_data, twod_flat_data \ = flat.sorted_flat_data(norm_spec_spat, spec_coo, gpm=onslit_tweak) # Also apply the sorting to the spatial coordinates twod_spat_coo_data = spat_coo_final[twod_gpm].ravel()[twod_srt] # TODO: Reset back to origin gpm if sticky is true? twod_gpm_data = gpm[twod_gpm].ravel()[twod_srt] # Only fit data with less than 30% variations # TODO: Make 30% a parameter? twod_gpm_data &= np.absolute(twod_flat_data - 1) < 0.3 # Here we ignore the formal photon counting errors and # simply assume that a typical error per pixel. This guess # is somewhat aribtrary. We then set the rejection # threshold with sigrej_twod # TODO: Make twod_sig and twod_sigrej parameters? twod_sig = 0.01 twod_ivar_data = twod_gpm_data.astype(float)/(twod_sig**2) twod_sigrej = 4.0 poly_basis = basis.fpoly(2.0*twod_spat_coo_data - 1.0, npoly) # Perform the full 2d fit twod_bspl, twod_gpm_fit, twod_flat_fit, _ , exit_status \ = utils.bspline_profile(twod_spec_coo_data, twod_flat_data, twod_ivar_data, poly_basis, ingpm=twod_gpm_data, nord=4, upper=twod_sigrej, lower=twod_sigrej, kwargs_bspline={'bkspace': spec_samp_coarse}, kwargs_reject={'groupbadpix': True, 'maxrej': 10}) if debug: # TODO: Make a plot that shows the residuals in the 2D # image resid = twod_flat_data - twod_flat_fit goodpix = twod_gpm_fit & twod_gpm_data badpix = np.invert(twod_gpm_fit) & twod_gpm_data plt.clf() ax = plt.gca() ax.plot(twod_spec_coo_data[goodpix], resid[goodpix], color='k', marker='o', markersize=0.2, mfc='k', fillstyle='full', linestyle='None', label='good points') ax.plot(twod_spec_coo_data[badpix], resid[badpix], color='red', marker='+', markersize=0.5, mfc='red', fillstyle='full', linestyle='None', label='masked') ax.axhline(twod_sigrej*twod_sig, color='lawngreen', linestyle='--', label='rejection thresholds', zorder=10, linewidth=2.0) ax.axhline(-twod_sigrej*twod_sig, color='lawngreen', linestyle='--', zorder=10, linewidth=2.0) # ax.set_ylim(-0.05, 0.05) ax.legend() ax.set_xlabel('Spectral Pixel') ax.set_ylabel('Residuals from pixelflat 2-d fit') ax.set_title('Spectral Residuals for slit={:d}'.format(slit_spat)) plt.show() plt.clf() ax = plt.gca() ax.plot(twod_spat_coo_data[goodpix], resid[goodpix], color='k', marker='o', markersize=0.2, mfc='k', fillstyle='full', linestyle='None', label='good points') ax.plot(twod_spat_coo_data[badpix], resid[badpix], color='red', marker='+', markersize=0.5, mfc='red', fillstyle='full', linestyle='None', label='masked') ax.axhline(twod_sigrej*twod_sig, color='lawngreen', linestyle='--', label='rejection thresholds', zorder=10, linewidth=2.0) ax.axhline(-twod_sigrej*twod_sig, color='lawngreen', linestyle='--', zorder=10, linewidth=2.0) # ax.set_ylim((-0.05, 0.05)) # ax.set_xlim(-0.02, 1.02) ax.legend() ax.set_xlabel('Normalized Slit Position') ax.set_ylabel('Residuals from pixelflat 2-d fit') ax.set_title('Spatial Residuals for slit={:d}'.format(slit_spat)) plt.show() # Save the 2D residual model twod_model[...] = 1. if exit_status > 1: msgs.warn('Two-dimensional fit to flat-field data failed! No higher order ' 'flat-field corrections included in model of slit {0}!'.format(slit_spat)) else: twod_model[twod_gpm] = twod_flat_fit[np.argsort(twod_srt)] # Construct the full flat-field model # TODO: Why is the 0.05 here for the illumflat compared to the 0.01 above? self.flat_model[onslit_tweak] = twod_model[onslit_tweak] \ * np.fmax(self.msillumflat[onslit_tweak], 0.05) \ * np.fmax(spec_model[onslit_tweak], 1.0) # Construct the pixel flat #self.mspixelflat[onslit] = rawflat[onslit]/self.flat_model[onslit] #self.mspixelflat[onslit_tweak] = 1. #trimmed_slitid_img_anew = self.slits.slit_img(pad=-trim, slitidx=slit_idx) #onslit_trimmed_anew = trimmed_slitid_img_anew == slit_spat self.mspixelflat[onslit_tweak] = rawflat[onslit_tweak]/self.flat_model[onslit_tweak] # TODO: Add some code here to treat the edges and places where fits # go bad? # Set the pixelflat to 1.0 wherever the flat was nonlinear self.mspixelflat[rawflat >= nonlinear_counts] = 1.0 # Set the pixelflat to 1.0 within trim pixels of all the slit edges trimmed_slitid_img_new = self.slits.slit_img(pad=-trim, initial=False) tweaked_slitid_img = self.slits.slit_img(initial=False) self.mspixelflat[(trimmed_slitid_img_new < 0) & (tweaked_slitid_img > 0)] = 1.0 # Do not apply pixelflat field corrections that are greater than # 100% to avoid creating edge effects, etc. self.mspixelflat = np.clip(self.mspixelflat, 0.5, 2.0)