def test_init(): # Instantiate a simple pypeitImage pypeitImage = pypeitimage.PypeItImage(np.ones((1000, 1000))) pypeitImage.fullmask = np.zeros((1000, 1000), dtype=np.int64) pypeitImage.detector = test_detector.detector_container.DetectorContainer(**test_detector.def_det) # Now the arcimage arcImage = buildimage.ArcImage.from_pypeitimage(pypeitImage)
def test_image(): # Just for a dummy image one_file = data_path('b1.fits.gz') data = fits.open(one_file)[0].data.astype(float) # pypeitImage = pypeitimage.PypeItImage(data) # assert pypeitImage.image.shape == (350, 2112)
def test_write(): # Just for a dummy image data = np.ones((1000, 1000)) ivar = np.ones_like(data) mask = np.ones_like(data).astype(int) # pypeitImage = pypeitimage.PypeItImage(data, ivar=ivar, mask=mask) # outfile = data_path('tst.fits') pypeitImage.write(outfile) # Test hdul = fits.open(outfile) assert len(hdul) == 4 assert hdul[2].name == 'IVAR'
def test_master_io(): # Instantiate a simple pypeitImage pypeitImage = pypeitimage.PypeItImage(np.ones((1000, 1000))) pypeitImage.fullmask = np.zeros((1000, 1000), dtype=np.int64) pypeitImage.detector = test_detector.detector_container.DetectorContainer(**test_detector.def_det) pypeitImage.PYP_SPEC = 'shane_kast_blue' # Now the arcimage arcImage = buildimage.ArcImage.from_pypeitimage(pypeitImage) # Write master_filename = masterframe.construct_file_name(arcImage, 'A_01_22', master_dir=data_path('')) arcImage.to_master_file(master_filename) # Read _arcImage = buildimage.ArcImage.from_file(data_path('MasterArc_A_01_22.fits')) assert isinstance(_arcImage.detector, test_detector.detector_container.DetectorContainer)
def test_run(): # Masters spectrograph = load_spectrograph('shane_kast_blue') edges, tilts_dict = load_kast_blue_masters(edges=True, tilts=True) # Instantiate frametype = 'pixelflat' par = pypeitpar.FrameGroupPar(frametype) flatField = flatfield.FlatField(spectrograph, par, det=1, tilts_dict=tilts_dict, tslits_dict=edges.convert_to_tslits_dict()) # Use the trace image flatField.rawflatimg = pypeitimage.PypeItImage(edges.img.copy()) mspixelflatnrm, msillumflat = flatField.run() assert np.isclose(np.median(mspixelflatnrm), 1.0)
def test_full(): pypeitImage = pypeitimage.PypeItImage(np.ones((1000, 1000))) # Mask pypeitImage.fullmask = np.zeros((1000, 1000), dtype=np.int64) # Full datamodel full_datamodel = pypeitImage.full_datamodel() assert 'gain' in full_datamodel.keys() assert 'detector' in pypeitImage.keys() # I/O outfile = data_path('tst_pypeitimage.fits') pypeitImage.to_file(outfile, overwrite=True) _pypeitImage = pypeitimage.PypeItImage.from_file(outfile) # Test assert isinstance(_pypeitImage.image, np.ndarray) assert _pypeitImage.ivar is None
def test_full(): pypeitImage = pypeitimage.PypeItImage(np.ones((1000, 1000))) pypeitImage.reinit_mask() # Full datamodel # full_datamodel = pypeitImage.full_datamodel() # assert 'gain' in full_datamodel.keys() assert 'detector' in pypeitImage.keys() # I/O outfile = data_path('tst_pypeitimage.fits') pypeitImage.to_file(outfile, overwrite=True) _pypeitImage = pypeitimage.PypeItImage.from_file(outfile) # Cleanup os.remove(outfile) # Test assert isinstance(_pypeitImage.image, np.ndarray) assert _pypeitImage.ivar is None
def from_master_file(cls, master_file, par=None): """ Instantiate the class from a master file Args: master_file (str): par (:class:`pypeit.par.pypeitpar.PypeItPar`, optional): Full par set Returns: :class:`pypeit.flatfield.FlatField`: With the flat images loaded up """ # Spectrograph spectrograph, extras = masterframe.items_from_master_file(master_file) head0 = extras[0] # Par if par is None: par = spectrograph.default_pypeit_par() # Master info master_dir = head0['MSTRDIR'] master_key = head0['MSTRKEY'] # Instantiate slf = cls(spectrograph, par['calibrations']['pixelflatframe'], master_dir=master_dir, master_key=master_key, reuse_masters=True) # Load rawflatimg, slf.mspixelflat, slf.msillumflat = slf.load( ifile=master_file) # Convert rawflatimg to a PypeItImage slf.rawflatimg = pypeitimage.PypeItImage(rawflatimg) # Return return slf
def process(self, par, bpm=bpm, flatimages=None, bias=None, slits=None, debug=False, dark=None): """ Process the image Note: The processing step order is currently 'frozen' as is. We may choose to allow optional ordering Here are the allowed steps, in the order they will be applied: subtract_overscan -- Analyze the overscan region and subtract from the image trim -- Trim the image down to the data (i.e. remove the overscan) orient -- Orient the image in the PypeIt orientation (spec, spat) with blue to red going down to up subtract_bias -- Subtract a bias image apply_gain -- Convert to counts, amp by amp flatten -- Divide by the pixel flat and (if provided) the illumination flat extras -- Generate the RN2 and IVAR images crmask -- Generate a CR mask Args: par (:class:`pypeit.par.pypeitpar.ProcessImagesPar`): Parameters that dictate the processing of the images. See :class:`pypeit.par.pypeitpar.ProcessImagesPar` for the defaults. bpm (`numpy.ndarray`_, optional): flatimages (:class:`pypeit.flatfield.FlatImages`): bias (`numpy.ndarray`_, optional): Bias image slits (:class:`pypeit.slittrace.SlitTraceSet`, optional): Used to calculate spatial flexure between the image and the slits Returns: :class:`pypeit.images.pypeitimage.PypeItImage`: """ self.par = par self._bpm = bpm # Get started # Standard order # -- May need to allow for other order some day.. if par['use_pattern']: # Note, this step *must* be done before use_overscan self.subtract_pattern() if par['use_overscan']: self.subtract_overscan() if par['trim']: self.trim() if par['orient']: self.orient() if par['use_biasimage']: # Bias frame, if it exists, is *not* trimmed nor oriented self.subtract_bias(bias) if par['use_darkimage']: # Dark frame, if it exists, is TODO:: check: trimmed, oriented (and oscan/bias subtracted?) self.subtract_dark(dark) if par['apply_gain']: self.apply_gain() # This needs to come after trim, orient # Calculate flexure -- May not be used, but always calculated when slits are provided if slits is not None and self.par['spat_flexure_correct']: self.spat_flexure_shift = flexure.spat_flexure_shift( self.image, slits) # Generate the illumination flat, as needed illum_flat = None if self.par['use_illumflat']: if flatimages is None: msgs.error( "Cannot illumflatten, no such image generated. Add one or more illumflat images to your PypeIt file!!" ) if slits is None: msgs.error("Need to provide slits to create illumination flat") illum_flat = flatimages.fit2illumflat( slits, flexure_shift=self.spat_flexure_shift) if debug: left, right = slits.select_edges( flexure=self.spat_flexure_shift) viewer, ch = display.show_image(illum_flat, chname='illum_flat') display.show_slits(viewer, ch, left, right) # , slits.id) # orig_image = self.image.copy() viewer, ch = display.show_image(orig_image, chname='orig_image') display.show_slits(viewer, ch, left, right) # , slits.id) # Apply the relative spectral illumination spec_illum = 1.0 if self.par['use_specillum']: if flatimages is None or flatimages.get_spec_illum() is None: msgs.error( "Spectral illumination correction desired but not generated/provided." ) else: spec_illum = flatimages.get_spec_illum().copy() # Flat field -- We cannot do illumination flat without a pixel flat (yet) if self.par['use_pixelflat'] or self.par['use_illumflat']: if flatimages is None or flatimages.get_pixelflat() is None: msgs.error("Flat fielding desired but not generated/provided.") else: self.flatten(flatimages.get_pixelflat() * spec_illum, illum_flat=illum_flat, bpm=self.bpm) # Fresh BPM bpm = self.spectrograph.bpm(self.filename, self.det, shape=self.image.shape) # Extras self.build_rn2img() self.build_ivar() # Generate a PypeItImage pypeitImage = pypeitimage.PypeItImage( self.image, ivar=self.ivar, rn2img=self.rn2img, bpm=bpm, detector=self.detector, spat_flexure=self.spat_flexure_shift, PYP_SPEC=self.spectrograph.spectrograph) pypeitImage.rawheadlist = self.headarr pypeitImage.process_steps = [ key for key in self.steps.keys() if self.steps[key] ] # Mask(s) if par['mask_cr']: pypeitImage.build_crmask(self.par) # nonlinear_counts = self.spectrograph.nonlinear_counts( self.detector, apply_gain=self.par['apply_gain']) # Build pypeitImage.build_mask(saturation=nonlinear_counts) # Return return pypeitImage
def process(self, process_steps, pixel_flat=None, illum_flat=None, bias=None): """ Process the image Note: The processing step order is currently 'frozen' as is. We may choose to allow optional ordering Here are the allowed steps, in the order they will be applied: subtract_overscan -- Analyze the overscan region and subtract from the image trim -- Trim the image down to the data (i.e. remove the overscan) orient -- Orient the image in the PypeIt orientation (spec, spat) with blue to red going down to up subtract_bias -- Subtract a bias image apply_gain -- Convert to counts, amp by amp flatten -- Divide by the pixel flat and (if provided) the illumination flat extras -- Generate the RN2 and IVAR images crmask -- Generate a CR mask Args: process_steps (list): List of processing steps pixel_flat (np.ndarray, optional): Pixel flat image illum_flat (np.ndarray, optional): Illumination flat bias (np.ndarray, optional): Bias image bpm (np.ndarray, optional): Bad pixel mask image Returns: :class:`pypeit.images.pypeitimage.PypeItImage`: """ # For error checking steps_copy = process_steps.copy() # Get started # Standard order # -- May need to allow for other order some day.. if 'subtract_overscan' in process_steps: self.subtract_overscan() steps_copy.remove('subtract_overscan') if 'trim' in process_steps: self.trim() steps_copy.remove('trim') if 'orient' in process_steps: self.orient() steps_copy.remove('orient') if 'subtract_bias' in process_steps: # Bias frame, if it exists, is trimmed and oriented self.subtract_bias(bias) steps_copy.remove('subtract_bias') if 'apply_gain' in process_steps: self.apply_gain() steps_copy.remove('apply_gain') # Flat field if 'flatten' in process_steps: self.flatten(pixel_flat, illum_flat=illum_flat, bpm=self.bpm) steps_copy.remove('flatten') # Fresh BPM bpm = self.spectrograph.bpm(self.filename, self.det, shape=self.image.shape) # Extras if 'extras' in process_steps: self.build_rn2img() self.build_ivar() steps_copy.remove('extras') # Generate a PypeItImage pypeitImage = pypeitimage.PypeItImage(self.image, binning=self.binning, ivar=self.ivar, rn2img=self.rn2img, bpm=bpm) # Mask(s) if 'crmask' in process_steps: if 'extras' in process_steps: var = utils.inverse(pypeitImage.ivar) else: var = np.ones_like(pypeitImage.image) # pypeitImage.build_crmask(self.spectrograph, self.det, self.par, pypeitImage.image, var) steps_copy.remove('crmask') nonlinear_counts = self.spectrograph.nonlinear_counts( self.det, apply_gain='apply_gain' in process_steps) pypeitImage.build_mask( pypeitImage.image, pypeitImage.ivar, saturation= nonlinear_counts, #self.spectrograph.detector[self.det-1]['saturation'], mincounts=self.spectrograph.detector[self.det - 1]['mincounts']) # Error checking assert len(steps_copy) == 0 # Return return pypeitImage
def test_dumb_instantiate(): pypeitImage = pypeitimage.PypeItImage(None) assert pypeitImage.image is None
def run(self, bias=None, flatimages=None, ignore_saturation=False, sigma_clip=True, bpm=None, sigrej=None, maxiters=5, slits=None, dark=None, combine_method='weightmean'): """ Generate a PypeItImage from a list of images This may also generate the ivar, crmask, rn2img and mask Args: bias (:class:`pypeit.images.buildimage.BiasImage`, optional): Bias image flatimages (:class:`pypeit.flatfield.FlatImages`, optional): For flat fielding dark (:class:`pypeit.images.buildimage.DarkImage`, optional): Dark image slits (:class:`pypeit.slittrace.SlitTraceSet`, optional): Slit object sigma_clip (bool, optional): Perform sigma clipping sigrej (int or float, optional): Rejection threshold for sigma clipping. Code defaults to determining this automatically based on the number of images provided. maxiters (int, optional): Number of iterations for the clipping bpm (`numpy.ndarray`_, optional): Bad pixel mask. Held in ImageMask ignore_saturation (:obj:`bool`, optional): If True, turn off the saturation flag in the individual images before stacking This avoids having such values set to 0 which for certain images (e.g. flat calibrations) can have unintended consequences. combine_method (str): Method to combine images Allowed options are 'weightmean', 'median' Returns: :class:`pypeit.images.pypeitimage.PypeItImage`: """ # Loop on the files nimages = len(self.files) lampstat = [] for kk, ifile in enumerate(self.files): # Load raw image rawImage = rawimage.RawImage(ifile, self.spectrograph, self.det) # Process pypeitImage = rawImage.process(self.par, bias=bias, bpm=bpm, dark=dark, flatimages=flatimages, slits=slits) #embed(header='96 of combineimage') # Are we all done? if nimages == 1: return pypeitImage elif kk == 0: # Get ready shape = (nimages, pypeitImage.image.shape[0], pypeitImage.image.shape[1]) img_stack = np.zeros(shape) ivar_stack= np.zeros(shape) rn2img_stack = np.zeros(shape) crmask_stack = np.zeros(shape, dtype=bool) # Mask bitmask = imagebitmask.ImageBitMask() mask_stack = np.zeros(shape, bitmask.minimum_dtype(asuint=True)) # Grab the lamp status lampstat += [self.spectrograph.get_lamps_status(pypeitImage.rawheadlist)] # Process img_stack[kk,:,:] = pypeitImage.image # Construct raw variance image and turn into inverse variance if pypeitImage.ivar is not None: ivar_stack[kk, :, :] = pypeitImage.ivar else: ivar_stack[kk, :, :] = 1. # Mask cosmic rays if pypeitImage.crmask is not None: crmask_stack[kk, :, :] = pypeitImage.crmask # Read noise squared image if pypeitImage.rn2img is not None: rn2img_stack[kk, :, :] = pypeitImage.rn2img # Final mask for this image # TODO This seems kludgy to me. Why not just pass ignore_saturation to process_one and ignore the saturation # when the mask is actually built, rather than untoggling the bit here if ignore_saturation: # Important for calibrations as we don't want replacement by 0 indx = pypeitImage.bitmask.flagged(pypeitImage.fullmask, flag=['SATURATION']) pypeitImage.fullmask[indx] = pypeitImage.bitmask.turn_off( pypeitImage.fullmask[indx], 'SATURATION') mask_stack[kk, :, :] = pypeitImage.fullmask # Check that the lamps being combined are all the same: if not lampstat[1:] == lampstat[:-1]: msgs.warn("The following files contain different lamp status") # Get the longest strings maxlen = max([len("Filename")]+[len(os.path.split(x)[1]) for x in self.files]) maxlmp = max([len("Lamp status")]+[len(x) for x in lampstat]) strout = "{0:" + str(maxlen) + "} {1:s}" # Print the messages print(msgs.indent() + '-'*maxlen + " " + '-'*maxlmp) print(msgs.indent() + strout.format("Filename", "Lamp status")) print(msgs.indent() + '-'*maxlen + " " + '-'*maxlmp) for ff, file in enumerate(self.files): print(msgs.indent() + strout.format(os.path.split(file)[1], " ".join(lampstat[ff].split("_")))) print(msgs.indent() + '-'*maxlen + " " + '-'*maxlmp) # Coadd them weights = np.ones(nimages)/float(nimages) img_list = [img_stack] var_stack = utils.inverse(ivar_stack) var_list = [var_stack, rn2img_stack] if combine_method == 'weightmean': img_list_out, var_list_out, gpm, nused = combine.weighted_combine( weights, img_list, var_list, (mask_stack == 0), sigma_clip=sigma_clip, sigma_clip_stack=img_stack, sigrej=sigrej, maxiters=maxiters) elif combine_method == 'median': img_list_out = [np.median(img_stack, axis=0)] var_list_out = [np.median(var_stack, axis=0)] var_list_out += [np.median(rn2img_stack, axis=0)] gpm = np.ones_like(img_list_out[0], dtype='bool') else: msgs.error("Bad choice for combine. Allowed options are 'median', 'weightmean'.") # Build the last one final_pypeitImage = pypeitimage.PypeItImage(img_list_out[0], ivar=utils.inverse(var_list_out[0]), bpm=pypeitImage.bpm, rn2img=var_list_out[1], crmask=np.logical_not(gpm), detector=pypeitImage.detector, PYP_SPEC=pypeitImage.PYP_SPEC) # Internals final_pypeitImage.rawheadlist = pypeitImage.rawheadlist final_pypeitImage.process_steps = pypeitImage.process_steps nonlinear_counts = self.spectrograph.nonlinear_counts(pypeitImage.detector, apply_gain=self.par['apply_gain']) final_pypeitImage.build_mask(saturation=nonlinear_counts) # Return return final_pypeitImage
def reduce(self, pseudo_dict, show=None, show_peaks=None): """ ..todo.. Please document me Args: pseudo_dict: show: show_peaks: Returns: """ show = self.show if show is None else show show_peaks = self.show_peaks if show_peaks is None else show_peaks sciImage = pypeitimage.PypeItImage(image=pseudo_dict['imgminsky'], ivar=pseudo_dict['sciivar'], bpm=np.zeros_like(pseudo_dict['inmask'].astype(int)), # Dummy bpm rn2img=np.zeros_like(pseudo_dict['inmask']).astype(float), # Dummy rn2img crmask=np.invert(pseudo_dict['inmask'].astype(bool))) sciImage.detector = self.stack_dict['detectors'][0] # slitmask_pseudo = pseudo_dict['slits'].slit_img() sciImage.build_mask(slitmask=slitmask_pseudo) # Make changes to parset specific to 2d coadds parcopy = copy.deepcopy(self.par) parcopy['reduce']['findobj']['trace_npoly'] = 3 # Low order traces since we are rectified #parcopy['calibrations']['save_masters'] = False #parcopy['scienceimage']['find_extrap_npoly'] = 1 # Use low order for trace extrapolation # Build the Calibrate object caliBrate = calibrations.Calibrations(None, self.par['calibrations'], self.spectrograph, None) caliBrate.slits = pseudo_dict['slits'] redux=reduce.Reduce.get_instance(sciImage, self.spectrograph, parcopy, caliBrate, 'science_coadd2d', ir_redux=self.ir_redux, det=self.det, show=show) #redux=reduce.Reduce.get_instance(sciImage, self.spectrograph, parcopy, pseudo_dict['slits'], # None, None, 'science_coadd2d', ir_redux=self.ir_redux, det=self.det, show=show) # Set the tilts and waveimg attributes from the psuedo_dict here, since we generate these dynamically from fits # normally, but this is not possible for coadds redux.tilts = pseudo_dict['tilts'] redux.waveimg = pseudo_dict['waveimg'] redux.binning = self.binning # Masking # TODO: Treat the masking of the slits objects # from every exposure, come up with an aggregate mask (if it is masked on one slit, # mask the slit for all) and that should be propagated into the slits object in the psuedo_dict slits = self.stack_dict['slits_list'][0] reduce_bpm = (slits.mask > 0) & (np.invert(slits.bitmask.flagged( slits.mask, flag=slits.bitmask.exclude_for_reducing))) redux.reduce_bpm = reduce_bpm if show: redux.show('image', image=pseudo_dict['imgminsky']*(sciImage.fullmask == 0), chname = 'imgminsky', slits=True, clear=True) # TODO: # Object finding, this appears inevitable for the moment, since we need to be able to call find_objects # outside of reduce. I think the solution here is to create a method in reduce for that performs the modified # 2d coadd reduce sobjs_obj, nobj, skymask_init = redux.find_objects( sciImage.image, show_peaks=show_peaks, manual_extract_dict=self.par['reduce']['extraction']['manual'].dict_for_objfind()) # Local sky-subtraction global_sky_pseudo = np.zeros_like(pseudo_dict['imgminsky']) # No global sky for co-adds since we go straight to local skymodel_pseudo, objmodel_pseudo, ivarmodel_pseudo, outmask_pseudo, sobjs = redux.local_skysub_extract( global_sky_pseudo, sobjs_obj, spat_pix=pseudo_dict['spat_img'], model_noise=False, show_profile=show, show=show) if self.ir_redux: sobjs.purge_neg() # TODO: Removed this, but I'm not sure that's what you want... # # Add the information about the fixed wavelength grid to the sobjs # for spec in sobjs: # idx = spec.slit_orderindx # # Fill # spec.BOX_WAVE_GRID_MASK, spec.OPT_WAVE_GRID_MASK = [pseudo_dict['wave_mask'][:,idx]]*2 # spec.BOX_WAVE_GRID, spec.OPT_WAVE_GRID = [pseudo_dict['wave_mid'][:,idx]]*2 # spec.BOX_WAVE_GRID_MIN, spec.OPT_WAVE_GRID_MIN = [pseudo_dict['wave_min'][:,idx]]*2 # spec.BOX_WAVE_GRID_MAX, spec.OPT_WAVE_GRID_MAX = [pseudo_dict['wave_max'][:,idx]]*2 # Add the rest to the pseudo_dict pseudo_dict['skymodel'] = skymodel_pseudo pseudo_dict['objmodel'] = objmodel_pseudo pseudo_dict['ivarmodel'] = ivarmodel_pseudo pseudo_dict['outmask'] = outmask_pseudo pseudo_dict['sobjs'] = sobjs self.pseudo_dict=pseudo_dict return pseudo_dict['imgminsky'], pseudo_dict['sciivar'], skymodel_pseudo, \ objmodel_pseudo, ivarmodel_pseudo, outmask_pseudo, sobjs, sciImage.detector, pseudo_dict['slits'], \ pseudo_dict['tilts'], pseudo_dict['waveimg']
def run(self, bias=None, flatimages=None, ignore_saturation=False, sigma_clip=True, bpm=None, sigrej=None, maxiters=5, slits=None, dark=None, combine_method='mean', mosaic=False): r""" Process and combine all images. All processing is performed by the :class:`~pypeit.images.rawimage.RawImage` class; see :func:`~pypeit.images.rawimage.RawImage.process`. If there is only one file (see :attr:`files`), this simply processes the file and returns the result. If there are multiple files, all the files are processed and the processed images are combined based on the ``combine_method``, where the options are: - 'mean': If ``sigma_clip`` is True, this is a sigma-clipped mean; otherwise, this is a simple average. The combination is done using :func:`~pypeit.core.combine.weighted_combine`. - 'median': This is a simple masked median (using `numpy.ma.median`_). The errors in the image are also propagated through the stacking procedure; however, this isn't a simple propagation of the inverse variance arrays. The image processing produces arrays with individual components used to construct the variance model for an individual frame. See :ref:`image_proc` and :func:`~pypeit.procimg.variance_model` for a description of these arrays. Briefly, the relevant arrays are the readnoise variance (:math:`V_{\rm rn}`), the "processing" variance (:math:`V_{\rm proc}`), and the image scaling (i.e., the flat-field correction) (:math:`s`). The variance calculation for the stacked image directly propagates the error in these. For example, the propagated processing variance (modulo the masking) is: .. math:: V_{\rm proc,stack} = \frac{\sum_i s_i^2 V_{{\rm proc},i}}\frac{s_{\rm stack}^2} where :math:`s_{\rm stack}` is the combined image scaling array, combined in the same way as the image data are combined. This ensures that the reconstruction of the uncertainty in the combined image calculated using :func:`~pypeit.procimg.variance_model` accurately includes, e.g., the processing uncertainty. The uncertainty in the combined image, however, recalculates the variance model, using the combined image (which should have less noise) to set the Poisson statistics. The same parameters used when processing the individual frames are applied to the combined frame; see :func:`~pypeit.images.rawimage.RawImage.build_ivar`. This calculation is then the equivalent of when the observed counts are replaced by the model object and sky counts during sky subtraction and spectral extraction. Bitmasks from individual frames in the stack are *not* propagated to the combined image, except to indicate when a pixel was masked for all images in the stack (cf., ``ignore_saturation``). Additionally, the instrument-specific bad-pixel mask, see the :func:`~pypeit.spectrographs.spectrograph.Spectrograph.bpm` method for each instrument subclass, saturated-pixel mask, and other default mask bits (e.g., NaN and non-positive inverse variance values) are all propagated to the combined-image mask; see :func:`~pypeit.images.pypeitimage.PypeItImage.build_mask`. .. warning:: All image processing of the data in :attr:`files` *must* result in images of the same shape. Args: bias (:class:`~pypeit.images.buildimage.BiasImage`, optional): Bias image for bias subtraction; passed directly to :func:`~pypeit.images.rawimage.RawImage.process` for all images. flatimages (:class:`~pypeit.flatfield.FlatImages`, optional): Flat-field images for flat fielding; passed directly to :func:`~pypeit.images.rawimage.RawImage.process` for all images. ignore_saturation (:obj:`bool`, optional): If True, turn off the saturation flag in the individual images before stacking. This avoids having such values set to 0, which for certain images (e.g. flat calibrations) can have unintended consequences. sigma_clip (:obj:`bool`, optional): When ``combine_method='mean'``, perform a sigma-clip the data; see :func:`~pypeit.core.combine.weighted_combine`. bpm (`numpy.ndarray`_, optional): Bad pixel mask; passed directly to :func:`~pypeit.images.rawimage.RawImage.process` for all images. sigrej (:obj:`float`, optional): When ``combine_method='mean'``, this sets the sigma-rejection thresholds used when sigma-clipping the image combination. Ignored if ``sigma_clip`` is False. If None and ``sigma_clip`` is True, the thresholds are determined automatically based on the number of images provided; see :func:`~pypeit.core.combine.weighted_combine``. maxiters (:obj:`int`, optional): When ``combine_method='mean'``) and sigma-clipping (``sigma_clip`` is True), this sets the maximum number of rejection iterations. If None, rejection iterations continue until no more data are rejected; see :func:`~pypeit.core.combine.weighted_combine``. slits (:class:`~pypeit.slittrace.SlitTraceSet`, optional): Slit edge trace locations; passed directly to :func:`~pypeit.images.rawimage.RawImage.process` for all images. dark (:class:`~pypeit.images.buildimage.DarkImage`, optional): Dark-current image; passed directly to :func:`~pypeit.images.rawimage.RawImage.process` for all images. combine_method (:obj:`str`, optional): Method used to combine images. Must be ``'mean'`` or ``'median'``; see above. mosaic (:obj:`bool`, optional): If multiple detectors are being processes, mosaic them into a single image. See :func:`~pypeit.images.rawimage.RawImage.process`. Returns: :class:`~pypeit.images.pypeitimage.PypeItImage`: The combination of all the processed images. """ # Check the input (i.e., bomb out *before* it does any processing) if self.nfiles == 0: msgs.error('Object contains no files to process!') if self.nfiles > 1 and combine_method not in ['mean', 'median']: msgs.error(f'Unknown image combination method, {combine_method}. Must be ' '"mean" or "median".') # If not provided, generate the bpm for this spectrograph and detector. # Regardless of the file used, this must result in the same bpm, so we # just use the first one. # TODO: Why is this done here? It's the same thing as what's done if # bpm is not passed to RawImage.process... # if bpm is None: # bpm = self.spectrograph.bpm(self.files[0], self.det) # Loop on the files for kk, ifile in enumerate(self.files): # Load raw image rawImage = rawimage.RawImage(ifile, self.spectrograph, self.det) # Process pypeitImage = rawImage.process(self.par, bias=bias, bpm=bpm, dark=dark, flatimages=flatimages, slits=slits, mosaic=mosaic) if self.nfiles == 1: # Only 1 file, so we're done pypeitImage.files = self.files return pypeitImage elif kk == 0: # Allocate arrays to collect data for each frame shape = (self.nfiles,) + pypeitImage.shape img_stack = np.zeros(shape, dtype=float) scl_stack = np.ones(shape, dtype=float) rn2img_stack = np.zeros(shape, dtype=float) basev_stack = np.zeros(shape, dtype=float) gpm_stack = np.zeros(shape, dtype=bool) lampstat = [None]*self.nfiles exptime = np.zeros(self.nfiles, dtype=float) # Save the lamp status lampstat[kk] = self.spectrograph.get_lamps_status(pypeitImage.rawheadlist) # Save the exposure time to check if it's consistent for all images. exptime[kk] = pypeitImage.exptime # Processed image img_stack[kk] = pypeitImage.image # Get the count scaling if pypeitImage.img_scale is not None: scl_stack[kk] = pypeitImage.img_scale # Read noise squared image if pypeitImage.rn2img is not None: rn2img_stack[kk] = pypeitImage.rn2img * scl_stack[kk]**2 # Processing variance image if pypeitImage.base_var is not None: basev_stack[kk] = pypeitImage.base_var * scl_stack[kk]**2 # Final mask for this image # TODO: This seems kludgy to me. Why not just pass ignore_saturation # to process_one and ignore the saturation when the mask is actually # built, rather than untoggling the bit here? if ignore_saturation: # Important for calibrations as we don't want replacement by 0 pypeitImage.update_mask('SATURATION', action='turn_off') # Get a simple boolean good-pixel mask for all the unmasked pixels gpm_stack[kk] = pypeitImage.select_flag(invert=True) # Check that the lamps being combined are all the same: if not lampstat[1:] == lampstat[:-1]: msgs.warn("The following files contain different lamp status") # Get the longest strings maxlen = max([len("Filename")]+[len(os.path.split(x)[1]) for x in self.files]) maxlmp = max([len("Lamp status")]+[len(x) for x in lampstat]) strout = "{0:" + str(maxlen) + "} {1:s}" # Print the messages print(msgs.indent() + '-'*maxlen + " " + '-'*maxlmp) print(msgs.indent() + strout.format("Filename", "Lamp status")) print(msgs.indent() + '-'*maxlen + " " + '-'*maxlmp) for ff, file in enumerate(self.files): print(msgs.indent() + strout.format(os.path.split(file)[1], " ".join(lampstat[ff].split("_")))) print(msgs.indent() + '-'*maxlen + " " + '-'*maxlmp) # Do a similar check for exptime if np.any(np.absolute(np.diff(exptime)) > 0): # TODO: This should likely throw an error instead! msgs.warn('Exposure time is not consistent for all images being combined! ' 'Using the average.') comb_texp = np.mean(exptime) else: comb_texp = exptime[0] # Coadd them if combine_method == 'mean': weights = np.ones(self.nfiles, dtype=float)/self.nfiles img_list_out, var_list_out, gpm, nstack \ = combine.weighted_combine(weights, [img_stack, scl_stack], # images to stack [rn2img_stack, basev_stack], # variances to stack gpm_stack, sigma_clip=sigma_clip, sigma_clip_stack=img_stack, # clipping based on img sigrej=sigrej, maxiters=maxiters) comb_img, comb_scl = img_list_out comb_rn2, comb_basev = var_list_out comb_rn2[gpm] /= comb_scl[gpm]**2 comb_basev[gpm] /= comb_scl[gpm]**2 elif combine_method == 'median': bpm_stack = np.logical_not(gpm_stack) nstack = np.sum(gpm_stack, axis=0) gpm = nstack > 0 comb_img = np.ma.median(np.ma.MaskedArray(img_stack, mask=bpm_stack),axis=0).filled(0.) # TODO: I'm not sure if this is right. Maybe we should just take # the masked average scale instead? comb_scl = np.ma.median(np.ma.MaskedArray(scl_stack, mask=bpm_stack),axis=0).filled(0.) # First calculate the error in the sum. The variance is set to 0 # for pixels masked in all images. comb_rn2 = np.ma.sum(np.ma.MaskedArray(rn2img_stack, mask=bpm_stack),axis=0).filled(0.) comb_basev = np.ma.sum(np.ma.MaskedArray(basev_stack, mask=bpm_stack),axis=0).filled(0.) # Convert to standard error in the median (pi/2 factor relates standard variance # in mean (sum(variance_i)/n^2) to standard variance in median) comb_rn2[gpm] *= np.pi/2/nstack[gpm]**2/comb_scl[gpm]**2 comb_basev[gpm] *= np.pi/2/nstack[gpm]**2/comb_scl[gpm]**2 else: # NOTE: Given the check at the beginning of the function, the code # should *never* make it here. msgs.error("Bad choice for combine. Allowed options are 'median', 'mean'.") # Recompute the inverse variance using the combined image comb_var = procimg.variance_model(comb_basev, counts=comb_img if self.par['shot_noise'] else None, count_scale=comb_scl, noise_floor=self.par['noise_floor']) # Build the combined image comb = pypeitimage.PypeItImage(image=comb_img, ivar=utils.inverse(comb_var), nimg=nstack, amp_img=pypeitImage.amp_img, det_img=pypeitImage.det_img, rn2img=comb_rn2, base_var=comb_basev, img_scale=comb_scl, bpm=np.logical_not(gpm).astype(np.uint8), # NOTE: The detector is needed here so # that we can get the dark current later. detector=pypeitImage.detector, PYP_SPEC=self.spectrograph.name, units='e-' if self.par['apply_gain'] else 'ADU', exptime=comb_texp, noise_floor=self.par['noise_floor'], shot_noise=self.par['shot_noise']) # Internals # TODO: Do we need these? comb.files = self.files comb.rawheadlist = pypeitImage.rawheadlist comb.process_steps = pypeitImage.process_steps # Build the base level mask comb.build_mask(saturation='default', mincounts='default') # Flag all pixels with no contributions from any of the stacked images. comb.update_mask('STCKMASK', indx=np.logical_not(gpm)) # Return return comb
def run(self, process_steps, bias, pixel_flat=None, illum_flat=None, ignore_saturation=False, sigma_clip=True, bpm=None, sigrej=None, maxiters=5): """ Generate a PypeItImage from a list of images Mainly a wrapper to coadd2d.weighted_combine() This may also generate the ivar, crmask, rn2img and mask Args: process_steps (list): bias (np.ndarray or None): Bias image or instruction pixel_flat (np.ndarray, optional): Flat image illum_flat (np.ndarray, optional): Illumination image sigma_clip (bool, optional): Perform sigma clipping sigrej (int or float, optional): Rejection threshold for sigma clipping. Code defaults to determining this automatically based on the number of images provided. maxiters (int, optional): Number of iterations for the clipping bpm (np.ndarray, optional): Bad pixel mask. Held in ImageMask ignore_saturation (bool, optional): If True, turn off the saturation flag in the individual images before stacking This avoids having such values set to 0 which for certain images (e.g. flat calibrations) can have unintended consequences. Returns: :class:`pypeit.images.pypeitimage.PypeItImage`: """ # Loop on the files nimages = len(self.files) for kk, ifile in enumerate(self.files): # Process a single image pypeitImage = self.process_one(ifile, process_steps, bias, pixel_flat=pixel_flat, illum_flat=illum_flat, bpm=bpm) # Are we all done? if len(self.files) == 1: return pypeitImage elif kk == 0: # Get ready shape = (nimages, pypeitImage.bpm.shape[0], pypeitImage.bpm.shape[1]) img_stack = np.zeros(shape) ivar_stack = np.zeros(shape) rn2img_stack = np.zeros(shape) crmask_stack = np.zeros(shape, dtype=bool) # Mask bitmask = maskimage.ImageBitMask() mask_stack = np.zeros(shape, bitmask.minimum_dtype(asuint=True)) # Process img_stack[kk, :, :] = pypeitImage.image # Construct raw variance image and turn into inverse variance if pypeitImage.ivar is not None: ivar_stack[kk, :, :] = pypeitImage.ivar else: ivar_stack[kk, :, :] = 1. # Mask cosmic rays if pypeitImage.crmask is not None: crmask_stack[kk, :, :] = pypeitImage.crmask # Read noise squared image if pypeitImage.rn2img is not None: rn2img_stack[kk, :, :] = pypeitImage.rn2img # Final mask for this image # TODO This seems kludgy to me. Why not just pass ignore_saturation to process_one and ignore the saturation # when the mask is actually built, rather than untoggling the bit here if ignore_saturation: # Important for calibrations as we don't want replacement by 0 indx = pypeitImage.bitmask.flagged(pypeitImage.mask, flag=['SATURATION']) pypeitImage.mask[indx] = pypeitImage.bitmask.turn_off( pypeitImage.mask[indx], 'SATURATION') mask_stack[kk, :, :] = pypeitImage.mask # Coadd them weights = np.ones(nimages) / float(nimages) img_list = [img_stack] var_stack = utils.inverse(ivar_stack) var_list = [var_stack, rn2img_stack] img_list_out, var_list_out, outmask, nused = combine.weighted_combine( weights, img_list, var_list, (mask_stack == 0), sigma_clip=sigma_clip, sigma_clip_stack=img_stack, sigrej=sigrej, maxiters=maxiters) # Build the last one final_pypeitImage = pypeitimage.PypeItImage( img_list_out[0], ivar=utils.inverse(var_list_out[0]), bpm=pypeitImage.bpm, rn2img=var_list_out[1], crmask=np.invert(outmask), binning=pypeitImage.binning) nonlinear_counts = self.spectrograph.nonlinear_counts( self.det, apply_gain='apply_gain' in process_steps) final_pypeitImage.build_mask( final_pypeitImage.image, final_pypeitImage.ivar, saturation= nonlinear_counts, #self.spectrograph.detector[self.det-1]['saturation'], mincounts=self.spectrograph.detector[self.det - 1]['mincounts']) # Return return final_pypeitImage