def trim(self, force=False): """ Trim the image to include only the science data Args: force (bool, optional): Force the processing even if the image was already processed """ step = inspect.stack()[0][3] # Check input image matches the original if self.rawimage.shape is not None and self.image.shape != self.rawimage.shape: msgs.warn( "Image shape does not match original. Returning current image" ) return self.image.copy() # Check if already trimmed if self.steps[step] and (not force): msgs.warn("Image was already trimmed. Returning current image") return self.image # Do it self.image = procimg.trim_frame(self.image, self.datasec_img < 1) self.datasec_img = procimg.trim_frame(self.datasec_img, self.datasec_img < 1) # self.steps[step] = True
def bias_subtract(self, msbias, trim=True, force=False, par=None): """ Subtract the bias. Parameters ---------- msbias : ndarray, str (optional) If ndarray, the input is a Bias image If str, the input is guides the Bias subtraction method e.g. 'overscan' trim Returns ------- """ # Check if the bias has already been subtracted if (inspect.stack()[0][3] in self.steps) & (not force): msgs.warn("Images already bias subtracted. Use force=True to reset proc_images " "and do it again. Returning...") return # Set the parameters if par is not None and not isinstance(par, pypeitpar.ProcessImagesPar): raise TypeError('Provided ParSet for must be type ProcessImagesPar.') if par is not None: self.proc_par = par # If trimming, get the image identifying amplifier used for the # data section datasec_img = self.spectrograph.get_datasec_img(self.files[0], det=self.det) msgs.info("Bias subtracting your image(s)") # Reset proc_images -- Is there any reason we wouldn't?? numamplifiers = self.spectrograph.detector[self.det-1]['numamplifiers'] for kk,image in enumerate(self.raw_images): # Bias subtract (move here from procimg) if isinstance(msbias, np.ndarray): msgs.info("Subtracting bias image from raw frame") # Trim? if trim: image = procimg.trim_frame(image, datasec_img < 1) temp = image-msbias elif isinstance(msbias, str) and msbias == 'overscan': msgs.info("Using overscan to subtract") temp = procimg.subtract_overscan(image, numamplifiers, self.datasec[kk], self.oscansec[kk], method=self.proc_par['overscan'], params=self.proc_par['overscan_par']) # Trim? if trim: temp = procimg.trim_frame(temp, datasec_img < 1) else: msgs.error('Could not subtract bias level with the input bias approach.') # Save if kk==0: # Instantiate proc_images self.proc_images = np.zeros((temp.shape[0], temp.shape[1], self.nloaded)) self.proc_images[:,:,kk] = temp.copy() # Step self.steps.append(inspect.stack()[0][3])
def build_rn2img(self, trim=True): """ Generate the model read noise squared image Currently only used by ScienceImage. Wrapper to procimg.rn_frame Returns ------- self.rn2img : ndarray """ msgs.info( "Generating read noise image from detector properties and amplifier layout)" ) datasec_img = self.spectrograph.get_datasec_img(self.files[0], det=self.det) if trim: datasec_img = procimg.trim_frame(datasec_img, datasec_img < 1) detector = self.spectrograph.detector[self.det - 1] self.rn2img = procimg.rn_frame(datasec_img, detector['gain'], detector['ronoise']) self.steps.append(inspect.stack()[0][3]) # Return return self.rn2img
def build_rawvarframe(self, trim=True): """ Generate the Raw Variance frame Currently only used by ScienceImage. Wrapper to procimg.variance_frame Returns ------- self.rawvarframe : ndarray """ msgs.info( "Generating raw variance frame (from detected counts [flat fielded])" ) datasec_img = self.spectrograph.get_datasec_img(self.files[0], det=self.det) if trim: datasec_img = procimg.trim_frame(datasec_img, datasec_img < 1) detector = self.spectrograph.detector[self.det - 1] self.rawvarframe = procimg.variance_frame( datasec_img, self.stack, detector['gain'], detector['ronoise'], darkcurr=detector['darkcurr'], exptime=self.exptime) # Step self.steps.append(inspect.stack()[0][3]) # Return return self.rawvarframe
def load_frame(self, iFile=None): idx = self.check_index(iFile) # Grab the filename fname = self.data_files[idx] detpar, rawimage, _, _, datasec, _ = self.spectrograph.get_rawimage(fname, self.det) rawimage = procimg.trim_frame(rawimage, datasec < 1) return self.spectrograph.orient_image(detpar, rawimage)
def apply_gain(self, trim=True): """ Apply gain (instead of ampsec scale) Parameters ---------- Returns ------- self.mspixelflat -- Modified internally """ # TODO: This is overkill when self.datasec is loaded, and this # call is made for a few of the steps. Can we be more # efficient? datasec_img = self.spectrograph.get_datasec_img(self.files[0], det=self.det) if trim: datasec_img = procimg.trim_frame(datasec_img, datasec_img < 1) if self.stack.shape != datasec_img.shape: raise ValueError('Shape mismatch: {0} {1}'.format( self.stack.shape, datasec_img.shape)) gain = self.spectrograph.detector[self.det - 1]['gain'] if self.spectrograph.detector[self.det-1]['numamplifiers'] == 1 \ and not isinstance(gain, list): gain = [gain] self.stack *= procimg.gain_frame( datasec_img, self.spectrograph.detector[self.det - 1]['numamplifiers'], gain) # Step self.steps.append(inspect.stack()[0][3]) # Return return self.stack
def test_trim(): datasec = np.zeros((10, 10), dtype=int) datasec[:5, :-3] = 1 datasec[5:, :-3] = 2 _datasec = procimg.trim_frame(datasec, datasec < 1) assert _datasec.shape == (10, 7), 'Trimming error' assert np.array_equal(datasec[datasec > 0], _datasec.flat), 'Values changed'
def main(args): # Load fits file cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(args.file, runtime=False) # Get the raw fits file name sciIdx = get_science_frame(usrdata) fname = data_files[sciIdx] # Load the spectrograph cfg = ConfigObj(cfg_lines) spectrograph_name = cfg['rdx']['spectrograph'] spectrograph = load_spectrograph(spectrograph_name, ifile=data_files[sciIdx]) msgs.info('Loaded spectrograph {0}'.format(spectrograph.spectrograph)) spectrograph_cfg_lines = spectrograph.config_specific_par(fname).to_config() par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines) # Get the master key file_base = os.path.basename(fname) ftdict = dict({file_base: 'science'}) fitstbl = PypeItMetaData(spectrograph, par, files=[data_files[sciIdx]], usrdata=Table(usrdata[sciIdx]), strict=True) fitstbl.finalize_usr_build(ftdict, setups[0]) mkey = fitstbl.master_key(0, det=args.det) # Load the frame data rawimage, _, _, datasec, _ = spectrograph.get_rawimage(fname, args.det) rawimage = procimg.trim_frame(rawimage, datasec < 1) frame = spectrograph.orient_image(rawimage, args.det) # Set paths if par['calibrations']['caldir'] == 'default': mdir = os.path.join(par['rdx']['redux_path'], 'Masters') else: mdir = par['calibrations']['caldir'] if not os.path.exists(mdir): mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir)) msgs.warn('Master file dir: {0} does not exist. Using {1}'.format(mdir, mdir_base)) mdir = mdir_base # Load the tslits_dict trc_file = os.path.join(mdir, MasterFrame.construct_file_name('Edges', mkey)) + '.gz' tslits_dict = edgetrace.EdgeTraceSet.from_file(trc_file).convert_to_tslits_dict() # Derive an appropriate output filename prefix = os.path.splitext(file_base) if prefix[1] == ".gz": prefix = os.path.splitext(prefix[0])[0] else: prefix = prefix[0] outname = "{0:s}_skyregions.fits".format(prefix) # Finally, initialise the GUI skyreg = SkySubGUI.initialize(args.det, frame, tslits_dict, outname=outname, runtime=False, printout=True) # Get the results skyreg.get_result()
def test_keck_deimos(): spectrograph = util.load_spectrograph('keck_deimos') example_file = os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'Keck_DEIMOS', '830G_L_8400', 'd0914_0002.fits.gz') # Get the shape dsec_img = spectrograph.get_datasec_img(example_file, det=2) shape = procimg.trim_frame(dsec_img, dsec_img < 1).shape # Simple bpm = spectrograph.bpm(shape=shape,det=4) assert bpm[0,0] == 1
def test_ampsec(spectrograph): """ Test sort_data """ datasec_img = spectrograph.get_datasec_img(data_path('b1.fits.gz'), det=1) datasec_img = procimg.trim_frame(datasec_img, datasec_img < 1) # Test assert datasec_img.shape == (2048, 350) #assert np.sum(np.isclose(datasec_img, 1)) == 2162688 # Data region #assert np.sum(np.isclose(datasec_img, 2)) == 2162688 # second amp assert np.sum(np.isclose(datasec_img, 1)) == 358400 # Data region assert np.sum(np.isclose(datasec_img, 2)) == 358400 # second amp
def test_kecklrisred(): s = spectrographs.keck_lris.KeckLRISRSpectrograph() example_file = os.path.join(os.environ['PYPEIT_DEV'], 'RAW_DATA', 'Keck_LRIS_red', 'long_600_7500_d560', 'LR.20160216.05529.fits.gz') assert os.path.isfile(example_file), 'Could not find example file for Keck LRIS red read.' data, _ = s.load_raw_frame(example_file) # dsec_img = s.get_datasec_img(example_file, det=1) shape = procimg.trim_frame(dsec_img, dsec_img < 1).shape bpm = s.bpm(shape=shape) assert data.shape == (2068,1110) assert bpm.shape == (2048,1024)
def test_kecklrisblue(): s = spectrographs.keck_lris.KeckLRISBSpectrograph() example_file = os.path.join(os.environ['PYPEIT_DEV'], 'RAW_DATA', 'Keck_LRIS_blue', 'long_400_3400_d560', 'LB.20160109.14149.fits.gz') assert os.path.isfile(example_file), 'Could not find example file for Keck LRIS blue read.' data, _ = s.load_raw_frame(example_file) # dsec_img = s.get_datasec_img(example_file, det=1) shape = procimg.trim_frame(dsec_img, dsec_img < 1).shape bpm = s.bpm(shape=shape) assert data.shape == (2048,1154) assert bpm.shape == (2048,1024)
def test_keckdeimos(): s = spectrographs.keck_deimos.KeckDEIMOSSpectrograph() example_file = os.path.join(os.environ['PYPEIT_DEV'], 'RAW_DATA', 'Keck_DEIMOS', '830G_L_8400', 'd0914_0002.fits.gz') assert os.path.isfile(example_file), 'Could not find example file for Keck DEIMOS read.' data, _ = s.load_raw_frame(example_file) # dsec_img = s.get_datasec_img(example_file, det=1) shape = procimg.trim_frame(dsec_img, dsec_img < 1).shape bpm = s.bpm(shape=shape) # filename=example_file) assert data.shape == (4096,2128) assert bpm.shape == (4096,2048)
def test_keck_lris_red(): # Spectrograph spectrograph = util.load_spectrograph('keck_lris_red') # example_file = os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'Keck_LRIS_red', 'long_600_7500_d560', 'LR.20160216.05529.fits.gz') # Get the shape dsec_img = spectrograph.get_datasec_img(example_file, det=2) shape = procimg.trim_frame(dsec_img, dsec_img < 1).shape # Simple bpm = spectrograph.bpm(shape=shape, filename=example_file, det=2) assert np.sum(bpm) > 0
def empty_bpm(self, filename, det, shape=None): """ Generate a generic (empty) bad-pixel mask. Even though they are both optional, either the precise shape for the image (``shape``) or an example file that can be read to get the shape (``filename``) *must* be provided. In the latter, the file is read, trimmed, and re-oriented to get the output shape. If both ``shape`` and ``filename`` are provided, ``shape`` is ignored. Args: filename (:obj:`str`): An example file to use to get the image shape. Can be None, but ``shape`` must be provided, if so. Note the overhead of this function is large if you ``filename``. You're better off providing ``shape``, if you know it. det (:obj:`int`): 1-indexed detector number to use when getting the image shape from the example file. shape (:obj:`tuple`, optional): Processed image shape. I.e., if the image for this instrument is re-oriented, trimmed, etc, this shape must be that of the re-oriented (trimmed, etc) image. This is required if ``filename`` is None, but ignored otherwise. Returns: `numpy.ndarray`_: An integer array with a masked value set to 1 and an unmasked value set to 0. The shape of the returned image should be that of the ``PypeIt`` processed image. This is the generic method for the base class, meaning that all pixels are returned as unmasked (0s). """ # Load the raw frame if filename is None: _shape = shape else: detector_par, _, _, _, rawdatasec_img, _ = self.get_rawimage( filename, det) # Trim + reorient trim = procimg.trim_frame(rawdatasec_img, rawdatasec_img < 1) orient = self.orient_image(detector_par, trim) #, det) _shape = orient.shape # Shape must be defined at this point. if _shape is None: msgs.error('Must specify shape if filename is None.') # Generate # TODO: Why isn't this a boolean array? return np.zeros(_shape, dtype=np.int8)
def get_datasec_img(self, filename, det): """ Generate and return the datasec image in the PypeIt reference frame, e.g. trimmed + oriented Returns: np.ndarray """ rdimg = self.get_rawdatasec_img(filename=filename, det=det) # Fuss rdimg = procimg.trim_frame(rdimg, rdimg < 1) dimg = self.orient_image(rdimg, det) # Return return dimg
def get_bpm(self): """ Load or generate the bad pixel mask TODO -- Should consider doing this outside of calibrations as it is more specific to the science frame This needs to be for the *trimmed* and correctly oriented image! Requirements: Instrument dependent Returns: ndarray: :attr:`msbpm` image of bad pixel mask """ # Check internals self._chk_set(['par', 'det']) # Generate a bad pixel mask (should not repeat) self.master_key_dict['bpm'] = self.fitstbl.master_key(self.frame, det=self.det) if self._cached('bpm', self.master_key_dict['bpm']): self.msbpm = self.calib_dict[self.master_key_dict['bpm']]['bpm'] return self.msbpm # Build the data-section image sci_image_file = self.fitstbl.frame_paths(self.frame) rdsec_img = self.spectrograph.get_rawdatasec_img(sci_image_file, det=self.det) # Instantiate the shape here, based on the shape of the science # image. This is the shape of most calibrations, although we are # allowing for arcs of different shape because of X-shooter etc. trim = procimg.trim_frame(rdsec_img, rdsec_img < 1) orient = self.spectrograph.orient_image(trim, self.det) self.shape = orient.shape # Build it self.msbpm = self.spectrograph.bpm(shape=self.shape, filename=sci_image_file, det=self.det) # Record it self._update_cache('bpm', 'bpm', self.msbpm) # Return return self.msbpm
def empty_bpm(self, filename, det, shape=None): """ Generate a generic (empty) bad-pixel mask. Even though they are both optional, either the precise shape for the image (`shape`) or an example file that can be read to get the shape (`filename` using :func:`get_image_shape`) *must* be provided. Args: filename (:obj:`str` or None): An example file to use to get the image shape. If None, shape must be provided det (:obj:`int`): 1-indexed detector number to use when getting the image shape from the example file. shape (tuple, optional): Processed image shape Required if filename is None Ignored if filename is not None Returns: `numpy.ndarray`_: An integer array with a masked value set to 1 and an unmasked value set to 0. All values are set to 0. """ # Load the raw frame if filename is not None: detector_par, _, _, _, rawdatasec_img, _ = self.get_rawimage( filename, det) # Trim + reorient trim = procimg.trim_frame(rawdatasec_img, rawdatasec_img < 1) orient = self.orient_image(detector_par, trim) #, det) # shape = orient.shape else: # This is risky if you don't really know what you are doing! if shape is None: msgs.error("Must specify shape if filename is None") # Generate bpm_img = np.zeros(shape, dtype=np.int8) # Return return bpm_img
def process(self, bias_subtract=None, apply_gain=False, trim=True, overwrite=False, pixel_flat=None, bpm=None, illum_flat=None): """ Process the images from loading to combining Args: bias_subtract (str, ndarray, None): Guides bias subtraction apply_gain (bool, optional): Apply gain to the various amplifier regions trim (bool, optional): overwrite (bool, optional): pixel_flat (ndarray, optional): This is the normalized pixel flat (i.e. no blaze, no slit illumination profile). The values of this array should have a scatter about 1.0 bpm (ndarray, optional): Bad pixel mask image illum_flat (ndarray, optional): Illumination flat Returns: ndarray: Stacked image """ # Over-write? # TODO: This should probably raise an error. if (inspect.stack()[0][3] in self.steps) & (not overwrite): msgs.warn( "Images already combined. Use overwrite=True to do it again.") return # Load images if 'load_images' not in self.steps: self.load_images() # Bias subtract if bias_subtract is not None: self.bias_subtract(bias_subtract, trim=trim) elif 'bias_subtract' not in self.steps: msgs.warn("Your images have not been bias subtracted!") # Create proc_images from raw_images if need be # Mainly if no bias subtraction was previously performed if self.proc_images is None: # Trim even if not bias subtracting temp = self.raw_images[0] if trim: datasec_img = self.spectrograph.get_datasec_img(self.files[0], det=self.det) temp = procimg.trim_frame(temp, datasec_img < 1) # Init proc_images array self.proc_images = np.zeros( (temp.shape[0], temp.shape[1], self.nloaded)) # Load it up for kk, image in enumerate(self.raw_images): self.proc_images[:,:,kk] = procimg.trim_frame(image, datasec_img < 1) \ if trim else image # Combine self.stack = self.proc_images[:, :, 0] if self.proc_images.shape[ 2] == 1 else self.combine() self.raw_stack = self.stack # Apply gain? if apply_gain: self.apply_gain(trim=trim) # Flat field? if pixel_flat is not None: self.stack = self.flat_field(pixel_flat, bpm, illum_flat=illum_flat) # Done return self.stack.copy()