def test_sec2slice(): sub = ':10,10:' subslice = parse.sec2slice(sub, require_dim=2) assert subslice[0].start is None subslice = parse.sec2slice(sub, include_end=True, require_dim=2) assert subslice[0].stop == 11 subslice = parse.sec2slice(sub, one_indexed=True, require_dim=2) assert subslice[0].stop == 9
def get_pixel_img(self, filename, section, det): """ Create an image identifying the amplifier used to read each pixel. This is in the *raw* data format .. todo:: - I find 1-indexing to be highly annoying... - Check for overlapping amplifiers? - Consider renaming this datasec_ampid or something like that. I.e., the image's main purpose is to tell you where the amplifiers are for the data section Note on data format -------------------- binning_pypeit = the binning in the PypeIt convention of (spec, spat) binning_raw = the binning in the format of the raw data. In other words: PypeIt requires spec to be the first dimension of the image as read into python. If the files are stored the other way with spat as the first dimension (as read into python), then the transpose flag manages this, which is basically the value of the self.detector[det-1]['specaxis'] above. (Note also that BTW the python convention of storing images is transposed relative to the fits convention and the datasec typically written to headers. However this flip is dealt with explicitly in the parse.spec2slice code and is NOT the transpose we are describing and flipping here). TODO Add a blurb on the PypeIt data model. Args: filename (str): Name of the file from which to read the image size. section (str): 'datasec' or 'oscansec' det (int): Detector number (1-indexed) Returns: `numpy.ndarray`: Integer array identifying the amplifier used to read each pixel. """ # Check the detector is defined self._check_detector() # Get the image shape raw_naxis = self.get_raw_image_shape(filename, det=det) binning_raw = self.get_meta_value(filename, 'binning') data_sections, one_indexed, include_end \ = self.get_image_section(filename, det, section=section) # Initialize the image (0 means no amplifier) pix_img = np.zeros(raw_naxis, dtype=int) for i in range(self.detector[det-1]['numamplifiers']): # Convert the data section from a string to a slice datasec = parse.sec2slice(data_sections[i], one_indexed=one_indexed, include_end=include_end, require_dim=2, binning=binning_raw) #transpose=transpose, # Assign the amplifier #self.datasec_img[datasec] = i+1 pix_img[datasec] = i+1 return pix_img
def get_datasec_img(self, filename, det=1, force=True): """ Create an image identifying the amplifier used to read each pixel. .. todo:: - I find 1-indexing to be highly annoying... - Check for overlapping amplifiers? - Consider renaming this datasec_ampid or something like that. I.e., the image's main purpose is to tell you where the amplifiers are for the data section Args: filename (str): Name of the file from which to read the image size. det (int): Detector number (1-indexed) force (:obj:`bool`, optional): Force the image to be remade Returns: `numpy.ndarray`: Integer array identifying the amplifier used to read each pixel. """ if self.datasec_img is None or force: # Check the detector is defined self._check_detector() # Get the image shape raw_naxis = self.get_raw_image_shape(filename, det=det) # This *always* returns spectral then spatial binning = self.get_meta_value(filename, 'binning') # This *always* returns spectral then spatial data_sections, one_indexed, include_end \ = self.get_image_section(filename, det, section='datasec') # Initialize the image (0 means no amplifier) self.datasec_img = np.zeros(raw_naxis, dtype=int) for i in range(self.detector[det - 1]['numamplifiers']): # Convert the data section from a string to a slice datasec = parse.sec2slice(data_sections[i], one_indexed=one_indexed, include_end=include_end, require_dim=2, binning=binning) # Assign the amplifier self.datasec_img[datasec] = i + 1 return self.datasec_img
def load_images(self, files=None, det=None, binning=None): """ Load image header, data, and relevant image sections into memory. This always forces the data to be re-read, even if it's already in memory. Args: files (:obj:`str`, :obj:`list`, optional): One or more files to read and process. If None, use :attr:`files`. det (:obj:`int`, optional): The 1-indexed detector to read. If None, :attr:`det` is used. binning (:obj:`str`, :obj:`list`, optional): Binning of the images in PypeIt format (a comma-separated string ordered by spatial then spectral binning in numbers of pixels). If None, this is parsed from the file headers. Returns: Five lists are returned:: - numpy arrays with the raw image data. See :func:`pypeit.spectrographs.spectrograph.Spectrograph.load_raw_frame`. - :class:`astropy.io.fits.Header` instances with the relevant header for the image data. See :func:`pypeit.spectrographs.spectrograph.Spectrograph.load_raw_frame`. - :obj:`str` objects with the PypeIt-format binning - :obj:`slice` objects that select the data sections of the returned image data, accounting for any image binning. - :obj:`slice` objects that select the overscan sections of the returned image data, accounting for any image binning. """ if files is not None: self._set_files(files) # Set the detector if det is not None: self.det = det # Zero out any previous load # TODO: Do we need to be more explicit than this? I.e., use del self.raw_images = [None] * self.nfiles self.headers = [None] * self.nfiles self.binning = [None] * self.nfiles if binning is None else binning self.datasec = [None] * self.nfiles self.oscansec = [None] * self.nfiles for i in range(self.nfiles): # Load the image data and headers self.raw_images[i], self.headers[i] \ = self.spectrograph.load_raw_frame(self.files[i], det=self.det) if self.binning[i] is None: # This *always* returns spectral then spatial self.binning[i] = self.spectrograph.get_meta_value( self.files[i], 'binning') # Get the data sections, one section per amplifier try: # This *always* returns spectral then spatial datasec, one_indexed, include_end \ = self.spectrograph.get_image_section(inp=self.headers[i], det=self.det, section='datasec') except: # This *always* returns spectral then spatial datasec, one_indexed, include_end \ = self.spectrograph.get_image_section(inp=self.files[i], det=self.det, section='datasec') self.datasec[i] = [ parse.sec2slice(sec, one_indexed=one_indexed, include_end=include_end, require_dim=2, binning=self.binning[i]) for sec in datasec ] # Get the overscan sections, one section per amplifier try: # This *always* returns spectral then spatial oscansec, one_indexed, include_end \ = self.spectrograph.get_image_section(inp=self.headers[i], det=self.det, section='oscansec') except: # This *always* returns spectral then spatial oscansec, one_indexed, include_end \ = self.spectrograph.get_image_section(inp=self.files[i], det=self.det, section='oscansec') # Parse, including handling binning self.oscansec[i] = [ parse.sec2slice(sec, one_indexed=one_indexed, include_end=include_end, require_dim=2, binning=self.binning[i]) for sec in oscansec ] # Include step self.steps.append(inspect.stack()[0][3])
def get_rawimage(self, raw_file, det): """ Read a raw KCWI data frame NOTE: The amplifiers are arranged as follows: | (0,ny) --------- (nx,ny) | | 3 | 4 | | --------- | | 1 | 2 | | (0,0) --------- (nx, 0) Parameters ---------- raw_file : str Filename det (int or None): Detector number Returns ------- array : ndarray Combined image hdu : HDUList Opened fits file. sections : list List of datasec, oscansec, ampsec sections. datasec, oscansec needs to be for an *unbinned* image as per standard convention """ # Check for file; allow for extra .gz, etc. suffix fil = glob.glob(raw_file + '*') if len(fil) != 1: msgs.error("Found {:d} files matching {:s}".format( len(fil), raw_file)) # Read msgs.info("Reading KCWI file: {:s}".format(fil[0])) hdu = fits.open(fil[0]) detpar = self.get_detector_par(hdu, det if det is None else 1) head0 = hdu[0].header raw_img = hdu[detpar['dataext']].data.astype(float) # Some properties of the image numamps = head0['NVIDINP'] # Exposure time (used by ProcessRawImage) headarr = self.get_headarr(hdu) exptime = self.get_meta_value(headarr, 'exptime') # get the x and y binning factors... binning = head0['BINNING'] xbin, ybin = [int(ibin) for ibin in binning.split(',')] binning_raw = binning # Always assume normal FITS header formatting one_indexed = True include_last = True for section in ['DSEC', 'BSEC']: # Initialize the image (0 means no amplifier) pix_img = np.zeros(raw_img.shape, dtype=int) for i in range(numamps): # Get the data section sec = head0[section + "{0:1d}".format(i + 1)] # Convert the data section from a string to a slice datasec = parse.sec2slice(sec, one_indexed=one_indexed, include_end=include_last, require_dim=2, binning=binning_raw) # Flip the datasec datasec = datasec[::-1] # Assign the amplifier pix_img[datasec] = i + 1 # Finish if section == 'DSEC': rawdatasec_img = pix_img.copy() elif section == 'BSEC': oscansec_img = pix_img.copy() # Return return detpar, raw_img, hdu, exptime, rawdatasec_img, oscansec_img
def get_rawimage(self, raw_file, det): """ Load up the raw image and generate a few other bits and pieces that are key for image processing Args: raw_file (str): det (int): Returns: tuple: raw_img (np.ndarray) -- Raw image for this detector hdu (astropy.io.fits.HDUList) exptime (float) rawdatasec_img (np.ndarray) oscansec_img (np.ndarray) binning_raw (tuple) """ # Raw image hdu = fits.open(raw_file) raw_img = hdu[self.detector[det-1]['dataext']].data.astype(float) # Extras headarr = self.get_headarr(hdu) # Exposure time (used by ProcessRawImage) exptime = self.get_meta_value(headarr, 'exptime') # Rawdatasec, oscansec images binning = self.get_meta_value(headarr, 'binning') if self.detector[det - 1]['specaxis'] == 1: binning_raw = (',').join(binning.split(',')[::-1]) else: binning_raw = binning for section in ['datasec', 'oscansec']: # Get the data section # Try using the image sections as header keywords # TODO -- Deal with user windowing of the CCD (e.g. Kast red) # Code like the following maybe useful #hdr = hdu[self.detector[det - 1]['dataext']].header #image_sections = [hdr[key] for key in self.detector[det - 1][section]] # Grab from DetectorPar in the Spectrograph class image_sections = self.detector[det-1][section] if not isinstance(image_sections, list): image_sections = [image_sections] # Always assume normal FITS header formatting one_indexed = True include_last = True # Initialize the image (0 means no amplifier) pix_img = np.zeros(raw_img.shape, dtype=int) for i in range(self.detector[det-1]['numamplifiers']): if image_sections[i] is not None: # Convert the data section from a string to a slice datasec = parse.sec2slice(image_sections[i], one_indexed=one_indexed, include_end=include_last, require_dim=2, binning=binning_raw) # Assign the amplifier pix_img[datasec] = i+1 # Finish if section == 'datasec': rawdatasec_img = pix_img.copy() else: oscansec_img = pix_img.copy() # Return return raw_img, hdu, exptime, rawdatasec_img, oscansec_img
def get_rawimage(self, raw_file, det): """ Load up the raw image and generate a few other bits and pieces that are key for image processing Parameters ---------- raw_file : :obj:`str` File to read det : :obj:`int` Detector to read Returns ------- detector_par : :class:`pypeit.par.pypeitpar.DetectorPar` raw_img : `numpy.ndarray`_ Raw image for this detector hdu : `astropy.io.fits.HDUList`_ Opened fits file exptime : :obj:`float` rawdatasec_img : `numpy.ndarray`_ oscansec_img : `numpy.ndarray`_ """ # Open hdu = fits.open(raw_file) # Grab the DetectorPar detector = self.get_detector_par(hdu, det) # Raw image raw_img = hdu[detector['dataext']].data.astype(float) # TODO -- Move to FLAMINGOS2 spectrograph # raw data from some spectrograph (i.e. FLAMINGOS2) have an addition extention, so I add the following two lines. # it's easier to change here than writing another get_rawimage function in the spectrograph file. if raw_img.ndim == 3: raw_img = raw_img[0] # Extras headarr = self.get_headarr(hdu) # Exposure time (used by ProcessRawImage) exptime = self.get_meta_value(headarr, 'exptime') # Rawdatasec, oscansec images binning = self.get_meta_value(headarr, 'binning') if detector['specaxis'] == 1: binning_raw = (',').join(binning.split(',')[::-1]) else: binning_raw = binning for section in ['datasec', 'oscansec']: # Get the data section # Try using the image sections as header keywords # TODO -- Deal with user windowing of the CCD (e.g. Kast red) # Code like the following maybe useful #hdr = hdu[detector[det - 1]['dataext']].header #image_sections = [hdr[key] for key in detector[det - 1][section]] # Grab from Detector image_sections = detector[section] #if not isinstance(image_sections, list): # image_sections = [image_sections] # Always assume normal FITS header formatting one_indexed = True include_last = True # Initialize the image (0 means no amplifier) pix_img = np.zeros(raw_img.shape, dtype=int) for i in range(detector['numamplifiers']): if image_sections is not None: # and image_sections[i] is not None: # Convert the data section from a string to a slice datasec = parse.sec2slice(image_sections[i], one_indexed=one_indexed, include_end=include_last, require_dim=2, binning=binning_raw) # Assign the amplifier pix_img[datasec] = i + 1 # Finish if section == 'datasec': rawdatasec_img = pix_img.copy() else: oscansec_img = pix_img.copy() # Return return detector, raw_img, hdu, exptime, rawdatasec_img, oscansec_img
def get_rawimage(self, raw_file, det): """ Read a raw KCWI data frame NOTE: The amplifiers are arranged as follows: | (0,ny) --------- (nx,ny) | | 3 | 4 | | --------- | | 1 | 2 | | (0,0) --------- (nx, 0) Parameters ---------- raw_file : :obj:`str` File to read det : :obj:`int` 1-indexed detector to read Returns ------- detector_par : :class:`pypeit.images.detector_container.DetectorContainer` Detector metadata parameters. raw_img : `numpy.ndarray`_ Raw image for this detector. hdu : `astropy.io.fits.HDUList`_ Opened fits file exptime : :obj:`float` Exposure time read from the file header rawdatasec_img : `numpy.ndarray`_ Data (Science) section of the detector as provided by setting the (1-indexed) number of the amplifier used to read each detector pixel. Pixels unassociated with any amplifier are set to 0. oscansec_img : `numpy.ndarray`_ Overscan section of the detector as provided by setting the (1-indexed) number of the amplifier used to read each detector pixel. Pixels unassociated with any amplifier are set to 0. """ # Check for file; allow for extra .gz, etc. suffix fil = glob.glob(raw_file + '*') if len(fil) != 1: msgs.error("Found {:d} files matching {:s}".format( len(fil), raw_file)) # Read msgs.info("Reading KCWI file: {:s}".format(fil[0])) hdu = io.fits_open(fil[0]) detpar = self.get_detector_par(hdu, det if det is None else 1) head0 = hdu[0].header raw_img = hdu[detpar['dataext']].data.astype(float) # Some properties of the image numamps = head0['NVIDINP'] # Exposure time (used by ProcessRawImage) headarr = self.get_headarr(hdu) exptime = self.get_meta_value(headarr, 'exptime') # get the x and y binning factors... #binning = self.get_meta_value(headarr, 'binning') # Always assume normal FITS header formatting one_indexed = True include_last = True for section in ['DSEC', 'BSEC']: # Initialize the image (0 means no amplifier) pix_img = np.zeros(raw_img.shape, dtype=int) for i in range(numamps): # Get the data section sec = head0[section + "{0:1d}".format(i + 1)] # Convert the data section from a string to a slice # TODO :: RJC - I think something has changed here... and the BPM is flipped (or not flipped) for different amp modes. # TODO :: RJC - Note, KCWI records binned sections, so there's no need to pass binning in as an arguement datasec = parse.sec2slice(sec, one_indexed=one_indexed, include_end=include_last, require_dim=2) #, binning=binning) # Flip the datasec datasec = datasec[::-1] # Assign the amplifier pix_img[datasec] = i + 1 # Finish if section == 'DSEC': rawdatasec_img = pix_img.copy() elif section == 'BSEC': oscansec_img = pix_img.copy() # Calculate the pattern frequency hdu = self.calc_pattern_freq(raw_img, rawdatasec_img, oscansec_img, hdu) # Return return detpar, raw_img, hdu, exptime, rawdatasec_img, oscansec_img
def get_rawimage(self, raw_file, det): """ Read raw images and generate a few other bits and pieces that are key for image processing. For LDT/DeVeny, the LOIS control system automatically adjusts the DATASEC and OSCANSEC regions if the CCD is used in a binning other than 1x1. The get_rawimage() method in the base class assumes these sections are fixed and adjusts them based on the binning -- incorrect for this instrument. This method is a stripped-down version of the base class method and additionally does NOT send the binning to parse.sec2slice(). Parameters ---------- raw_file : :obj:`str` File to read det : :obj:`int` 1-indexed detector to read Returns ------- detector_par : :class:`pypeit.images.detector_container.DetectorContainer` Detector metadata parameters. raw_img : `numpy.ndarray`_ Raw image for this detector. hdu : `astropy.io.fits.HDUList`_ Opened fits file exptime : :obj:`float` Exposure time *in seconds*. rawdatasec_img : `numpy.ndarray`_ Data (Science) section of the detector as provided by setting the (1-indexed) number of the amplifier used to read each detector pixel. Pixels unassociated with any amplifier are set to 0. oscansec_img : `numpy.ndarray`_ Overscan section of the detector as provided by setting the (1-indexed) number of the amplifier used to read each detector pixel. Pixels unassociated with any amplifier are set to 0. """ # Open hdu = io.fits_open(raw_file) # Grab the DetectorContainer and extract the raw image detector = self.get_detector_par(det, hdu=hdu) raw_img = hdu[detector['dataext']].data.astype(float) # Exposure time (used by RawImage) from the header headarr = self.get_headarr(hdu) exptime = self.get_meta_value(headarr, 'exptime') for section in ['datasec', 'oscansec']: # Get the data section from Detector image_sections = detector[section] # Initialize the image (0 means no amplifier) pix_img = np.zeros(raw_img.shape, dtype=int) for i in range(detector['numamplifiers']): if image_sections is not None: # Convert the (FITS) data section from a string to a slice # DO NOT send the binning (default: None) datasec = parse.sec2slice(image_sections[i], one_indexed=True, include_end=True, require_dim=2) # Assign the amplifier pix_img[datasec] = i + 1 # Finish if section == 'datasec': rawdatasec_img = pix_img.copy() else: oscansec_img = pix_img.copy() # Return return detector, raw_img, hdu, exptime, rawdatasec_img, oscansec_img