def test_sec2slice():
sub = ':10,10:'
subslice = parse.sec2slice(sub, require_dim=2)
assert subslice[0].start is None
subslice = parse.sec2slice(sub, include_end=True, require_dim=2)
assert subslice[0].stop == 11
subslice = parse.sec2slice(sub, one_indexed=True, require_dim=2)
assert subslice[0].stop == 9
def get_pixel_img(self, filename, section, det):
"""
Create an image identifying the amplifier used to read each pixel.
This is in the *raw* data format
.. todo::
- I find 1-indexing to be highly annoying...
- Check for overlapping amplifiers?
- Consider renaming this datasec_ampid or something like
that. I.e., the image's main purpose is to tell you where
the amplifiers are for the data section
Note on data format
--------------------
binning_pypeit = the binning in the PypeIt convention of (spec, spat)
binning_raw = the binning in the format of the raw data.
In other words: PypeIt requires spec to be the first dimension of the image as read into python. If the
files are stored the other way with spat as the first dimension (as read into python), then the transpose
flag manages this, which is basically the value of the self.detector[det-1]['specaxis'] above.
(Note also that BTW the python convention of storing images is transposed relative to the fits convention
and the datasec typically written to headers. However this flip is dealt with explicitly in the
parse.spec2slice code and is NOT the transpose we are describing and flipping here).
TODO Add a blurb on the PypeIt data model.
Args:
filename (str):
Name of the file from which to read the image size.
section (str): 'datasec' or 'oscansec'
det (int):
Detector number (1-indexed)
Returns:
`numpy.ndarray`: Integer array identifying the amplifier
used to read each pixel.
"""
# Check the detector is defined
self._check_detector()
# Get the image shape
raw_naxis = self.get_raw_image_shape(filename, det=det)
binning_raw = self.get_meta_value(filename, 'binning')
data_sections, one_indexed, include_end \
= self.get_image_section(filename, det, section=section)
# Initialize the image (0 means no amplifier)
pix_img = np.zeros(raw_naxis, dtype=int)
for i in range(self.detector[det-1]['numamplifiers']):
# Convert the data section from a string to a slice
datasec = parse.sec2slice(data_sections[i], one_indexed=one_indexed,
include_end=include_end, require_dim=2,
binning=binning_raw) #transpose=transpose,
# Assign the amplifier
#self.datasec_img[datasec] = i+1
pix_img[datasec] = i+1
return pix_img
def get_datasec_img(self, filename, det=1, force=True):
"""
Create an image identifying the amplifier used to read each pixel.
.. todo::
- I find 1-indexing to be highly annoying...
- Check for overlapping amplifiers?
- Consider renaming this datasec_ampid or something like
that. I.e., the image's main purpose is to tell you where
the amplifiers are for the data section
Args:
filename (str):
Name of the file from which to read the image size.
det (int):
Detector number (1-indexed)
force (:obj:`bool`, optional):
Force the image to be remade
Returns:
`numpy.ndarray`: Integer array identifying the amplifier
used to read each pixel.
"""
if self.datasec_img is None or force:
# Check the detector is defined
self._check_detector()
# Get the image shape
raw_naxis = self.get_raw_image_shape(filename, det=det)
# This *always* returns spectral then spatial
binning = self.get_meta_value(filename, 'binning')
# This *always* returns spectral then spatial
data_sections, one_indexed, include_end \
= self.get_image_section(filename, det, section='datasec')
# Initialize the image (0 means no amplifier)
self.datasec_img = np.zeros(raw_naxis, dtype=int)
for i in range(self.detector[det - 1]['numamplifiers']):
# Convert the data section from a string to a slice
datasec = parse.sec2slice(data_sections[i],
one_indexed=one_indexed,
include_end=include_end,
require_dim=2,
binning=binning)
# Assign the amplifier
self.datasec_img[datasec] = i + 1
return self.datasec_img
def load_images(self, files=None, det=None, binning=None):
"""
Load image header, data, and relevant image sections into
memory.
This always forces the data to be re-read, even if it's already
in memory.
Args:
files (:obj:`str`, :obj:`list`, optional):
One or more files to read and process. If None, use
:attr:`files`.
det (:obj:`int`, optional):
The 1-indexed detector to read. If None, :attr:`det` is
used.
binning (:obj:`str`, :obj:`list`, optional):
Binning of the images in PypeIt format (a
comma-separated string ordered by spatial then spectral
binning in numbers of pixels). If None, this is parsed
from the file headers.
Returns:
Five lists are returned::
- numpy arrays with the raw image data. See
:func:`pypeit.spectrographs.spectrograph.Spectrograph.load_raw_frame`.
- :class:`astropy.io.fits.Header` instances with the
relevant header for the image data. See
:func:`pypeit.spectrographs.spectrograph.Spectrograph.load_raw_frame`.
- :obj:`str` objects with the PypeIt-format binning
- :obj:`slice` objects that select the data sections of
the returned image data, accounting for any image binning.
- :obj:`slice` objects that select the overscan sections
of the returned image data, accounting for any image binning.
"""
if files is not None:
self._set_files(files)
# Set the detector
if det is not None:
self.det = det
# Zero out any previous load
# TODO: Do we need to be more explicit than this? I.e., use del
self.raw_images = [None] * self.nfiles
self.headers = [None] * self.nfiles
self.binning = [None] * self.nfiles if binning is None else binning
self.datasec = [None] * self.nfiles
self.oscansec = [None] * self.nfiles
for i in range(self.nfiles):
# Load the image data and headers
self.raw_images[i], self.headers[i] \
= self.spectrograph.load_raw_frame(self.files[i], det=self.det)
if self.binning[i] is None:
# This *always* returns spectral then spatial
self.binning[i] = self.spectrograph.get_meta_value(
self.files[i], 'binning')
# Get the data sections, one section per amplifier
try:
# This *always* returns spectral then spatial
datasec, one_indexed, include_end \
= self.spectrograph.get_image_section(inp=self.headers[i], det=self.det,
section='datasec')
except:
# This *always* returns spectral then spatial
datasec, one_indexed, include_end \
= self.spectrograph.get_image_section(inp=self.files[i], det=self.det,
section='datasec')
self.datasec[i] = [
parse.sec2slice(sec,
one_indexed=one_indexed,
include_end=include_end,
require_dim=2,
binning=self.binning[i]) for sec in datasec
]
# Get the overscan sections, one section per amplifier
try:
# This *always* returns spectral then spatial
oscansec, one_indexed, include_end \
= self.spectrograph.get_image_section(inp=self.headers[i], det=self.det,
section='oscansec')
except:
# This *always* returns spectral then spatial
oscansec, one_indexed, include_end \
= self.spectrograph.get_image_section(inp=self.files[i], det=self.det,
section='oscansec')
# Parse, including handling binning
self.oscansec[i] = [
parse.sec2slice(sec,
one_indexed=one_indexed,
include_end=include_end,
require_dim=2,
binning=self.binning[i]) for sec in oscansec
]
# Include step
self.steps.append(inspect.stack()[0][3])
def get_rawimage(self, raw_file, det):
"""
Read a raw KCWI data frame
NOTE: The amplifiers are arranged as follows:
| (0,ny) --------- (nx,ny)
| | 3 | 4 |
| ---------
| | 1 | 2 |
| (0,0) --------- (nx, 0)
Parameters
----------
raw_file : str
Filename
det (int or None):
Detector number
Returns
-------
array : ndarray
Combined image
hdu : HDUList
Opened fits file.
sections : list
List of datasec, oscansec, ampsec sections. datasec,
oscansec needs to be for an *unbinned* image as per
standard convention
"""
# Check for file; allow for extra .gz, etc. suffix
fil = glob.glob(raw_file + '*')
if len(fil) != 1:
msgs.error("Found {:d} files matching {:s}".format(
len(fil), raw_file))
# Read
msgs.info("Reading KCWI file: {:s}".format(fil[0]))
hdu = fits.open(fil[0])
detpar = self.get_detector_par(hdu, det if det is None else 1)
head0 = hdu[0].header
raw_img = hdu[detpar['dataext']].data.astype(float)
# Some properties of the image
numamps = head0['NVIDINP']
# Exposure time (used by ProcessRawImage)
headarr = self.get_headarr(hdu)
exptime = self.get_meta_value(headarr, 'exptime')
# get the x and y binning factors...
binning = head0['BINNING']
xbin, ybin = [int(ibin) for ibin in binning.split(',')]
binning_raw = binning
# Always assume normal FITS header formatting
one_indexed = True
include_last = True
for section in ['DSEC', 'BSEC']:
# Initialize the image (0 means no amplifier)
pix_img = np.zeros(raw_img.shape, dtype=int)
for i in range(numamps):
# Get the data section
sec = head0[section + "{0:1d}".format(i + 1)]
# Convert the data section from a string to a slice
datasec = parse.sec2slice(sec,
one_indexed=one_indexed,
include_end=include_last,
require_dim=2,
binning=binning_raw)
# Flip the datasec
datasec = datasec[::-1]
# Assign the amplifier
pix_img[datasec] = i + 1
# Finish
if section == 'DSEC':
rawdatasec_img = pix_img.copy()
elif section == 'BSEC':
oscansec_img = pix_img.copy()
# Return
return detpar, raw_img, hdu, exptime, rawdatasec_img, oscansec_img
def get_rawimage(self, raw_file, det):
"""
Load up the raw image and generate a few other bits and pieces
that are key for image processing
Args:
raw_file (str):
det (int):
Returns:
tuple:
raw_img (np.ndarray) -- Raw image for this detector
hdu (astropy.io.fits.HDUList)
exptime (float)
rawdatasec_img (np.ndarray)
oscansec_img (np.ndarray)
binning_raw (tuple)
"""
# Raw image
hdu = fits.open(raw_file)
raw_img = hdu[self.detector[det-1]['dataext']].data.astype(float)
# Extras
headarr = self.get_headarr(hdu)
# Exposure time (used by ProcessRawImage)
exptime = self.get_meta_value(headarr, 'exptime')
# Rawdatasec, oscansec images
binning = self.get_meta_value(headarr, 'binning')
if self.detector[det - 1]['specaxis'] == 1:
binning_raw = (',').join(binning.split(',')[::-1])
else:
binning_raw = binning
for section in ['datasec', 'oscansec']:
# Get the data section
# Try using the image sections as header keywords
# TODO -- Deal with user windowing of the CCD (e.g. Kast red)
# Code like the following maybe useful
#hdr = hdu[self.detector[det - 1]['dataext']].header
#image_sections = [hdr[key] for key in self.detector[det - 1][section]]
# Grab from DetectorPar in the Spectrograph class
image_sections = self.detector[det-1][section]
if not isinstance(image_sections, list):
image_sections = [image_sections]
# Always assume normal FITS header formatting
one_indexed = True
include_last = True
# Initialize the image (0 means no amplifier)
pix_img = np.zeros(raw_img.shape, dtype=int)
for i in range(self.detector[det-1]['numamplifiers']):
if image_sections[i] is not None:
# Convert the data section from a string to a slice
datasec = parse.sec2slice(image_sections[i], one_indexed=one_indexed,
include_end=include_last, require_dim=2,
binning=binning_raw)
# Assign the amplifier
pix_img[datasec] = i+1
# Finish
if section == 'datasec':
rawdatasec_img = pix_img.copy()
else:
oscansec_img = pix_img.copy()
# Return
return raw_img, hdu, exptime, rawdatasec_img, oscansec_img
def get_rawimage(self, raw_file, det):
"""
Load up the raw image and generate a few other bits and pieces
that are key for image processing
Parameters
----------
raw_file : :obj:`str`
File to read
det : :obj:`int`
Detector to read
Returns
-------
detector_par : :class:`pypeit.par.pypeitpar.DetectorPar`
raw_img : `numpy.ndarray`_
Raw image for this detector
hdu : `astropy.io.fits.HDUList`_
Opened fits file
exptime : :obj:`float`
rawdatasec_img : `numpy.ndarray`_
oscansec_img : `numpy.ndarray`_
"""
# Open
hdu = fits.open(raw_file)
# Grab the DetectorPar
detector = self.get_detector_par(hdu, det)
# Raw image
raw_img = hdu[detector['dataext']].data.astype(float)
# TODO -- Move to FLAMINGOS2 spectrograph
# raw data from some spectrograph (i.e. FLAMINGOS2) have an addition extention, so I add the following two lines.
# it's easier to change here than writing another get_rawimage function in the spectrograph file.
if raw_img.ndim == 3:
raw_img = raw_img[0]
# Extras
headarr = self.get_headarr(hdu)
# Exposure time (used by ProcessRawImage)
exptime = self.get_meta_value(headarr, 'exptime')
# Rawdatasec, oscansec images
binning = self.get_meta_value(headarr, 'binning')
if detector['specaxis'] == 1:
binning_raw = (',').join(binning.split(',')[::-1])
else:
binning_raw = binning
for section in ['datasec', 'oscansec']:
# Get the data section
# Try using the image sections as header keywords
# TODO -- Deal with user windowing of the CCD (e.g. Kast red)
# Code like the following maybe useful
#hdr = hdu[detector[det - 1]['dataext']].header
#image_sections = [hdr[key] for key in detector[det - 1][section]]
# Grab from Detector
image_sections = detector[section]
#if not isinstance(image_sections, list):
# image_sections = [image_sections]
# Always assume normal FITS header formatting
one_indexed = True
include_last = True
# Initialize the image (0 means no amplifier)
pix_img = np.zeros(raw_img.shape, dtype=int)
for i in range(detector['numamplifiers']):
if image_sections is not None: # and image_sections[i] is not None:
# Convert the data section from a string to a slice
datasec = parse.sec2slice(image_sections[i],
one_indexed=one_indexed,
include_end=include_last,
require_dim=2,
binning=binning_raw)
# Assign the amplifier
pix_img[datasec] = i + 1
# Finish
if section == 'datasec':
rawdatasec_img = pix_img.copy()
else:
oscansec_img = pix_img.copy()
# Return
return detector, raw_img, hdu, exptime, rawdatasec_img, oscansec_img
def get_rawimage(self, raw_file, det):
"""
Read a raw KCWI data frame
NOTE: The amplifiers are arranged as follows:
| (0,ny) --------- (nx,ny)
| | 3 | 4 |
| ---------
| | 1 | 2 |
| (0,0) --------- (nx, 0)
Parameters
----------
raw_file : :obj:`str`
File to read
det : :obj:`int`
1-indexed detector to read
Returns
-------
detector_par : :class:`pypeit.images.detector_container.DetectorContainer`
Detector metadata parameters.
raw_img : `numpy.ndarray`_
Raw image for this detector.
hdu : `astropy.io.fits.HDUList`_
Opened fits file
exptime : :obj:`float`
Exposure time read from the file header
rawdatasec_img : `numpy.ndarray`_
Data (Science) section of the detector as provided by setting the
(1-indexed) number of the amplifier used to read each detector
pixel. Pixels unassociated with any amplifier are set to 0.
oscansec_img : `numpy.ndarray`_
Overscan section of the detector as provided by setting the
(1-indexed) number of the amplifier used to read each detector
pixel. Pixels unassociated with any amplifier are set to 0.
"""
# Check for file; allow for extra .gz, etc. suffix
fil = glob.glob(raw_file + '*')
if len(fil) != 1:
msgs.error("Found {:d} files matching {:s}".format(
len(fil), raw_file))
# Read
msgs.info("Reading KCWI file: {:s}".format(fil[0]))
hdu = io.fits_open(fil[0])
detpar = self.get_detector_par(hdu, det if det is None else 1)
head0 = hdu[0].header
raw_img = hdu[detpar['dataext']].data.astype(float)
# Some properties of the image
numamps = head0['NVIDINP']
# Exposure time (used by ProcessRawImage)
headarr = self.get_headarr(hdu)
exptime = self.get_meta_value(headarr, 'exptime')
# get the x and y binning factors...
#binning = self.get_meta_value(headarr, 'binning')
# Always assume normal FITS header formatting
one_indexed = True
include_last = True
for section in ['DSEC', 'BSEC']:
# Initialize the image (0 means no amplifier)
pix_img = np.zeros(raw_img.shape, dtype=int)
for i in range(numamps):
# Get the data section
sec = head0[section + "{0:1d}".format(i + 1)]
# Convert the data section from a string to a slice
# TODO :: RJC - I think something has changed here... and the BPM is flipped (or not flipped) for different amp modes.
# TODO :: RJC - Note, KCWI records binned sections, so there's no need to pass binning in as an arguement
datasec = parse.sec2slice(sec,
one_indexed=one_indexed,
include_end=include_last,
require_dim=2) #, binning=binning)
# Flip the datasec
datasec = datasec[::-1]
# Assign the amplifier
pix_img[datasec] = i + 1
# Finish
if section == 'DSEC':
rawdatasec_img = pix_img.copy()
elif section == 'BSEC':
oscansec_img = pix_img.copy()
# Calculate the pattern frequency
hdu = self.calc_pattern_freq(raw_img, rawdatasec_img, oscansec_img,
hdu)
# Return
return detpar, raw_img, hdu, exptime, rawdatasec_img, oscansec_img
def get_rawimage(self, raw_file, det):
"""
Read raw images and generate a few other bits and pieces
that are key for image processing.
For LDT/DeVeny, the LOIS control system automatically adjusts the
DATASEC and OSCANSEC regions if the CCD is used in a binning other
than 1x1. The get_rawimage() method in the base class assumes these
sections are fixed and adjusts them based on the binning -- incorrect
for this instrument.
This method is a stripped-down version of the base class method and
additionally does NOT send the binning to parse.sec2slice().
Parameters
----------
raw_file : :obj:`str`
File to read
det : :obj:`int`
1-indexed detector to read
Returns
-------
detector_par : :class:`pypeit.images.detector_container.DetectorContainer`
Detector metadata parameters.
raw_img : `numpy.ndarray`_
Raw image for this detector.
hdu : `astropy.io.fits.HDUList`_
Opened fits file
exptime : :obj:`float`
Exposure time *in seconds*.
rawdatasec_img : `numpy.ndarray`_
Data (Science) section of the detector as provided by setting the
(1-indexed) number of the amplifier used to read each detector
pixel. Pixels unassociated with any amplifier are set to 0.
oscansec_img : `numpy.ndarray`_
Overscan section of the detector as provided by setting the
(1-indexed) number of the amplifier used to read each detector
pixel. Pixels unassociated with any amplifier are set to 0.
"""
# Open
hdu = io.fits_open(raw_file)
# Grab the DetectorContainer and extract the raw image
detector = self.get_detector_par(det, hdu=hdu)
raw_img = hdu[detector['dataext']].data.astype(float)
# Exposure time (used by RawImage) from the header
headarr = self.get_headarr(hdu)
exptime = self.get_meta_value(headarr, 'exptime')
for section in ['datasec', 'oscansec']:
# Get the data section from Detector
image_sections = detector[section]
# Initialize the image (0 means no amplifier)
pix_img = np.zeros(raw_img.shape, dtype=int)
for i in range(detector['numamplifiers']):
if image_sections is not None:
# Convert the (FITS) data section from a string to a slice
# DO NOT send the binning (default: None)
datasec = parse.sec2slice(image_sections[i],
one_indexed=True,
include_end=True,
require_dim=2)
# Assign the amplifier
pix_img[datasec] = i + 1
# Finish
if section == 'datasec':
rawdatasec_img = pix_img.copy()
else:
oscansec_img = pix_img.copy()
# Return
return detector, raw_img, hdu, exptime, rawdatasec_img, oscansec_img
