def trim(self, force=False):
"""
Trim the image to include only the science data
Args:
force (bool, optional):
Force the processing even if the image was already processed
"""
step = inspect.stack()[0][3]
# Check input image matches the original
if self.rawimage.shape is not None and self.image.shape != self.rawimage.shape:
msgs.warn(
"Image shape does not match original. Returning current image"
)
return self.image.copy()
# Check if already trimmed
if self.steps[step] and (not force):
msgs.warn("Image was already trimmed. Returning current image")
return self.image
# Do it
self.image = procimg.trim_frame(self.image, self.datasec_img < 1)
self.datasec_img = procimg.trim_frame(self.datasec_img,
self.datasec_img < 1)
#
self.steps[step] = True
def bias_subtract(self, msbias, trim=True, force=False, par=None):
"""
Subtract the bias.
Parameters
----------
msbias : ndarray, str (optional)
If ndarray, the input is a Bias image
If str, the input is guides the Bias subtraction method e.g. 'overscan'
trim
Returns
-------
"""
# Check if the bias has already been subtracted
if (inspect.stack()[0][3] in self.steps) & (not force):
msgs.warn("Images already bias subtracted. Use force=True to reset proc_images "
"and do it again. Returning...")
return
# Set the parameters
if par is not None and not isinstance(par, pypeitpar.ProcessImagesPar):
raise TypeError('Provided ParSet for must be type ProcessImagesPar.')
if par is not None:
self.proc_par = par
# If trimming, get the image identifying amplifier used for the
# data section
datasec_img = self.spectrograph.get_datasec_img(self.files[0], det=self.det)
msgs.info("Bias subtracting your image(s)")
# Reset proc_images -- Is there any reason we wouldn't??
numamplifiers = self.spectrograph.detector[self.det-1]['numamplifiers']
for kk,image in enumerate(self.raw_images):
# Bias subtract (move here from procimg)
if isinstance(msbias, np.ndarray):
msgs.info("Subtracting bias image from raw frame")
# Trim?
if trim:
image = procimg.trim_frame(image, datasec_img < 1)
temp = image-msbias
elif isinstance(msbias, str) and msbias == 'overscan':
msgs.info("Using overscan to subtract")
temp = procimg.subtract_overscan(image, numamplifiers, self.datasec[kk],
self.oscansec[kk],
method=self.proc_par['overscan'],
params=self.proc_par['overscan_par'])
# Trim?
if trim:
temp = procimg.trim_frame(temp, datasec_img < 1)
else:
msgs.error('Could not subtract bias level with the input bias approach.')
# Save
if kk==0:
# Instantiate proc_images
self.proc_images = np.zeros((temp.shape[0], temp.shape[1], self.nloaded))
self.proc_images[:,:,kk] = temp.copy()
# Step
self.steps.append(inspect.stack()[0][3])
def build_rn2img(self, trim=True):
"""
Generate the model read noise squared image
Currently only used by ScienceImage.
Wrapper to procimg.rn_frame
Returns
-------
self.rn2img : ndarray
"""
msgs.info(
"Generating read noise image from detector properties and amplifier layout)"
)
datasec_img = self.spectrograph.get_datasec_img(self.files[0],
det=self.det)
if trim:
datasec_img = procimg.trim_frame(datasec_img, datasec_img < 1)
detector = self.spectrograph.detector[self.det - 1]
self.rn2img = procimg.rn_frame(datasec_img, detector['gain'],
detector['ronoise'])
self.steps.append(inspect.stack()[0][3])
# Return
return self.rn2img
def build_rawvarframe(self, trim=True):
"""
Generate the Raw Variance frame
Currently only used by ScienceImage.
Wrapper to procimg.variance_frame
Returns
-------
self.rawvarframe : ndarray
"""
msgs.info(
"Generating raw variance frame (from detected counts [flat fielded])"
)
datasec_img = self.spectrograph.get_datasec_img(self.files[0],
det=self.det)
if trim:
datasec_img = procimg.trim_frame(datasec_img, datasec_img < 1)
detector = self.spectrograph.detector[self.det - 1]
self.rawvarframe = procimg.variance_frame(
datasec_img,
self.stack,
detector['gain'],
detector['ronoise'],
darkcurr=detector['darkcurr'],
exptime=self.exptime)
# Step
self.steps.append(inspect.stack()[0][3])
# Return
return self.rawvarframe
def load_frame(self, iFile=None):
idx = self.check_index(iFile)
# Grab the filename
fname = self.data_files[idx]
detpar, rawimage, _, _, datasec, _ = self.spectrograph.get_rawimage(fname, self.det)
rawimage = procimg.trim_frame(rawimage, datasec < 1)
return self.spectrograph.orient_image(detpar, rawimage)
def apply_gain(self, trim=True):
"""
Apply gain (instead of ampsec scale)
Parameters
----------
Returns
-------
self.mspixelflat -- Modified internally
"""
# TODO: This is overkill when self.datasec is loaded, and this
# call is made for a few of the steps. Can we be more
# efficient?
datasec_img = self.spectrograph.get_datasec_img(self.files[0],
det=self.det)
if trim:
datasec_img = procimg.trim_frame(datasec_img, datasec_img < 1)
if self.stack.shape != datasec_img.shape:
raise ValueError('Shape mismatch: {0} {1}'.format(
self.stack.shape, datasec_img.shape))
gain = self.spectrograph.detector[self.det - 1]['gain']
if self.spectrograph.detector[self.det-1]['numamplifiers'] == 1 \
and not isinstance(gain, list):
gain = [gain]
self.stack *= procimg.gain_frame(
datasec_img,
self.spectrograph.detector[self.det - 1]['numamplifiers'], gain)
# Step
self.steps.append(inspect.stack()[0][3])
# Return
return self.stack
def test_trim():
datasec = np.zeros((10, 10), dtype=int)
datasec[:5, :-3] = 1
datasec[5:, :-3] = 2
_datasec = procimg.trim_frame(datasec, datasec < 1)
assert _datasec.shape == (10, 7), 'Trimming error'
assert np.array_equal(datasec[datasec > 0],
_datasec.flat), 'Values changed'
def main(args):
# Load fits file
cfg_lines, data_files, frametype, usrdata, setups = parse_pypeit_file(args.file, runtime=False)
# Get the raw fits file name
sciIdx = get_science_frame(usrdata)
fname = data_files[sciIdx]
# Load the spectrograph
cfg = ConfigObj(cfg_lines)
spectrograph_name = cfg['rdx']['spectrograph']
spectrograph = load_spectrograph(spectrograph_name, ifile=data_files[sciIdx])
msgs.info('Loaded spectrograph {0}'.format(spectrograph.spectrograph))
spectrograph_cfg_lines = spectrograph.config_specific_par(fname).to_config()
par = PypeItPar.from_cfg_lines(cfg_lines=spectrograph_cfg_lines, merge_with=cfg_lines)
# Get the master key
file_base = os.path.basename(fname)
ftdict = dict({file_base: 'science'})
fitstbl = PypeItMetaData(spectrograph, par, files=[data_files[sciIdx]], usrdata=Table(usrdata[sciIdx]), strict=True)
fitstbl.finalize_usr_build(ftdict, setups[0])
mkey = fitstbl.master_key(0, det=args.det)
# Load the frame data
rawimage, _, _, datasec, _ = spectrograph.get_rawimage(fname, args.det)
rawimage = procimg.trim_frame(rawimage, datasec < 1)
frame = spectrograph.orient_image(rawimage, args.det)
# Set paths
if par['calibrations']['caldir'] == 'default':
mdir = os.path.join(par['rdx']['redux_path'], 'Masters')
else:
mdir = par['calibrations']['caldir']
if not os.path.exists(mdir):
mdir_base = os.path.join(os.getcwd(), os.path.basename(mdir))
msgs.warn('Master file dir: {0} does not exist. Using {1}'.format(mdir, mdir_base))
mdir = mdir_base
# Load the tslits_dict
trc_file = os.path.join(mdir, MasterFrame.construct_file_name('Edges', mkey)) + '.gz'
tslits_dict = edgetrace.EdgeTraceSet.from_file(trc_file).convert_to_tslits_dict()
# Derive an appropriate output filename
prefix = os.path.splitext(file_base)
if prefix[1] == ".gz":
prefix = os.path.splitext(prefix[0])[0]
else:
prefix = prefix[0]
outname = "{0:s}_skyregions.fits".format(prefix)
# Finally, initialise the GUI
skyreg = SkySubGUI.initialize(args.det, frame, tslits_dict, outname=outname, runtime=False, printout=True)
# Get the results
skyreg.get_result()
def test_keck_deimos():
spectrograph = util.load_spectrograph('keck_deimos')
example_file = os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'Keck_DEIMOS', '830G_L_8400',
'd0914_0002.fits.gz')
# Get the shape
dsec_img = spectrograph.get_datasec_img(example_file, det=2)
shape = procimg.trim_frame(dsec_img, dsec_img < 1).shape
# Simple
bpm = spectrograph.bpm(shape=shape,det=4)
assert bpm[0,0] == 1
def test_ampsec(spectrograph):
""" Test sort_data
"""
datasec_img = spectrograph.get_datasec_img(data_path('b1.fits.gz'), det=1)
datasec_img = procimg.trim_frame(datasec_img, datasec_img < 1)
# Test
assert datasec_img.shape == (2048, 350)
#assert np.sum(np.isclose(datasec_img, 1)) == 2162688 # Data region
#assert np.sum(np.isclose(datasec_img, 2)) == 2162688 # second amp
assert np.sum(np.isclose(datasec_img, 1)) == 358400 # Data region
assert np.sum(np.isclose(datasec_img, 2)) == 358400 # second amp
def test_kecklrisred():
s = spectrographs.keck_lris.KeckLRISRSpectrograph()
example_file = os.path.join(os.environ['PYPEIT_DEV'], 'RAW_DATA', 'Keck_LRIS_red',
'long_600_7500_d560', 'LR.20160216.05529.fits.gz')
assert os.path.isfile(example_file), 'Could not find example file for Keck LRIS red read.'
data, _ = s.load_raw_frame(example_file)
#
dsec_img = s.get_datasec_img(example_file, det=1)
shape = procimg.trim_frame(dsec_img, dsec_img < 1).shape
bpm = s.bpm(shape=shape)
assert data.shape == (2068,1110)
assert bpm.shape == (2048,1024)
def test_kecklrisblue():
s = spectrographs.keck_lris.KeckLRISBSpectrograph()
example_file = os.path.join(os.environ['PYPEIT_DEV'], 'RAW_DATA', 'Keck_LRIS_blue',
'long_400_3400_d560', 'LB.20160109.14149.fits.gz')
assert os.path.isfile(example_file), 'Could not find example file for Keck LRIS blue read.'
data, _ = s.load_raw_frame(example_file)
#
dsec_img = s.get_datasec_img(example_file, det=1)
shape = procimg.trim_frame(dsec_img, dsec_img < 1).shape
bpm = s.bpm(shape=shape)
assert data.shape == (2048,1154)
assert bpm.shape == (2048,1024)
def test_keckdeimos():
s = spectrographs.keck_deimos.KeckDEIMOSSpectrograph()
example_file = os.path.join(os.environ['PYPEIT_DEV'], 'RAW_DATA', 'Keck_DEIMOS',
'830G_L_8400', 'd0914_0002.fits.gz')
assert os.path.isfile(example_file), 'Could not find example file for Keck DEIMOS read.'
data, _ = s.load_raw_frame(example_file)
#
dsec_img = s.get_datasec_img(example_file, det=1)
shape = procimg.trim_frame(dsec_img, dsec_img < 1).shape
bpm = s.bpm(shape=shape) # filename=example_file)
assert data.shape == (4096,2128)
assert bpm.shape == (4096,2048)
def test_keck_lris_red():
# Spectrograph
spectrograph = util.load_spectrograph('keck_lris_red')
#
example_file = os.path.join(os.getenv('PYPEIT_DEV'), 'RAW_DATA', 'Keck_LRIS_red',
'long_600_7500_d560', 'LR.20160216.05529.fits.gz')
# Get the shape
dsec_img = spectrograph.get_datasec_img(example_file, det=2)
shape = procimg.trim_frame(dsec_img, dsec_img < 1).shape
# Simple
bpm = spectrograph.bpm(shape=shape, filename=example_file, det=2)
assert np.sum(bpm) > 0
def empty_bpm(self, filename, det, shape=None):
"""
Generate a generic (empty) bad-pixel mask.
Even though they are both optional, either the precise shape for the
image (``shape``) or an example file that can be read to get the
shape (``filename``) *must* be provided. In the latter, the file is
read, trimmed, and re-oriented to get the output shape. If both
``shape`` and ``filename`` are provided, ``shape`` is ignored.
Args:
filename (:obj:`str`):
An example file to use to get the image shape. Can be None,
but ``shape`` must be provided, if so. Note the overhead of
this function is large if you ``filename``. You're better off
providing ``shape``, if you know it.
det (:obj:`int`):
1-indexed detector number to use when getting the image
shape from the example file.
shape (:obj:`tuple`, optional):
Processed image shape. I.e., if the image for this instrument
is re-oriented, trimmed, etc, this shape must be that of the
re-oriented (trimmed, etc) image. This is required if
``filename`` is None, but ignored otherwise.
Returns:
`numpy.ndarray`_: An integer array with a masked value set to 1
and an unmasked value set to 0. The shape of the returned image
should be that of the ``PypeIt`` processed image. This is the
generic method for the base class, meaning that all pixels are
returned as unmasked (0s).
"""
# Load the raw frame
if filename is None:
_shape = shape
else:
detector_par, _, _, _, rawdatasec_img, _ = self.get_rawimage(
filename, det)
# Trim + reorient
trim = procimg.trim_frame(rawdatasec_img, rawdatasec_img < 1)
orient = self.orient_image(detector_par, trim) #, det)
_shape = orient.shape
# Shape must be defined at this point.
if _shape is None:
msgs.error('Must specify shape if filename is None.')
# Generate
# TODO: Why isn't this a boolean array?
return np.zeros(_shape, dtype=np.int8)
def get_datasec_img(self, filename, det):
"""
Generate and return the datasec image in the PypeIt reference
frame, e.g. trimmed + oriented
Returns:
np.ndarray
"""
rdimg = self.get_rawdatasec_img(filename=filename, det=det)
# Fuss
rdimg = procimg.trim_frame(rdimg, rdimg < 1)
dimg = self.orient_image(rdimg, det)
# Return
return dimg
def get_bpm(self):
"""
Load or generate the bad pixel mask
TODO -- Should consider doing this outside of calibrations as it is
more specific to the science frame
This needs to be for the *trimmed* and correctly oriented image!
Requirements:
Instrument dependent
Returns:
ndarray: :attr:`msbpm` image of bad pixel mask
"""
# Check internals
self._chk_set(['par', 'det'])
# Generate a bad pixel mask (should not repeat)
self.master_key_dict['bpm'] = self.fitstbl.master_key(self.frame,
det=self.det)
if self._cached('bpm', self.master_key_dict['bpm']):
self.msbpm = self.calib_dict[self.master_key_dict['bpm']]['bpm']
return self.msbpm
# Build the data-section image
sci_image_file = self.fitstbl.frame_paths(self.frame)
rdsec_img = self.spectrograph.get_rawdatasec_img(sci_image_file,
det=self.det)
# Instantiate the shape here, based on the shape of the science
# image. This is the shape of most calibrations, although we are
# allowing for arcs of different shape because of X-shooter etc.
trim = procimg.trim_frame(rdsec_img, rdsec_img < 1)
orient = self.spectrograph.orient_image(trim, self.det)
self.shape = orient.shape
# Build it
self.msbpm = self.spectrograph.bpm(shape=self.shape,
filename=sci_image_file,
det=self.det)
# Record it
self._update_cache('bpm', 'bpm', self.msbpm)
# Return
return self.msbpm
def empty_bpm(self, filename, det, shape=None):
"""
Generate a generic (empty) bad-pixel mask.
Even though they are both optional, either the precise shape for
the image (`shape`) or an example file that can be read to get
the shape (`filename` using :func:`get_image_shape`) *must* be
provided.
Args:
filename (:obj:`str` or None):
An example file to use to get the image shape.
If None, shape must be provided
det (:obj:`int`):
1-indexed detector number to use when getting the image
shape from the example file.
shape (tuple, optional):
Processed image shape
Required if filename is None
Ignored if filename is not None
Returns:
`numpy.ndarray`_: An integer array with a masked value set
to 1 and an unmasked value set to 0. All values are set to 0.
"""
# Load the raw frame
if filename is not None:
detector_par, _, _, _, rawdatasec_img, _ = self.get_rawimage(
filename, det)
# Trim + reorient
trim = procimg.trim_frame(rawdatasec_img, rawdatasec_img < 1)
orient = self.orient_image(detector_par, trim) #, det)
#
shape = orient.shape
else: # This is risky if you don't really know what you are doing!
if shape is None:
msgs.error("Must specify shape if filename is None")
# Generate
bpm_img = np.zeros(shape, dtype=np.int8)
# Return
return bpm_img
def process(self,
bias_subtract=None,
apply_gain=False,
trim=True,
overwrite=False,
pixel_flat=None,
bpm=None,
illum_flat=None):
"""
Process the images from loading to combining
Args:
bias_subtract (str, ndarray, None): Guides bias subtraction
apply_gain (bool, optional): Apply gain to the various amplifier regions
trim (bool, optional):
overwrite (bool, optional):
pixel_flat (ndarray, optional):
This is the normalized pixel flat (i.e. no blaze, no slit illumination profile).
The values of this array should have a scatter about 1.0
bpm (ndarray, optional): Bad pixel mask image
illum_flat (ndarray, optional): Illumination flat
Returns:
ndarray: Stacked image
"""
# Over-write?
# TODO: This should probably raise an error.
if (inspect.stack()[0][3] in self.steps) & (not overwrite):
msgs.warn(
"Images already combined. Use overwrite=True to do it again.")
return
# Load images
if 'load_images' not in self.steps:
self.load_images()
# Bias subtract
if bias_subtract is not None:
self.bias_subtract(bias_subtract, trim=trim)
elif 'bias_subtract' not in self.steps:
msgs.warn("Your images have not been bias subtracted!")
# Create proc_images from raw_images if need be
# Mainly if no bias subtraction was previously performed
if self.proc_images is None:
# Trim even if not bias subtracting
temp = self.raw_images[0]
if trim:
datasec_img = self.spectrograph.get_datasec_img(self.files[0],
det=self.det)
temp = procimg.trim_frame(temp, datasec_img < 1)
# Init proc_images array
self.proc_images = np.zeros(
(temp.shape[0], temp.shape[1], self.nloaded))
# Load it up
for kk, image in enumerate(self.raw_images):
self.proc_images[:,:,kk] = procimg.trim_frame(image, datasec_img < 1) \
if trim else image
# Combine
self.stack = self.proc_images[:, :, 0] if self.proc_images.shape[
2] == 1 else self.combine()
self.raw_stack = self.stack
# Apply gain?
if apply_gain:
self.apply_gain(trim=trim)
# Flat field?
if pixel_flat is not None:
self.stack = self.flat_field(pixel_flat,
bpm,
illum_flat=illum_flat)
# Done
return self.stack.copy()
