def test_init():
# Instantiate a simple pypeitImage
pypeitImage = pypeitimage.PypeItImage(np.ones((1000, 1000)))
pypeitImage.fullmask = np.zeros((1000, 1000), dtype=np.int64)
pypeitImage.detector = test_detector.detector_container.DetectorContainer(**test_detector.def_det)
# Now the arcimage
arcImage = buildimage.ArcImage.from_pypeitimage(pypeitImage)
def test_image():
# Just for a dummy image
one_file = data_path('b1.fits.gz')
data = fits.open(one_file)[0].data.astype(float)
#
pypeitImage = pypeitimage.PypeItImage(data)
#
assert pypeitImage.image.shape == (350, 2112)
def test_write():
# Just for a dummy image
data = np.ones((1000, 1000))
ivar = np.ones_like(data)
mask = np.ones_like(data).astype(int)
#
pypeitImage = pypeitimage.PypeItImage(data, ivar=ivar, mask=mask)
#
outfile = data_path('tst.fits')
pypeitImage.write(outfile)
# Test
hdul = fits.open(outfile)
assert len(hdul) == 4
assert hdul[2].name == 'IVAR'
def test_master_io():
# Instantiate a simple pypeitImage
pypeitImage = pypeitimage.PypeItImage(np.ones((1000, 1000)))
pypeitImage.fullmask = np.zeros((1000, 1000), dtype=np.int64)
pypeitImage.detector = test_detector.detector_container.DetectorContainer(**test_detector.def_det)
pypeitImage.PYP_SPEC = 'shane_kast_blue'
# Now the arcimage
arcImage = buildimage.ArcImage.from_pypeitimage(pypeitImage)
# Write
master_filename = masterframe.construct_file_name(arcImage, 'A_01_22', master_dir=data_path(''))
arcImage.to_master_file(master_filename)
# Read
_arcImage = buildimage.ArcImage.from_file(data_path('MasterArc_A_01_22.fits'))
assert isinstance(_arcImage.detector, test_detector.detector_container.DetectorContainer)
def test_run():
# Masters
spectrograph = load_spectrograph('shane_kast_blue')
edges, tilts_dict = load_kast_blue_masters(edges=True, tilts=True)
# Instantiate
frametype = 'pixelflat'
par = pypeitpar.FrameGroupPar(frametype)
flatField = flatfield.FlatField(spectrograph,
par,
det=1,
tilts_dict=tilts_dict,
tslits_dict=edges.convert_to_tslits_dict())
# Use the trace image
flatField.rawflatimg = pypeitimage.PypeItImage(edges.img.copy())
mspixelflatnrm, msillumflat = flatField.run()
assert np.isclose(np.median(mspixelflatnrm), 1.0)
def test_full():
pypeitImage = pypeitimage.PypeItImage(np.ones((1000, 1000)))
# Mask
pypeitImage.fullmask = np.zeros((1000, 1000), dtype=np.int64)
# Full datamodel
full_datamodel = pypeitImage.full_datamodel()
assert 'gain' in full_datamodel.keys()
assert 'detector' in pypeitImage.keys()
# I/O
outfile = data_path('tst_pypeitimage.fits')
pypeitImage.to_file(outfile, overwrite=True)
_pypeitImage = pypeitimage.PypeItImage.from_file(outfile)
# Test
assert isinstance(_pypeitImage.image, np.ndarray)
assert _pypeitImage.ivar is None
def test_full():
pypeitImage = pypeitimage.PypeItImage(np.ones((1000, 1000)))
pypeitImage.reinit_mask()
# Full datamodel
# full_datamodel = pypeitImage.full_datamodel()
# assert 'gain' in full_datamodel.keys()
assert 'detector' in pypeitImage.keys()
# I/O
outfile = data_path('tst_pypeitimage.fits')
pypeitImage.to_file(outfile, overwrite=True)
_pypeitImage = pypeitimage.PypeItImage.from_file(outfile)
# Cleanup
os.remove(outfile)
# Test
assert isinstance(_pypeitImage.image, np.ndarray)
assert _pypeitImage.ivar is None
def from_master_file(cls, master_file, par=None):
"""
Instantiate the class from a master file
Args:
master_file (str):
par (:class:`pypeit.par.pypeitpar.PypeItPar`, optional):
Full par set
Returns:
:class:`pypeit.flatfield.FlatField`:
With the flat images loaded up
"""
# Spectrograph
spectrograph, extras = masterframe.items_from_master_file(master_file)
head0 = extras[0]
# Par
if par is None:
par = spectrograph.default_pypeit_par()
# Master info
master_dir = head0['MSTRDIR']
master_key = head0['MSTRKEY']
# Instantiate
slf = cls(spectrograph,
par['calibrations']['pixelflatframe'],
master_dir=master_dir,
master_key=master_key,
reuse_masters=True)
# Load
rawflatimg, slf.mspixelflat, slf.msillumflat = slf.load(
ifile=master_file)
# Convert rawflatimg to a PypeItImage
slf.rawflatimg = pypeitimage.PypeItImage(rawflatimg)
# Return
return slf
def process(self,
par,
bpm=bpm,
flatimages=None,
bias=None,
slits=None,
debug=False,
dark=None):
"""
Process the image
Note: The processing step order is currently 'frozen' as is.
We may choose to allow optional ordering
Here are the allowed steps, in the order they will be applied:
subtract_overscan -- Analyze the overscan region and subtract from the image
trim -- Trim the image down to the data (i.e. remove the overscan)
orient -- Orient the image in the PypeIt orientation (spec, spat) with blue to red going down to up
subtract_bias -- Subtract a bias image
apply_gain -- Convert to counts, amp by amp
flatten -- Divide by the pixel flat and (if provided) the illumination flat
extras -- Generate the RN2 and IVAR images
crmask -- Generate a CR mask
Args:
par (:class:`pypeit.par.pypeitpar.ProcessImagesPar`):
Parameters that dictate the processing of the images. See
:class:`pypeit.par.pypeitpar.ProcessImagesPar` for the
defaults.
bpm (`numpy.ndarray`_, optional):
flatimages (:class:`pypeit.flatfield.FlatImages`):
bias (`numpy.ndarray`_, optional):
Bias image
slits (:class:`pypeit.slittrace.SlitTraceSet`, optional):
Used to calculate spatial flexure between the image and the slits
Returns:
:class:`pypeit.images.pypeitimage.PypeItImage`:
"""
self.par = par
self._bpm = bpm
# Get started
# Standard order
# -- May need to allow for other order some day..
if par['use_pattern']: # Note, this step *must* be done before use_overscan
self.subtract_pattern()
if par['use_overscan']:
self.subtract_overscan()
if par['trim']:
self.trim()
if par['orient']:
self.orient()
if par['use_biasimage']: # Bias frame, if it exists, is *not* trimmed nor oriented
self.subtract_bias(bias)
if par['use_darkimage']: # Dark frame, if it exists, is TODO:: check: trimmed, oriented (and oscan/bias subtracted?)
self.subtract_dark(dark)
if par['apply_gain']:
self.apply_gain()
# This needs to come after trim, orient
# Calculate flexure -- May not be used, but always calculated when slits are provided
if slits is not None and self.par['spat_flexure_correct']:
self.spat_flexure_shift = flexure.spat_flexure_shift(
self.image, slits)
# Generate the illumination flat, as needed
illum_flat = None
if self.par['use_illumflat']:
if flatimages is None:
msgs.error(
"Cannot illumflatten, no such image generated. Add one or more illumflat images to your PypeIt file!!"
)
if slits is None:
msgs.error("Need to provide slits to create illumination flat")
illum_flat = flatimages.fit2illumflat(
slits, flexure_shift=self.spat_flexure_shift)
if debug:
left, right = slits.select_edges(
flexure=self.spat_flexure_shift)
viewer, ch = display.show_image(illum_flat,
chname='illum_flat')
display.show_slits(viewer, ch, left, right) # , slits.id)
#
orig_image = self.image.copy()
viewer, ch = display.show_image(orig_image,
chname='orig_image')
display.show_slits(viewer, ch, left, right) # , slits.id)
# Apply the relative spectral illumination
spec_illum = 1.0
if self.par['use_specillum']:
if flatimages is None or flatimages.get_spec_illum() is None:
msgs.error(
"Spectral illumination correction desired but not generated/provided."
)
else:
spec_illum = flatimages.get_spec_illum().copy()
# Flat field -- We cannot do illumination flat without a pixel flat (yet)
if self.par['use_pixelflat'] or self.par['use_illumflat']:
if flatimages is None or flatimages.get_pixelflat() is None:
msgs.error("Flat fielding desired but not generated/provided.")
else:
self.flatten(flatimages.get_pixelflat() * spec_illum,
illum_flat=illum_flat,
bpm=self.bpm)
# Fresh BPM
bpm = self.spectrograph.bpm(self.filename,
self.det,
shape=self.image.shape)
# Extras
self.build_rn2img()
self.build_ivar()
# Generate a PypeItImage
pypeitImage = pypeitimage.PypeItImage(
self.image,
ivar=self.ivar,
rn2img=self.rn2img,
bpm=bpm,
detector=self.detector,
spat_flexure=self.spat_flexure_shift,
PYP_SPEC=self.spectrograph.spectrograph)
pypeitImage.rawheadlist = self.headarr
pypeitImage.process_steps = [
key for key in self.steps.keys() if self.steps[key]
]
# Mask(s)
if par['mask_cr']:
pypeitImage.build_crmask(self.par)
#
nonlinear_counts = self.spectrograph.nonlinear_counts(
self.detector, apply_gain=self.par['apply_gain'])
# Build
pypeitImage.build_mask(saturation=nonlinear_counts)
# Return
return pypeitImage
def process(self,
process_steps,
pixel_flat=None,
illum_flat=None,
bias=None):
"""
Process the image
Note: The processing step order is currently 'frozen' as is.
We may choose to allow optional ordering
Here are the allowed steps, in the order they will be applied:
subtract_overscan -- Analyze the overscan region and subtract from the image
trim -- Trim the image down to the data (i.e. remove the overscan)
orient -- Orient the image in the PypeIt orientation (spec, spat) with blue to red going down to up
subtract_bias -- Subtract a bias image
apply_gain -- Convert to counts, amp by amp
flatten -- Divide by the pixel flat and (if provided) the illumination flat
extras -- Generate the RN2 and IVAR images
crmask -- Generate a CR mask
Args:
process_steps (list):
List of processing steps
pixel_flat (np.ndarray, optional):
Pixel flat image
illum_flat (np.ndarray, optional):
Illumination flat
bias (np.ndarray, optional):
Bias image
bpm (np.ndarray, optional):
Bad pixel mask image
Returns:
:class:`pypeit.images.pypeitimage.PypeItImage`:
"""
# For error checking
steps_copy = process_steps.copy()
# Get started
# Standard order
# -- May need to allow for other order some day..
if 'subtract_overscan' in process_steps:
self.subtract_overscan()
steps_copy.remove('subtract_overscan')
if 'trim' in process_steps:
self.trim()
steps_copy.remove('trim')
if 'orient' in process_steps:
self.orient()
steps_copy.remove('orient')
if 'subtract_bias' in process_steps: # Bias frame, if it exists, is trimmed and oriented
self.subtract_bias(bias)
steps_copy.remove('subtract_bias')
if 'apply_gain' in process_steps:
self.apply_gain()
steps_copy.remove('apply_gain')
# Flat field
if 'flatten' in process_steps:
self.flatten(pixel_flat, illum_flat=illum_flat, bpm=self.bpm)
steps_copy.remove('flatten')
# Fresh BPM
bpm = self.spectrograph.bpm(self.filename,
self.det,
shape=self.image.shape)
# Extras
if 'extras' in process_steps:
self.build_rn2img()
self.build_ivar()
steps_copy.remove('extras')
# Generate a PypeItImage
pypeitImage = pypeitimage.PypeItImage(self.image,
binning=self.binning,
ivar=self.ivar,
rn2img=self.rn2img,
bpm=bpm)
# Mask(s)
if 'crmask' in process_steps:
if 'extras' in process_steps:
var = utils.inverse(pypeitImage.ivar)
else:
var = np.ones_like(pypeitImage.image)
#
pypeitImage.build_crmask(self.spectrograph, self.det, self.par,
pypeitImage.image, var)
steps_copy.remove('crmask')
nonlinear_counts = self.spectrograph.nonlinear_counts(
self.det, apply_gain='apply_gain' in process_steps)
pypeitImage.build_mask(
pypeitImage.image,
pypeitImage.ivar,
saturation=
nonlinear_counts, #self.spectrograph.detector[self.det-1]['saturation'],
mincounts=self.spectrograph.detector[self.det - 1]['mincounts'])
# Error checking
assert len(steps_copy) == 0
# Return
return pypeitImage
def test_dumb_instantiate():
pypeitImage = pypeitimage.PypeItImage(None)
assert pypeitImage.image is None
def run(self, bias=None, flatimages=None, ignore_saturation=False, sigma_clip=True,
bpm=None, sigrej=None, maxiters=5, slits=None, dark=None, combine_method='weightmean'):
"""
Generate a PypeItImage from a list of images
This may also generate the ivar, crmask, rn2img and mask
Args:
bias (:class:`pypeit.images.buildimage.BiasImage`, optional): Bias image
flatimages (:class:`pypeit.flatfield.FlatImages`, optional): For flat fielding
dark (:class:`pypeit.images.buildimage.DarkImage`, optional): Dark image
slits (:class:`pypeit.slittrace.SlitTraceSet`, optional): Slit object
sigma_clip (bool, optional):
Perform sigma clipping
sigrej (int or float, optional): Rejection threshold for sigma clipping.
Code defaults to determining this automatically based on the number of images provided.
maxiters (int, optional):
Number of iterations for the clipping
bpm (`numpy.ndarray`_, optional):
Bad pixel mask. Held in ImageMask
ignore_saturation (:obj:`bool`, optional):
If True, turn off the saturation flag in the individual images before stacking
This avoids having such values set to 0 which for certain images (e.g. flat calibrations)
can have unintended consequences.
combine_method (str):
Method to combine images
Allowed options are 'weightmean', 'median'
Returns:
:class:`pypeit.images.pypeitimage.PypeItImage`:
"""
# Loop on the files
nimages = len(self.files)
lampstat = []
for kk, ifile in enumerate(self.files):
# Load raw image
rawImage = rawimage.RawImage(ifile, self.spectrograph, self.det)
# Process
pypeitImage = rawImage.process(self.par, bias=bias, bpm=bpm, dark=dark,
flatimages=flatimages, slits=slits)
#embed(header='96 of combineimage')
# Are we all done?
if nimages == 1:
return pypeitImage
elif kk == 0:
# Get ready
shape = (nimages, pypeitImage.image.shape[0], pypeitImage.image.shape[1])
img_stack = np.zeros(shape)
ivar_stack= np.zeros(shape)
rn2img_stack = np.zeros(shape)
crmask_stack = np.zeros(shape, dtype=bool)
# Mask
bitmask = imagebitmask.ImageBitMask()
mask_stack = np.zeros(shape, bitmask.minimum_dtype(asuint=True))
# Grab the lamp status
lampstat += [self.spectrograph.get_lamps_status(pypeitImage.rawheadlist)]
# Process
img_stack[kk,:,:] = pypeitImage.image
# Construct raw variance image and turn into inverse variance
if pypeitImage.ivar is not None:
ivar_stack[kk, :, :] = pypeitImage.ivar
else:
ivar_stack[kk, :, :] = 1.
# Mask cosmic rays
if pypeitImage.crmask is not None:
crmask_stack[kk, :, :] = pypeitImage.crmask
# Read noise squared image
if pypeitImage.rn2img is not None:
rn2img_stack[kk, :, :] = pypeitImage.rn2img
# Final mask for this image
# TODO This seems kludgy to me. Why not just pass ignore_saturation to process_one and ignore the saturation
# when the mask is actually built, rather than untoggling the bit here
if ignore_saturation: # Important for calibrations as we don't want replacement by 0
indx = pypeitImage.bitmask.flagged(pypeitImage.fullmask, flag=['SATURATION'])
pypeitImage.fullmask[indx] = pypeitImage.bitmask.turn_off(
pypeitImage.fullmask[indx], 'SATURATION')
mask_stack[kk, :, :] = pypeitImage.fullmask
# Check that the lamps being combined are all the same:
if not lampstat[1:] == lampstat[:-1]:
msgs.warn("The following files contain different lamp status")
# Get the longest strings
maxlen = max([len("Filename")]+[len(os.path.split(x)[1]) for x in self.files])
maxlmp = max([len("Lamp status")]+[len(x) for x in lampstat])
strout = "{0:" + str(maxlen) + "} {1:s}"
# Print the messages
print(msgs.indent() + '-'*maxlen + " " + '-'*maxlmp)
print(msgs.indent() + strout.format("Filename", "Lamp status"))
print(msgs.indent() + '-'*maxlen + " " + '-'*maxlmp)
for ff, file in enumerate(self.files):
print(msgs.indent() + strout.format(os.path.split(file)[1], " ".join(lampstat[ff].split("_"))))
print(msgs.indent() + '-'*maxlen + " " + '-'*maxlmp)
# Coadd them
weights = np.ones(nimages)/float(nimages)
img_list = [img_stack]
var_stack = utils.inverse(ivar_stack)
var_list = [var_stack, rn2img_stack]
if combine_method == 'weightmean':
img_list_out, var_list_out, gpm, nused = combine.weighted_combine(
weights, img_list, var_list, (mask_stack == 0),
sigma_clip=sigma_clip, sigma_clip_stack=img_stack, sigrej=sigrej, maxiters=maxiters)
elif combine_method == 'median':
img_list_out = [np.median(img_stack, axis=0)]
var_list_out = [np.median(var_stack, axis=0)]
var_list_out += [np.median(rn2img_stack, axis=0)]
gpm = np.ones_like(img_list_out[0], dtype='bool')
else:
msgs.error("Bad choice for combine. Allowed options are 'median', 'weightmean'.")
# Build the last one
final_pypeitImage = pypeitimage.PypeItImage(img_list_out[0],
ivar=utils.inverse(var_list_out[0]),
bpm=pypeitImage.bpm,
rn2img=var_list_out[1],
crmask=np.logical_not(gpm),
detector=pypeitImage.detector,
PYP_SPEC=pypeitImage.PYP_SPEC)
# Internals
final_pypeitImage.rawheadlist = pypeitImage.rawheadlist
final_pypeitImage.process_steps = pypeitImage.process_steps
nonlinear_counts = self.spectrograph.nonlinear_counts(pypeitImage.detector,
apply_gain=self.par['apply_gain'])
final_pypeitImage.build_mask(saturation=nonlinear_counts)
# Return
return final_pypeitImage
def reduce(self, pseudo_dict, show=None, show_peaks=None):
"""
..todo.. Please document me
Args:
pseudo_dict:
show:
show_peaks:
Returns:
"""
show = self.show if show is None else show
show_peaks = self.show_peaks if show_peaks is None else show_peaks
sciImage = pypeitimage.PypeItImage(image=pseudo_dict['imgminsky'],
ivar=pseudo_dict['sciivar'],
bpm=np.zeros_like(pseudo_dict['inmask'].astype(int)), # Dummy bpm
rn2img=np.zeros_like(pseudo_dict['inmask']).astype(float), # Dummy rn2img
crmask=np.invert(pseudo_dict['inmask'].astype(bool)))
sciImage.detector = self.stack_dict['detectors'][0]
#
slitmask_pseudo = pseudo_dict['slits'].slit_img()
sciImage.build_mask(slitmask=slitmask_pseudo)
# Make changes to parset specific to 2d coadds
parcopy = copy.deepcopy(self.par)
parcopy['reduce']['findobj']['trace_npoly'] = 3 # Low order traces since we are rectified
#parcopy['calibrations']['save_masters'] = False
#parcopy['scienceimage']['find_extrap_npoly'] = 1 # Use low order for trace extrapolation
# Build the Calibrate object
caliBrate = calibrations.Calibrations(None, self.par['calibrations'], self.spectrograph, None)
caliBrate.slits = pseudo_dict['slits']
redux=reduce.Reduce.get_instance(sciImage, self.spectrograph, parcopy, caliBrate,
'science_coadd2d', ir_redux=self.ir_redux, det=self.det, show=show)
#redux=reduce.Reduce.get_instance(sciImage, self.spectrograph, parcopy, pseudo_dict['slits'],
# None, None, 'science_coadd2d', ir_redux=self.ir_redux, det=self.det, show=show)
# Set the tilts and waveimg attributes from the psuedo_dict here, since we generate these dynamically from fits
# normally, but this is not possible for coadds
redux.tilts = pseudo_dict['tilts']
redux.waveimg = pseudo_dict['waveimg']
redux.binning = self.binning
# Masking
# TODO: Treat the masking of the slits objects
# from every exposure, come up with an aggregate mask (if it is masked on one slit,
# mask the slit for all) and that should be propagated into the slits object in the psuedo_dict
slits = self.stack_dict['slits_list'][0]
reduce_bpm = (slits.mask > 0) & (np.invert(slits.bitmask.flagged(
slits.mask, flag=slits.bitmask.exclude_for_reducing)))
redux.reduce_bpm = reduce_bpm
if show:
redux.show('image', image=pseudo_dict['imgminsky']*(sciImage.fullmask == 0), chname = 'imgminsky', slits=True, clear=True)
# TODO:
# Object finding, this appears inevitable for the moment, since we need to be able to call find_objects
# outside of reduce. I think the solution here is to create a method in reduce for that performs the modified
# 2d coadd reduce
sobjs_obj, nobj, skymask_init = redux.find_objects(
sciImage.image, show_peaks=show_peaks,
manual_extract_dict=self.par['reduce']['extraction']['manual'].dict_for_objfind())
# Local sky-subtraction
global_sky_pseudo = np.zeros_like(pseudo_dict['imgminsky']) # No global sky for co-adds since we go straight to local
skymodel_pseudo, objmodel_pseudo, ivarmodel_pseudo, outmask_pseudo, sobjs = redux.local_skysub_extract(
global_sky_pseudo, sobjs_obj, spat_pix=pseudo_dict['spat_img'], model_noise=False,
show_profile=show, show=show)
if self.ir_redux:
sobjs.purge_neg()
# TODO: Removed this, but I'm not sure that's what you want...
# # Add the information about the fixed wavelength grid to the sobjs
# for spec in sobjs:
# idx = spec.slit_orderindx
# # Fill
# spec.BOX_WAVE_GRID_MASK, spec.OPT_WAVE_GRID_MASK = [pseudo_dict['wave_mask'][:,idx]]*2
# spec.BOX_WAVE_GRID, spec.OPT_WAVE_GRID = [pseudo_dict['wave_mid'][:,idx]]*2
# spec.BOX_WAVE_GRID_MIN, spec.OPT_WAVE_GRID_MIN = [pseudo_dict['wave_min'][:,idx]]*2
# spec.BOX_WAVE_GRID_MAX, spec.OPT_WAVE_GRID_MAX = [pseudo_dict['wave_max'][:,idx]]*2
# Add the rest to the pseudo_dict
pseudo_dict['skymodel'] = skymodel_pseudo
pseudo_dict['objmodel'] = objmodel_pseudo
pseudo_dict['ivarmodel'] = ivarmodel_pseudo
pseudo_dict['outmask'] = outmask_pseudo
pseudo_dict['sobjs'] = sobjs
self.pseudo_dict=pseudo_dict
return pseudo_dict['imgminsky'], pseudo_dict['sciivar'], skymodel_pseudo, \
objmodel_pseudo, ivarmodel_pseudo, outmask_pseudo, sobjs, sciImage.detector, pseudo_dict['slits'], \
pseudo_dict['tilts'], pseudo_dict['waveimg']
def run(self, bias=None, flatimages=None, ignore_saturation=False, sigma_clip=True,
bpm=None, sigrej=None, maxiters=5, slits=None, dark=None, combine_method='mean',
mosaic=False):
r"""
Process and combine all images.
All processing is performed by the
:class:`~pypeit.images.rawimage.RawImage` class; see
:func:`~pypeit.images.rawimage.RawImage.process`.
If there is only one file (see :attr:`files`), this simply processes the
file and returns the result.
If there are multiple files, all the files are processed and the
processed images are combined based on the ``combine_method``, where the
options are:
- 'mean': If ``sigma_clip`` is True, this is a sigma-clipped mean;
otherwise, this is a simple average. The combination is done
using :func:`~pypeit.core.combine.weighted_combine`.
- 'median': This is a simple masked median (using
`numpy.ma.median`_).
The errors in the image are also propagated through the stacking
procedure; however, this isn't a simple propagation of the inverse
variance arrays. The image processing produces arrays with individual
components used to construct the variance model for an individual frame.
See :ref:`image_proc` and :func:`~pypeit.procimg.variance_model` for a
description of these arrays. Briefly, the relevant arrays are the
readnoise variance (:math:`V_{\rm rn}`), the "processing" variance
(:math:`V_{\rm proc}`), and the image scaling (i.e., the flat-field
correction) (:math:`s`). The variance calculation for the stacked image
directly propagates the error in these. For example, the propagated
processing variance (modulo the masking) is:
.. math::
V_{\rm proc,stack} = \frac{\sum_i s_i^2 V_{{\rm
proc},i}}\frac{s_{\rm stack}^2}
where :math:`s_{\rm stack}` is the combined image scaling array,
combined in the same way as the image data are combined. This ensures
that the reconstruction of the uncertainty in the combined image
calculated using :func:`~pypeit.procimg.variance_model` accurately
includes, e.g., the processing uncertainty.
The uncertainty in the combined image, however, recalculates the
variance model, using the combined image (which should have less noise)
to set the Poisson statistics. The same parameters used when processing
the individual frames are applied to the combined frame; see
:func:`~pypeit.images.rawimage.RawImage.build_ivar`. This calculation
is then the equivalent of when the observed counts are replaced by the
model object and sky counts during sky subtraction and spectral
extraction.
Bitmasks from individual frames in the stack are *not* propagated to the
combined image, except to indicate when a pixel was masked for all
images in the stack (cf., ``ignore_saturation``). Additionally, the
instrument-specific bad-pixel mask, see the
:func:`~pypeit.spectrographs.spectrograph.Spectrograph.bpm` method for
each instrument subclass, saturated-pixel mask, and other default mask
bits (e.g., NaN and non-positive inverse variance values) are all
propagated to the combined-image mask; see
:func:`~pypeit.images.pypeitimage.PypeItImage.build_mask`.
.. warning::
All image processing of the data in :attr:`files` *must* result
in images of the same shape.
Args:
bias (:class:`~pypeit.images.buildimage.BiasImage`, optional):
Bias image for bias subtraction; passed directly to
:func:`~pypeit.images.rawimage.RawImage.process` for all images.
flatimages (:class:`~pypeit.flatfield.FlatImages`, optional):
Flat-field images for flat fielding; passed directly to
:func:`~pypeit.images.rawimage.RawImage.process` for all images.
ignore_saturation (:obj:`bool`, optional):
If True, turn off the saturation flag in the individual images
before stacking. This avoids having such values set to 0, which
for certain images (e.g. flat calibrations) can have unintended
consequences.
sigma_clip (:obj:`bool`, optional):
When ``combine_method='mean'``, perform a sigma-clip the data;
see :func:`~pypeit.core.combine.weighted_combine`.
bpm (`numpy.ndarray`_, optional):
Bad pixel mask; passed directly to
:func:`~pypeit.images.rawimage.RawImage.process` for all images.
sigrej (:obj:`float`, optional):
When ``combine_method='mean'``, this sets the sigma-rejection
thresholds used when sigma-clipping the image combination.
Ignored if ``sigma_clip`` is False. If None and ``sigma_clip``
is True, the thresholds are determined automatically based on
the number of images provided; see
:func:`~pypeit.core.combine.weighted_combine``.
maxiters (:obj:`int`, optional):
When ``combine_method='mean'``) and sigma-clipping
(``sigma_clip`` is True), this sets the maximum number of
rejection iterations. If None, rejection iterations continue
until no more data are rejected; see
:func:`~pypeit.core.combine.weighted_combine``.
slits (:class:`~pypeit.slittrace.SlitTraceSet`, optional):
Slit edge trace locations; passed directly to
:func:`~pypeit.images.rawimage.RawImage.process` for all images.
dark (:class:`~pypeit.images.buildimage.DarkImage`, optional):
Dark-current image; passed directly to
:func:`~pypeit.images.rawimage.RawImage.process` for all images.
combine_method (:obj:`str`, optional):
Method used to combine images. Must be ``'mean'`` or
``'median'``; see above.
mosaic (:obj:`bool`, optional):
If multiple detectors are being processes, mosaic them into a
single image. See
:func:`~pypeit.images.rawimage.RawImage.process`.
Returns:
:class:`~pypeit.images.pypeitimage.PypeItImage`: The combination of
all the processed images.
"""
# Check the input (i.e., bomb out *before* it does any processing)
if self.nfiles == 0:
msgs.error('Object contains no files to process!')
if self.nfiles > 1 and combine_method not in ['mean', 'median']:
msgs.error(f'Unknown image combination method, {combine_method}. Must be '
'"mean" or "median".')
# If not provided, generate the bpm for this spectrograph and detector.
# Regardless of the file used, this must result in the same bpm, so we
# just use the first one.
# TODO: Why is this done here? It's the same thing as what's done if
# bpm is not passed to RawImage.process...
# if bpm is None:
# bpm = self.spectrograph.bpm(self.files[0], self.det)
# Loop on the files
for kk, ifile in enumerate(self.files):
# Load raw image
rawImage = rawimage.RawImage(ifile, self.spectrograph, self.det)
# Process
pypeitImage = rawImage.process(self.par, bias=bias, bpm=bpm, dark=dark,
flatimages=flatimages, slits=slits, mosaic=mosaic)
if self.nfiles == 1:
# Only 1 file, so we're done
pypeitImage.files = self.files
return pypeitImage
elif kk == 0:
# Allocate arrays to collect data for each frame
shape = (self.nfiles,) + pypeitImage.shape
img_stack = np.zeros(shape, dtype=float)
scl_stack = np.ones(shape, dtype=float)
rn2img_stack = np.zeros(shape, dtype=float)
basev_stack = np.zeros(shape, dtype=float)
gpm_stack = np.zeros(shape, dtype=bool)
lampstat = [None]*self.nfiles
exptime = np.zeros(self.nfiles, dtype=float)
# Save the lamp status
lampstat[kk] = self.spectrograph.get_lamps_status(pypeitImage.rawheadlist)
# Save the exposure time to check if it's consistent for all images.
exptime[kk] = pypeitImage.exptime
# Processed image
img_stack[kk] = pypeitImage.image
# Get the count scaling
if pypeitImage.img_scale is not None:
scl_stack[kk] = pypeitImage.img_scale
# Read noise squared image
if pypeitImage.rn2img is not None:
rn2img_stack[kk] = pypeitImage.rn2img * scl_stack[kk]**2
# Processing variance image
if pypeitImage.base_var is not None:
basev_stack[kk] = pypeitImage.base_var * scl_stack[kk]**2
# Final mask for this image
# TODO: This seems kludgy to me. Why not just pass ignore_saturation
# to process_one and ignore the saturation when the mask is actually
# built, rather than untoggling the bit here?
if ignore_saturation: # Important for calibrations as we don't want replacement by 0
pypeitImage.update_mask('SATURATION', action='turn_off')
# Get a simple boolean good-pixel mask for all the unmasked pixels
gpm_stack[kk] = pypeitImage.select_flag(invert=True)
# Check that the lamps being combined are all the same:
if not lampstat[1:] == lampstat[:-1]:
msgs.warn("The following files contain different lamp status")
# Get the longest strings
maxlen = max([len("Filename")]+[len(os.path.split(x)[1]) for x in self.files])
maxlmp = max([len("Lamp status")]+[len(x) for x in lampstat])
strout = "{0:" + str(maxlen) + "} {1:s}"
# Print the messages
print(msgs.indent() + '-'*maxlen + " " + '-'*maxlmp)
print(msgs.indent() + strout.format("Filename", "Lamp status"))
print(msgs.indent() + '-'*maxlen + " " + '-'*maxlmp)
for ff, file in enumerate(self.files):
print(msgs.indent()
+ strout.format(os.path.split(file)[1], " ".join(lampstat[ff].split("_"))))
print(msgs.indent() + '-'*maxlen + " " + '-'*maxlmp)
# Do a similar check for exptime
if np.any(np.absolute(np.diff(exptime)) > 0):
# TODO: This should likely throw an error instead!
msgs.warn('Exposure time is not consistent for all images being combined! '
'Using the average.')
comb_texp = np.mean(exptime)
else:
comb_texp = exptime[0]
# Coadd them
if combine_method == 'mean':
weights = np.ones(self.nfiles, dtype=float)/self.nfiles
img_list_out, var_list_out, gpm, nstack \
= combine.weighted_combine(weights,
[img_stack, scl_stack], # images to stack
[rn2img_stack, basev_stack], # variances to stack
gpm_stack, sigma_clip=sigma_clip,
sigma_clip_stack=img_stack, # clipping based on img
sigrej=sigrej, maxiters=maxiters)
comb_img, comb_scl = img_list_out
comb_rn2, comb_basev = var_list_out
comb_rn2[gpm] /= comb_scl[gpm]**2
comb_basev[gpm] /= comb_scl[gpm]**2
elif combine_method == 'median':
bpm_stack = np.logical_not(gpm_stack)
nstack = np.sum(gpm_stack, axis=0)
gpm = nstack > 0
comb_img = np.ma.median(np.ma.MaskedArray(img_stack, mask=bpm_stack),axis=0).filled(0.)
# TODO: I'm not sure if this is right. Maybe we should just take
# the masked average scale instead?
comb_scl = np.ma.median(np.ma.MaskedArray(scl_stack, mask=bpm_stack),axis=0).filled(0.)
# First calculate the error in the sum. The variance is set to 0
# for pixels masked in all images.
comb_rn2 = np.ma.sum(np.ma.MaskedArray(rn2img_stack, mask=bpm_stack),axis=0).filled(0.)
comb_basev = np.ma.sum(np.ma.MaskedArray(basev_stack, mask=bpm_stack),axis=0).filled(0.)
# Convert to standard error in the median (pi/2 factor relates standard variance
# in mean (sum(variance_i)/n^2) to standard variance in median)
comb_rn2[gpm] *= np.pi/2/nstack[gpm]**2/comb_scl[gpm]**2
comb_basev[gpm] *= np.pi/2/nstack[gpm]**2/comb_scl[gpm]**2
else:
# NOTE: Given the check at the beginning of the function, the code
# should *never* make it here.
msgs.error("Bad choice for combine. Allowed options are 'median', 'mean'.")
# Recompute the inverse variance using the combined image
comb_var = procimg.variance_model(comb_basev,
counts=comb_img if self.par['shot_noise'] else None,
count_scale=comb_scl,
noise_floor=self.par['noise_floor'])
# Build the combined image
comb = pypeitimage.PypeItImage(image=comb_img, ivar=utils.inverse(comb_var), nimg=nstack,
amp_img=pypeitImage.amp_img, det_img=pypeitImage.det_img,
rn2img=comb_rn2, base_var=comb_basev, img_scale=comb_scl,
bpm=np.logical_not(gpm).astype(np.uint8),
# NOTE: The detector is needed here so
# that we can get the dark current later.
detector=pypeitImage.detector,
PYP_SPEC=self.spectrograph.name,
units='e-' if self.par['apply_gain'] else 'ADU',
exptime=comb_texp, noise_floor=self.par['noise_floor'],
shot_noise=self.par['shot_noise'])
# Internals
# TODO: Do we need these?
comb.files = self.files
comb.rawheadlist = pypeitImage.rawheadlist
comb.process_steps = pypeitImage.process_steps
# Build the base level mask
comb.build_mask(saturation='default', mincounts='default')
# Flag all pixels with no contributions from any of the stacked images.
comb.update_mask('STCKMASK', indx=np.logical_not(gpm))
# Return
return comb
def run(self,
process_steps,
bias,
pixel_flat=None,
illum_flat=None,
ignore_saturation=False,
sigma_clip=True,
bpm=None,
sigrej=None,
maxiters=5):
"""
Generate a PypeItImage from a list of images
Mainly a wrapper to coadd2d.weighted_combine()
This may also generate the ivar, crmask, rn2img and mask
Args:
process_steps (list):
bias (np.ndarray or None):
Bias image or instruction
pixel_flat (np.ndarray, optional):
Flat image
illum_flat (np.ndarray, optional):
Illumination image
sigma_clip (bool, optional):
Perform sigma clipping
sigrej (int or float, optional): Rejection threshold for sigma clipping.
Code defaults to determining this automatically based on the number of images provided.
maxiters (int, optional):
Number of iterations for the clipping
bpm (np.ndarray, optional):
Bad pixel mask. Held in ImageMask
ignore_saturation (bool, optional):
If True, turn off the saturation flag in the individual images before stacking
This avoids having such values set to 0 which for certain images (e.g. flat calibrations)
can have unintended consequences.
Returns:
:class:`pypeit.images.pypeitimage.PypeItImage`:
"""
# Loop on the files
nimages = len(self.files)
for kk, ifile in enumerate(self.files):
# Process a single image
pypeitImage = self.process_one(ifile,
process_steps,
bias,
pixel_flat=pixel_flat,
illum_flat=illum_flat,
bpm=bpm)
# Are we all done?
if len(self.files) == 1:
return pypeitImage
elif kk == 0:
# Get ready
shape = (nimages, pypeitImage.bpm.shape[0],
pypeitImage.bpm.shape[1])
img_stack = np.zeros(shape)
ivar_stack = np.zeros(shape)
rn2img_stack = np.zeros(shape)
crmask_stack = np.zeros(shape, dtype=bool)
# Mask
bitmask = maskimage.ImageBitMask()
mask_stack = np.zeros(shape,
bitmask.minimum_dtype(asuint=True))
# Process
img_stack[kk, :, :] = pypeitImage.image
# Construct raw variance image and turn into inverse variance
if pypeitImage.ivar is not None:
ivar_stack[kk, :, :] = pypeitImage.ivar
else:
ivar_stack[kk, :, :] = 1.
# Mask cosmic rays
if pypeitImage.crmask is not None:
crmask_stack[kk, :, :] = pypeitImage.crmask
# Read noise squared image
if pypeitImage.rn2img is not None:
rn2img_stack[kk, :, :] = pypeitImage.rn2img
# Final mask for this image
# TODO This seems kludgy to me. Why not just pass ignore_saturation to process_one and ignore the saturation
# when the mask is actually built, rather than untoggling the bit here
if ignore_saturation: # Important for calibrations as we don't want replacement by 0
indx = pypeitImage.bitmask.flagged(pypeitImage.mask,
flag=['SATURATION'])
pypeitImage.mask[indx] = pypeitImage.bitmask.turn_off(
pypeitImage.mask[indx], 'SATURATION')
mask_stack[kk, :, :] = pypeitImage.mask
# Coadd them
weights = np.ones(nimages) / float(nimages)
img_list = [img_stack]
var_stack = utils.inverse(ivar_stack)
var_list = [var_stack, rn2img_stack]
img_list_out, var_list_out, outmask, nused = combine.weighted_combine(
weights,
img_list,
var_list, (mask_stack == 0),
sigma_clip=sigma_clip,
sigma_clip_stack=img_stack,
sigrej=sigrej,
maxiters=maxiters)
# Build the last one
final_pypeitImage = pypeitimage.PypeItImage(
img_list_out[0],
ivar=utils.inverse(var_list_out[0]),
bpm=pypeitImage.bpm,
rn2img=var_list_out[1],
crmask=np.invert(outmask),
binning=pypeitImage.binning)
nonlinear_counts = self.spectrograph.nonlinear_counts(
self.det, apply_gain='apply_gain' in process_steps)
final_pypeitImage.build_mask(
final_pypeitImage.image,
final_pypeitImage.ivar,
saturation=
nonlinear_counts, #self.spectrograph.detector[self.det-1]['saturation'],
mincounts=self.spectrograph.detector[self.det - 1]['mincounts'])
# Return
return final_pypeitImage
