def testGetValConvienceFunction(self):
"""Test the getval convience function in both the fits and
stpyfits namespace."""
val = stpyfits.getval(self.data('cdva2.fits'), 'NAXIS', 0)
val1 = fits.getval(self.data('cdva2.fits'), 'NAXIS', 0)
assert_equal(val, 2)
assert_equal(val1, 0)
def testGetValConvienceFunction(self):
"""Test the getval convience function in both the fits and
stpyfits namespace."""
val = stpyfits.getval(self.data('cdva2.fits'), 'NAXIS', 0)
val1 = fits.getval(self.data('cdva2.fits'), 'NAXIS', 0)
assert_equal(val, 2)
assert_equal(val1, 0)
def testGetvalExtensionHDU(self):
"""Regression test for an issue that came up with the fact that
ImageHDU has a different argument signature from PrimaryHDU.
"""
data = np.ones((100, 100))
hdu = stpyfits.ImageHDU(data=data)
hdu.header['PIXVALUE'] = 1
hdu.header['FOO'] = 'test'
hdul = stpyfits.HDUList([stpyfits.PrimaryHDU(), hdu])
hdul.writeto(self.temp('test.fits'))
assert_equal(stpyfits.getval(self.temp('test.fits'), 'FOO', ext=1),
'test')
def testGetvalExtensionHDU(self):
"""Regression test for an issue that came up with the fact that
ImageHDU has a different argument signature from PrimaryHDU.
"""
data = np.ones((100, 100))
hdu = stpyfits.ImageHDU(data=data)
hdu.header['PIXVALUE'] = 1
hdu.header['FOO'] = 'test'
hdul = stpyfits.HDUList([stpyfits.PrimaryHDU(), hdu])
hdul.writeto(self.temp('test.fits'))
assert_equal(stpyfits.getval(self.temp('test.fits'), 'FOO', ext=1),
'test')
def clean(input,
suffix,
stat="pmode1",
maxiter=15,
sigrej=2.0,
lower=None,
upper=None,
binwidth=0.3,
mask1=None,
mask2=None,
dqbits=None,
rpt_clean=0,
atol=0.01,
clobber=False,
verbose=True):
r"""Remove horizontal stripes from ACS WFC post-SM4 data.
Parameters
----------
input : str or list of str
Input filenames in one of these formats:
* a Python list of filenames
* a partial filename with wildcards ('\*flt.fits')
* filename of an ASN table ('j12345670_asn.fits')
* an at-file (``@input``)
suffix : str
The string to use to add to each input file name to
indicate an output product. This string will be appended
to the suffix in each input filename to create the
new output filename. For example, setting `suffix='csck'`
will create '\*_csck.fits' images.
stat : { 'pmode1', 'pmode2', 'mean', 'mode', 'median', 'midpt' } (Default = 'pmode1')
Specifies the statistics to be used for computation of the
background in image rows:
* 'pmode1' - SEXTRACTOR-like mode estimate based on a
modified `Pearson's rule <https://en.wikipedia.org/wiki/Nonparametric_skew#Pearson.27s_rule>`_:
``2.5*median-1.5*mean``;
* 'pmode2' - mode estimate based on
`Pearson's rule <https://en.wikipedia.org/wiki/Nonparametric_skew#Pearson.27s_rule>`_:
``3*median-2*mean``;
* 'mean' - the mean of the distribution of the "good" pixels (after
clipping, masking, etc.);
* 'mode' - the mode of the distribution of the "good" pixels;
* 'median' - the median of the distribution of the "good" pixels;
* 'midpt' - estimate of the median of the distribution of the "good"
pixels based on an algorithm similar to IRAF's ``imagestats`` task
(``CDF(midpt)=1/2``).
.. note::
The midpoint and mode are computed in two passes through the
image. In the first pass the standard deviation of the pixels
is calculated and used with the *binwidth* parameter to compute
the resolution of the data histogram. The midpoint is estimated
by integrating the histogram and computing by interpolation
the data value at which exactly half the pixels are below that
data value and half are above it. The mode is computed by
locating the maximum of the data histogram and fitting the peak
by parabolic interpolation.
maxiter : int
This parameter controls the maximum number of iterations
to perform when computing the statistics used to compute the
row-by-row corrections.
sigrej : float
This parameters sets the sigma level for the rejection applied
during each iteration of statistics computations for the
row-by-row corrections.
lower : float, None (Default = None)
Lower limit of usable pixel values for computing the background.
This value should be specified in the units of the input image(s).
upper : float, None (Default = None)
Upper limit of usable pixel values for computing the background.
This value should be specified in the units of the input image(s).
binwidth : float (Default = 0.1)
Histogram's bin width, in sigma units, used to sample the
distribution of pixel brightness values in order to compute the
background statistics. This parameter is aplicable *only* to *stat*
parameter values of `'mode'` or `'midpt'`.
clobber : bool
Specify whether or not to 'clobber' (delete then replace)
previously generated products with the same names.
mask1 : str, numpy.ndarray, None, or list of these types
Mask images for ``SCI,1``, one for each input file.
Pixels with zero values will be masked out, in addition to clipping.
mask2 : str, numpy.ndarray, None, or list of these types
Mask images for ``SCI,2``, one for each input file.
Pixels with zero values will be masked out, in addition to clipping.
This is not used for subarrays.
dqbits : int, str, None (Default = None)
Integer sum of all the DQ bit values from the input image's DQ array
that should be considered "good" when building masks for de-striping
computations. For example, if pixels in the DQ array can be
combinations of 1, 2, 4, and 8 flags and one wants to consider
DQ "defects" having flags 2 and 4 as being acceptable for de-striping
computations, then `dqbits` should be set to 2+4=6. Then a DQ pixel
having values 2,4, or 6 will be considered a good pixel, while a DQ
pixel with a value, e.g., 1+2=3, 4+8=12, etc. will be flagged
as a "bad" pixel.
Alternatively, one can enter a comma- or '+'-separated list of
integer bit flags that should be added to obtain the final
"good" bits. For example, both ``4,8`` and ``4+8`` are equivalent to
setting `dqbits` to 12.
| Set `dqbits` to 0 to make *all* non-zero pixels in the DQ
mask to be considered "bad" pixels, and the corresponding image
pixels not to be used for de-striping computations.
| Default value (`None`) will turn off the use of image's DQ array
for de-striping computations.
| In order to reverse the meaning of the `dqbits`
parameter from indicating values of the "good" DQ flags
to indicating the "bad" DQ flags, prepend '~' to the string
value. For example, in order not to use pixels with
DQ flags 4 and 8 for sky computations and to consider
as "good" all other pixels (regardless of their DQ flag),
set `dqbits` to ``~4+8``, or ``~4,8``. To obtain the
same effect with an `int` input value (except for 0),
enter -(4+8+1)=-9. Following this convention,
a `dqbits` string value of ``'~0'`` would be equivalent to
setting ``dqbits=None``.
.. note::
DQ masks (if used), *will be* combined with user masks specified
in the `mask1` and `mask2` parameters (if any).
rpt_clean : int
An integer indicating how many *additional* times stripe cleaning
should be performed on the input image. Default = 0.
atol : float, None
The threshold for maximum absolute value of bias stripe correction
below which repeated cleanings can stop. When `atol` is `None`
cleaning will be repeated `rpt_clean` number of times.
Default = 0.01 [e].
verbose : bool
Print informational messages. Default = True.
"""
from stsci.tools import parseinput # Optional package dependency
flist = parseinput.parseinput(input)[0]
if isinstance(mask1, str):
mlist1 = parseinput.parseinput(mask1)[0]
elif isinstance(mask1, np.ndarray):
mlist1 = [mask1.copy()]
elif mask1 is None:
mlist1 = []
elif isinstance(mask1, list):
mlist1 = []
for m in mask1:
if isinstance(m, np.ndarray):
mlist1.append(m.copy())
elif isinstance(m, str):
mlist1 += parseinput.parseinput(m)[0]
else:
raise TypeError("'mask1' must be a list of str or "
"numpy.ndarray values.")
else:
raise TypeError("'mask1' must be either a str, or a "
"numpy.ndarray, or a list of the two type of "
"values.")
if isinstance(mask2, str):
mlist2 = parseinput.parseinput(mask2)[0]
elif isinstance(mask2, np.ndarray):
mlist2 = [mask2.copy()]
elif mask2 is None:
mlist2 = []
elif isinstance(mask2, list):
mlist2 = []
for m in mask2:
if isinstance(m, np.ndarray):
mlist2.append(m.copy())
elif isinstance(m, str):
mlist2 += parseinput.parseinput(m)[0]
else:
raise TypeError("'mask2' must be a list of str or "
"numpy.ndarray values.")
else:
raise TypeError("'mask2' must be either a str or a "
"numpy.ndarray, or a list of the two type of "
"values.")
n_input = len(flist)
n_mask1 = len(mlist1)
n_mask2 = len(mlist2)
if n_input == 0:
raise ValueError("No input file(s) provided or "
"the file(s) do not exist")
if n_mask1 == 0:
mlist1 = [None] * n_input
elif n_mask1 != n_input:
raise ValueError('Insufficient masks for [SCI,1]')
if n_mask2 == 0:
mlist2 = [None] * n_input
elif n_mask2 != n_input:
raise ValueError('Insufficient masks for [SCI,2]')
for image, maskfile1, maskfile2 in zip(flist, mlist1, mlist2):
# Skip processing pre-SM4 images
if (fits.getval(image, 'EXPSTART') <= SM4_MJD):
LOG.warning(f'{image} is pre-SM4. Skipping...')
continue
# Data must be in ELECTRONS
if (fits.getval(image, 'BUNIT', ext=1) != 'ELECTRONS'):
LOG.warning(f'{image} is not in ELECTRONS. Skipping...')
continue
# Skip processing CTECORR-ed images
if (fits.getval(image, 'PCTECORR') == 'COMPLETE'):
LOG.warning(f'{image} already has PCTECORR applied. Skipping...')
continue
# generate output filename for each input based on specification
# of the output suffix
output = image.replace('.fits', '_' + suffix + '.fits')
LOG.info('Processing ' + image)
# verify masks defined (or not) simultaneously:
if (fits.getval(image, 'CCDAMP') == 'ABCD'
and ((mask1 is not None and mask2 is None) or
(mask1 is None and mask2 is not None))):
raise ValueError("Both 'mask1' and 'mask2' must be specified "
"or not specified together.")
maskdata = _read_mask(maskfile1, maskfile2)
perform_correction(image,
output,
stat=stat,
maxiter=maxiter,
sigrej=sigrej,
lower=lower,
upper=upper,
binwidth=binwidth,
mask=maskdata,
dqbits=dqbits,
rpt_clean=rpt_clean,
atol=atol,
clobber=clobber,
verbose=verbose)
LOG.info(output + ' created')
def clean(input, suffix, stat="pmode1", maxiter=15, sigrej=2.0,
lower=None, upper=None, binwidth=0.3,
mask1=None, mask2=None, dqbits=None,
rpt_clean=0, atol=0.01, clobber=False, verbose=True):
r"""Remove horizontal stripes from ACS WFC post-SM4 data.
Parameters
----------
input : str or list of str
Input filenames in one of these formats:
* a Python list of filenames
* a partial filename with wildcards ('\*flt.fits')
* filename of an ASN table ('j12345670_asn.fits')
* an at-file (``@input``)
suffix : str
The string to use to add to each input file name to
indicate an output product. This string will be appended
to the suffix in each input filename to create the
new output filename. For example, setting `suffix='csck'`
will create '\*_csck.fits' images.
stat : { 'pmode1', 'pmode2', 'mean', 'mode', 'median', 'midpt' } (Default = 'pmode1')
Specifies the statistics to be used for computation of the
background in image rows:
* 'pmode1' - SEXTRACTOR-like mode estimate based on a
modified `Pearson's rule <http://en.wikipedia.org/wiki/Nonparametric_skew#Pearson.27s_rule>`_:
``2.5*median-1.5*mean``;
* 'pmode2' - mode estimate based on
`Pearson's rule <http://en.wikipedia.org/wiki/Nonparametric_skew#Pearson.27s_rule>`_:
``3*median-2*mean``;
* 'mean' - the mean of the distribution of the "good" pixels (after
clipping, masking, etc.);
* 'mode' - the mode of the distribution of the "good" pixels;
* 'median' - the median of the distribution of the "good" pixels;
* 'midpt' - estimate of the median of the distribution of the "good"
pixels based on an algorithm similar to IRAF's ``imagestats`` task
(``CDF(midpt)=1/2``).
.. note::
The midpoint and mode are computed in two passes through the
image. In the first pass the standard deviation of the pixels
is calculated and used with the *binwidth* parameter to compute
the resolution of the data histogram. The midpoint is estimated
by integrating the histogram and computing by interpolation
the data value at which exactly half the pixels are below that
data value and half are above it. The mode is computed by
locating the maximum of the data histogram and fitting the peak
by parabolic interpolation.
maxiter : int
This parameter controls the maximum number of iterations
to perform when computing the statistics used to compute the
row-by-row corrections.
sigrej : float
This parameters sets the sigma level for the rejection applied
during each iteration of statistics computations for the
row-by-row corrections.
lower : float, None (Default = None)
Lower limit of usable pixel values for computing the background.
This value should be specified in the units of the input image(s).
upper : float, None (Default = None)
Upper limit of usable pixel values for computing the background.
This value should be specified in the units of the input image(s).
binwidth : float (Default = 0.1)
Histogram's bin width, in sigma units, used to sample the
distribution of pixel brightness values in order to compute the
background statistics. This parameter is aplicable *only* to *stat*
parameter values of `'mode'` or `'midpt'`.
clobber : bool
Specify whether or not to 'clobber' (delete then replace)
previously generated products with the same names.
mask1 : str, numpy.ndarray, None, or list of these types
Mask images for ``SCI,1``, one for each input file.
Pixels with zero values will be masked out, in addition to clipping.
mask2 : str, numpy.ndarray, None, or list of these types
Mask images for ``SCI,2``, one for each input file.
Pixels with zero values will be masked out, in addition to clipping.
This is not used for subarrays.
dqbits : int, str, None (Default = None)
Integer sum of all the DQ bit values from the input image's DQ array
that should be considered "good" when building masks for de-striping
computations. For example, if pixels in the DQ array can be
combinations of 1, 2, 4, and 8 flags and one wants to consider
DQ "defects" having flags 2 and 4 as being acceptable for de-striping
computations, then `dqbits` should be set to 2+4=6. Then a DQ pixel
having values 2,4, or 6 will be considered a good pixel, while a DQ
pixel with a value, e.g., 1+2=3, 4+8=12, etc. will be flagged
as a "bad" pixel.
Alternatively, one can enter a comma- or '+'-separated list of
integer bit flags that should be added to obtain the final
"good" bits. For example, both ``4,8`` and ``4+8`` are equivalent to
setting `dqbits` to 12.
| Set `dqbits` to 0 to make *all* non-zero pixels in the DQ
mask to be considered "bad" pixels, and the corresponding image
pixels not to be used for de-striping computations.
| Default value (`None`) will turn off the use of image's DQ array
for de-striping computations.
| In order to reverse the meaning of the `dqbits`
parameter from indicating values of the "good" DQ flags
to indicating the "bad" DQ flags, prepend '~' to the string
value. For example, in order not to use pixels with
DQ flags 4 and 8 for sky computations and to consider
as "good" all other pixels (regardless of their DQ flag),
set `dqbits` to ``~4+8``, or ``~4,8``. To obtain the
same effect with an `int` input value (except for 0),
enter -(4+8+1)=-9. Following this convention,
a `dqbits` string value of ``'~0'`` would be equivalent to
setting ``dqbits=None``.
.. note::
DQ masks (if used), *will be* combined with user masks specified
in the `mask1` and `mask2` parameters (if any).
rpt_clean : int
An integer indicating how many *additional* times stripe cleaning
should be performed on the input image. Default = 0.
atol : float, None
The threshold for maximum absolute value of bias stripe correction
below which repeated cleanings can stop. When `atol` is `None`
cleaning will be repeated `rpt_clean` number of times.
Default = 0.01 [e].
verbose : bool
Print informational messages. Default = True.
"""
from stsci.tools import parseinput # Optional package dependency
flist = parseinput.parseinput(input)[0]
if isinstance(mask1, str):
mlist1 = parseinput.parseinput(mask1)[0]
elif isinstance(mask1, np.ndarray):
mlist1 = [mask1.copy()]
elif mask1 is None:
mlist1 = []
elif isinstance(mask1, list):
mlist1 = []
for m in mask1:
if isinstance(m, np.ndarray):
mlist1.append(m.copy())
elif isinstance(m, str):
mlist1 += parseinput.parseinput(m)[0]
else:
raise TypeError("'mask1' must be a list of str or "
"numpy.ndarray values.")
else:
raise TypeError("'mask1' must be either a str, or a "
"numpy.ndarray, or a list of the two type of "
"values.")
if isinstance(mask2, str):
mlist2 = parseinput.parseinput(mask2)[0]
elif isinstance(mask2, np.ndarray):
mlist2 = [mask2.copy()]
elif mask2 is None:
mlist2 = []
elif isinstance(mask2, list):
mlist2 = []
for m in mask2:
if isinstance(m, np.ndarray):
mlist2.append(m.copy())
elif isinstance(m, str):
mlist2 += parseinput.parseinput(m)[0]
else:
raise TypeError("'mask2' must be a list of str or "
"numpy.ndarray values.")
else:
raise TypeError("'mask2' must be either a str or a "
"numpy.ndarray, or a list of the two type of "
"values.")
n_input = len(flist)
n_mask1 = len(mlist1)
n_mask2 = len(mlist2)
if n_input == 0:
raise ValueError("No input file(s) provided or "
"the file(s) do not exist")
if n_mask1 == 0:
mlist1 = [None] * n_input
elif n_mask1 != n_input:
raise ValueError('Insufficient masks for [SCI,1]')
if n_mask2 == 0:
mlist2 = [None] * n_input
elif n_mask2 != n_input:
raise ValueError('Insufficient masks for [SCI,2]')
for image, maskfile1, maskfile2 in zip(flist, mlist1, mlist2):
# Skip processing pre-SM4 images
if (fits.getval(image, 'EXPSTART') <= MJD_SM4):
LOG.warning('{0} is pre-SM4. Skipping...'.format(image))
continue
# Data must be in ELECTRONS
if (fits.getval(image, 'BUNIT', ext=1) != 'ELECTRONS'):
LOG.warning('{0} is not in ELECTRONS. Skipping...'.format(image))
continue
# Skip processing CTECORR-ed images
if (fits.getval(image, 'PCTECORR') == 'COMPLETE'):
LOG.warning('{0} already has PCTECORR applied. '
'Skipping...'.format(image))
continue
# generate output filename for each input based on specification
# of the output suffix
output = image.replace('.fits', '_' + suffix + '.fits')
LOG.info('Processing ' + image)
# verify masks defined (or not) simultaneously:
if (fits.getval(image, 'CCDAMP') == 'ABCD' and
((mask1 is not None and mask2 is None) or
(mask1 is None and mask2 is not None))):
raise ValueError("Both 'mask1' and 'mask2' must be specified "
"or not specified together.")
maskdata = _read_mask(maskfile1, maskfile2)
perform_correction(image, output, stat=stat, maxiter=maxiter,
sigrej=sigrej, lower=lower, upper=upper,
binwidth=binwidth, mask=maskdata, dqbits=dqbits,
rpt_clean=rpt_clean, atol=atol,
clobber=clobber, verbose=verbose)
LOG.info(output + ' created')