def build_mask(dqarr, bitvalue):
""" Builds a bit-mask from an input DQ array and a bitvalue flag"""
bitvalue = bitmask.interpret_bit_flags(bitvalue)
if bitvalue is None:
return (np.ones(dqarr.shape, dtype=np.uint32))
return np.logical_not(np.bitwise_and(dqarr, ~bitvalue)).astype(np.uint32)
def test_interpret_duplicate_flag_warning():
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
assert bitmask.interpret_bit_flags([2, 4, 4]) == 6
assert len(w)
assert issubclass(w[-1].category, UserWarning)
assert "Duplicate" in str(w[-1].message)
def _mergeUserMaskAndDQ(dq, mask, dqbits):
# Optional package dependency
try:
from stsci.tools.bitmask import (interpret_bit_flags,
bitfield_to_boolean_mask)
except ImportError:
from stsci.tools.bitmask import (
interpret_bits_value as interpret_bit_flags,
bitmask2mask as bitfield_to_boolean_mask
)
dqbits = interpret_bit_flags(dqbits)
if dqbits is None:
if mask is None:
return None
else:
return mask.copy().astype(dtype=np.uint8)
if dq is None:
raise ValueError("DQ array is None while 'dqbits' is not None.")
dqmask = bitfield_to_boolean_mask(dq, dqbits, good_mask_value=1,
dtype=np.uint8)
if mask is None:
return dqmask
# merge user mask with DQ mask:
dqmask[mask == 0] = 0
return dqmask
def _mergeUserMaskAndDQ(dq, mask, dqbits):
# Optional package dependency
try:
from stsci.tools.bitmask import (interpret_bit_flags,
bitfield_to_boolean_mask)
except ImportError:
from stsci.tools.bitmask import (interpret_bits_value as
interpret_bit_flags, bitmask2mask as
bitfield_to_boolean_mask)
dqbits = interpret_bit_flags(dqbits)
if dqbits is None:
if mask is None:
return None
else:
return mask.copy().astype(dtype=np.uint8)
if dq is None:
raise ValueError("DQ array is None while 'dqbits' is not None.")
dqmask = bitfield_to_boolean_mask(dq,
dqbits,
good_mask_value=1,
dtype=np.uint8)
if mask is None:
return dqmask
# merge user mask with DQ mask:
dqmask[mask == 0] = 0
return dqmask
def build_mask(dqarr, bitvalue):
""" Builds a bit-mask from an input DQ array and a bitvalue flag
"""
bitvalue = interpret_bit_flags(bitvalue)
if bitvalue is None:
return (np.ones(dqarr.shape, dtype=np.uint8))
return np.logical_not(np.bitwise_and(dqarr, ~bitvalue)).astype(np.uint8)
def process(self, input):
self.log.setLevel(logging.DEBUG)
img = datamodels.ModelContainer(input)
self._dqbits = interpret_bit_flags(self.dqbits)
# set sky stattistics:
self._skystat = SkyStats(skystat=self.skystat,
lower=self.lower,
upper=self.upper,
nclip=self.nclip,
lsig=self.lsigma,
usig=self.usigma,
binwidth=self.binwidth)
# group images by their "group id":
grp_img = img.models_grouped
# create a list of "Sky" Images and/or Groups:
images = []
grp_id = 1
for g in grp_img:
if len(g) > 1:
images.append(
SkyGroup(list(map(self._imodel2skyim, g)), id=grp_id))
grp_id += 1
elif len(g) == 1:
images.append(self._imodel2skyim(g[0]))
else:
raise AssertionError("Logical error in the pipeline code.")
# match/compute sky values:
match(images,
skymethod=self.skymethod,
match_down=self.match_down,
subtract=self.subtract)
# set sky background value in each image's meta:
for im in images:
if isinstance(im, SkyImage):
if im.is_sky_valid:
self._set_sky_background(im.meta['image_model'], im.sky)
im.meta['image_model'].meta.cal_step.skymatch = "COMPLETE"
else:
im.meta['image_model'].meta.cal_step.skymatch = "SKIPPED"
else:
for gim in im:
if gim.is_sky_valid:
self._set_sky_background(gim.meta['image_model'],
gim.sky)
gim.meta[
'image_model'].meta.cal_step.skymatch = "COMPLETE"
else:
gim.meta[
'image_model'].meta.cal_step.skymatch = "SKIPPED"
return img
def test_interpret_non_flag(flag):
with pytest.raises(ValueError):
bitmask.interpret_bit_flags(bit_flags=flag)
def setCommonInput(configObj, createOutwcs=True):
"""
The common interface interpreter for MultiDrizzle tasks which not only runs
'process_input()' but 'createImageObject()' and 'defineOutput()' as well to
fully setup all inputs for use with the rest of the MultiDrizzle steps either
as stand-alone tasks or internally to MultiDrizzle itself.
Parameters
----------
configObj : object
configObj instance or simple dictionary of input parameters
imageObjectList : list of imageObject objects
list of imageObject instances, 1 for each input exposure
outwcs : object
imageObject instance defining the final output frame
Notes
-----
At a minimum, the configObj instance (dictionary) should contain:
configObj = {'input':None,'output':None }
If provided, the configObj should contain the values of all the multidrizzle parameters
as set by the user with TEAL. If no configObj is given, it will retrieve
the default values automatically. In either case, the values from the input_dict
will be merged in with the configObj before being used by the rest of the
code.
Examples
--------
You can set *createOutwcs=False* for the cases where you only want the
images processed and no output wcs information in necessary; as in:
>>> imageObjectList,outwcs = processInput.processCommonInput(configObj)
"""
# make sure 'updatewcs' is set to False when running from GUI or if missing
# from configObj:
if 'updatewcs' not in configObj:
configObj['updatewcs'] = False
if not createOutwcs or not configObj['coeffs']:
# we're probably just working on single images here
configObj['updatewcs']=False
# maybe we can chunk this part up some more so that we can call just the
# parts we want
# Interpret input, read and convert and update input files, then return
# list of input filenames and derived output filename
asndict, ivmlist, output = process_input(
configObj['input'], configObj['output'],
updatewcs=configObj['updatewcs'], wcskey=configObj['wcskey'],
**configObj['STATE OF INPUT FILES'])
if not asndict:
return None, None
# convert the filenames from asndict into a list of full filenames
files = [fileutil.buildRootname(f) for f in asndict['order']]
original_files = asndict['original_file_names']
# interpret MDRIZTAB, if specified, and update configObj accordingly
# This can be done here because MDRIZTAB does not include values for
# input, output, or updatewcs.
if 'mdriztab' in configObj and configObj['mdriztab']:
print("Reading in MDRIZTAB parameters for {} files".format(len(files)))
mdriztab_dict = mdzhandler.getMdriztabParameters(files)
# Update configObj with values from mpars
cfgpars.mergeConfigObj(configObj, mdriztab_dict)
# Convert interpreted list of input files from process_input into a list
# of imageObject instances for use by the MultiDrizzle tasks.
instrpars = configObj['INSTRUMENT PARAMETERS']
# pass in 'proc_unit' to initialize unit conversions as necessary
instrpars['proc_unit'] = configObj['proc_unit']
undistort = True
if not configObj['coeffs']:
undistort = False
# determine whether parallel processing will be performed
use_parallel = False
if util.can_parallel:
# look to see whether steps which can be run using multiprocessing
# have been turned on
for stepnum in parallel_steps:
sname = util.getSectionName(configObj,stepnum[0])
if configObj[sname][stepnum[1]]:
use_parallel = True
break
# interpret all 'bits' related parameters and convert them to integers
configObj['resetbits'] = interpret_bit_flags(configObj['resetbits'])
step3name = util.getSectionName(configObj,3)
configObj[step3name]['driz_sep_bits'] = interpret_bit_flags(
configObj[step3name]['driz_sep_bits']
)
step7name = util.getSectionName(configObj,7)
configObj[step7name]['final_bits'] = interpret_bit_flags(
configObj[step7name]['final_bits']
)
# Verify any refimage parameters to be used
step3aname = util.getSectionName(configObj,'3a')
if not util.verifyRefimage(configObj[step3aname]['driz_sep_refimage']):
msg = 'No refimage with WCS found!\n '+\
' This could be caused by one of 2 problems:\n'+\
' * filename does not specify an extension with a valid WCS.\n'+\
' * can not find the file.\n'+\
'Please check the filename specified in the "refimage" parameter.'
print(textutil.textbox(msg))
return None,None
step7aname = util.getSectionName(configObj,'7a')
if not util.verifyRefimage(configObj[step7aname]['final_refimage']):
msg = 'No refimage with WCS found!\n '+\
' This could be caused by one of 2 problems:\n'+\
' * filename does not specify an extension with a valid WCS.\n'+\
' * can not find the file.\n'+\
'Please check the filename specified in the "refimage" parameter.'
print(textutil.textbox(msg))
return None,None
# Build imageObject list for all the valid, shift-updated input files
log.info('-Creating imageObject List as input for processing steps.')
if 'in_memory' in configObj:
virtual = configObj['in_memory']
else:
virtual = False
imageObjectList = createImageObjectList(files, instrpars,
group=configObj['group'],
undistort=undistort,
inmemory=virtual)
# Add original file names as "hidden" attributes of imageObject
assert(len(original_files) == len(imageObjectList)) #TODO: remove after extensive testing
for i in range(len(imageObjectList)):
imageObjectList[i]._original_file_name = original_files[i]
# apply context parameter
applyContextPar(imageObjectList, configObj['context'])
# reset DQ bits if requested by user
resetDQBits(imageObjectList, cr_bits_value=configObj['resetbits'])
# Add info about input IVM files at this point to the imageObjectList
addIVMInputs(imageObjectList, ivmlist)
if createOutwcs:
log.info('-Creating output WCS.')
# Build output WCS and update imageObjectList with output WCS info
outwcs = wcs_functions.make_outputwcs(imageObjectList, output,
configObj=configObj, perfect=True)
outwcs.final_wcs.printwcs()
else:
outwcs = None
try:
# Provide user with some information on resource usage for this run
# raises ValueError Exception in interactive mode and user quits
num_cores = configObj.get('num_cores') if use_parallel else 1
reportResourceUsage(imageObjectList, outwcs, num_cores)
except ValueError:
imageObjectList = None
return imageObjectList, outwcs
def setCommonInput(configObj, createOutwcs=True):
"""
The common interface interpreter for MultiDrizzle tasks which not only runs
'process_input()' but 'createImageObject()' and 'defineOutput()' as well to
fully setup all inputs for use with the rest of the MultiDrizzle steps either
as stand-alone tasks or internally to MultiDrizzle itself.
Parameters
----------
configObj : object
configObj instance or simple dictionary of input parameters
imageObjectList : list of imageObject objects
list of imageObject instances, 1 for each input exposure
outwcs : object
imageObject instance defining the final output frame
Notes
-----
At a minimum, the configObj instance (dictionary) should contain:
configObj = {'input':None,'output':None }
If provided, the configObj should contain the values of all the multidrizzle parameters
as set by the user with TEAL. If no configObj is given, it will retrieve
the default values automatically. In either case, the values from the input_dict
will be merged in with the configObj before being used by the rest of the
code.
Examples
--------
You can set *createOutwcs=False* for the cases where you only want the
images processed and no output wcs information in necessary; as in:
>>> imageObjectList,outwcs = processInput.processCommonInput(configObj)
"""
# make sure 'updatewcs' is set to False when running from GUI or if missing
# from configObj:
if 'updatewcs' not in configObj:
configObj['updatewcs'] = False
if not createOutwcs or not configObj['coeffs']:
# we're probably just working on single images here
configObj['updatewcs'] = False
# maybe we can chunk this part up some more so that we can call just the
# parts we want
# Interpret input, read and convert and update input files, then return
# list of input filenames and derived output filename
asndict, ivmlist, output = process_input(
configObj['input'],
configObj['output'],
updatewcs=configObj['updatewcs'],
wcskey=configObj['wcskey'],
**configObj['STATE OF INPUT FILES'])
if not asndict:
return None, None
# convert the filenames from asndict into a list of full filenames
files = [fileutil.buildRootname(f) for f in asndict['order']]
original_files = asndict['original_file_names']
# interpret MDRIZTAB, if specified, and update configObj accordingly
# This can be done here because MDRIZTAB does not include values for
# input, output, or updatewcs.
if 'mdriztab' in configObj and configObj['mdriztab']:
print("Reading in MDRIZTAB parameters for {} files".format(len(files)))
mdriztab_dict = mdzhandler.getMdriztabParameters(files)
# Update configObj with values from mpars
cfgpars.mergeConfigObj(configObj, mdriztab_dict)
# Convert interpreted list of input files from process_input into a list
# of imageObject instances for use by the MultiDrizzle tasks.
instrpars = configObj['INSTRUMENT PARAMETERS']
# pass in 'proc_unit' to initialize unit conversions as necessary
instrpars['proc_unit'] = configObj['proc_unit']
undistort = True
if not configObj['coeffs']:
undistort = False
# determine whether parallel processing will be performed
use_parallel = False
if util.can_parallel:
# look to see whether steps which can be run using multiprocessing
# have been turned on
for stepnum in parallel_steps:
sname = util.getSectionName(configObj, stepnum[0])
if configObj[sname][stepnum[1]]:
use_parallel = True
break
# interpret all 'bits' related parameters and convert them to integers
configObj['resetbits'] = interpret_bit_flags(configObj['resetbits'])
step3name = util.getSectionName(configObj, 3)
configObj[step3name]['driz_sep_bits'] = interpret_bit_flags(
configObj[step3name]['driz_sep_bits'])
step4name = util.getSectionName(configObj, 4)
if len(files) > 5 and 'minmed' in configObj[step4name]['combine_type']:
msg = '“minmed” is highly recommended for three images, \n'+\
' and is good for four to six images, \n'+\
' but should be avoided for ten or more images.\n'
print(textutil.textbox(msg))
step7name = util.getSectionName(configObj, 7)
configObj[step7name]['final_bits'] = interpret_bit_flags(
configObj[step7name]['final_bits'])
# Verify any refimage parameters to be used
step3aname = util.getSectionName(configObj, '3a')
if not util.verifyRefimage(configObj[step3aname]['driz_sep_refimage']):
msg = 'No refimage with WCS found!\n '+\
' This could be caused by one of 2 problems:\n'+\
' * filename does not specify an extension with a valid WCS.\n'+\
' * can not find the file.\n'+\
'Please check the filename specified in the "refimage" parameter.'
print(textutil.textbox(msg))
return None, None
step7aname = util.getSectionName(configObj, '7a')
if not util.verifyRefimage(configObj[step7aname]['final_refimage']):
msg = 'No refimage with WCS found!\n '+\
' This could be caused by one of 2 problems:\n'+\
' * filename does not specify an extension with a valid WCS.\n'+\
' * can not find the file.\n'+\
'Please check the filename specified in the "refimage" parameter.'
print(textutil.textbox(msg))
return None, None
# Build imageObject list for all the valid, shift-updated input files
log.info('-Creating imageObject List as input for processing steps.')
if 'in_memory' in configObj:
virtual = configObj['in_memory']
else:
virtual = False
imageObjectList = createImageObjectList(files,
instrpars,
output=asndict['output'],
group=configObj['group'],
undistort=undistort,
inmemory=virtual)
# Add original file names as "hidden" attributes of imageObject
assert (len(original_files) == len(imageObjectList)
) #TODO: remove after extensive testing
for i in range(len(imageObjectList)):
imageObjectList[i]._original_file_name = original_files[i]
# apply context parameter
applyContextPar(imageObjectList, configObj['context'])
# reset DQ bits if requested by user
resetDQBits(imageObjectList, cr_bits_value=configObj['resetbits'])
# Add info about input IVM files at this point to the imageObjectList
addIVMInputs(imageObjectList, ivmlist)
if createOutwcs:
log.info('-Creating output WCS.')
# Build output WCS and update imageObjectList with output WCS info
outwcs = wcs_functions.make_outputwcs(imageObjectList,
output,
configObj=configObj,
perfect=True)
outwcs.final_wcs.printwcs()
else:
outwcs = None
try:
# Provide user with some information on resource usage for this run
# raises ValueError Exception in interactive mode and user quits
num_cores = configObj.get('num_cores') if use_parallel else 1
reportResourceUsage(imageObjectList, outwcs, num_cores)
except ValueError:
imageObjectList = None
return imageObjectList, outwcs
def test_interpret_valid_str_bit_flags(flag, expected):
assert(
bitmask.interpret_bit_flags(bit_flags=flag) == expected
)
def test_interpret_valid_int_bit_flags(flag, flip, expected):
assert(
bitmask.interpret_bit_flags(bit_flags=flag, flip_bits=flip) == expected
)
def _skymatch(imageList, paramDict, in_memory, clean, logfile):
# '_skymatch' converts input imageList and other parameters to
# data structures accepted by the "skymatch" package.
# It also creates a temporary mask by combining 'static' mask,
# DQ image, and user-supplied mask. The combined mask is then
# passed to 'skymatch' to be used for excluding "bad" pixels.
skyKW="MDRIZSKY" #header keyword that contains the sky that's been subtracted
nimg = len(imageList)
if nimg == 0:
ml.logentry("Skymatch needs at least one images to perform{0}" \
"sky matching. Nothing to be done.",os.linesep)
return
# create a list of input file names as provided by the user:
user_fnames = []
loaded_fnames = []
filemaskinfos = nimg * [ None ]
for img in imageList:
user_fnames.append(img._original_file_name)
loaded_fnames.append(img._filename)
# parse sky mask catalog file (if any):
catfile = paramDict['skymask_cat']
if catfile:
#extname = imageList[0].scienceExt
#assert(extname is not None and extname != '')
catfile = catfile.strip()
mfindx = parse_at_file(fname = catfile,
default_ext = ('SCI','*'),
default_mask_ext = 0,
clobber = False,
fnamesOnly = True,
doNotOpenDQ = True,
match2Images = user_fnames,
im_fmode = 'update',
dq_fmode = 'readonly',
msk_fmode = 'readonly',
logfile = MultiFileLog(console=True),
verbose = True)
for p in mfindx:
filemaskinfos[p[1]] = p[0]
# step through the list of input images and create
# combined (static + DQ + user supplied, if any) mask, and
# create a list of FileExtMaskInfo objects to be passed
# to 'skymatch' function.
#
# This needs to be done in several steps, mostly due to the fact that
# the mask catalogs use "original" (e.g., GEIS, WAIVER FITS) file names
# while ultimately we want to open the version converted to MEF. Second
# reason is that we want to combine user supplied masks with DQ+static
# masks provided by astrodrizzle.
new_fi = []
sky_bits = interpret_bit_flags(paramDict['sky_bits'])
for i in range(nimg):
# extract extension information:
extname = imageList[i].scienceExt
extver = imageList[i].group
if extver is None:
extver = imageList[i].getExtensions()
assert(extname is not None and extname != '')
assert(extver)
# create a new FileExtMaskInfo object
fi = FileExtMaskInfo(default_ext=(extname,'*'),
default_mask_ext=0,
clobber=False,
doNotOpenDQ=True,
fnamesOnly=False,
im_fmode='update',
dq_fmode='readonly',
msk_fmode='readonly')
# set image file and extensions:
fi.image = loaded_fnames[i]
extlist = [ (extname,ev) for ev in extver ]
fi.append_ext(extlist)
# set user masks if any (this will open the files for a later use):
fi0 = filemaskinfos[i]
if fi0 is not None:
nmask = len(fi0.mask_images)
for m in range(nmask):
mask = fi0.mask_images[m]
ext = fi0.maskext[m]
fi.append_mask(mask, ext)
fi.finalize()
# combine user masks with static masks:
assert(len(extlist) == fi.count) #TODO: <-- remove after thorough testing
masklist = []
mextlist = []
for k in range(fi.count):
if fi.mask_images[k].closed:
umask = None
else:
umask = fi.mask_images[k].hdu[fi.maskext[k]].data
(mask, mext) = _buildStaticDQUserMask(imageList[i], extlist[k],
sky_bits, paramDict['use_static'],
fi.mask_images[k], fi.maskext[k], in_memory)
masklist.append(mask)
mextlist.append(mext)
# replace the original user-supplied masks with the
# newly computed combined static+DQ+user masks:
fi.clear_masks()
for k in range(fi.count):
if in_memory and mask is not None:
# os.stat() on the "original_fname" of the mask will fail
# since this is a "virtual" mask. Therefore we need to compute
# mask_stat ourselves. We will simply use id(data) for this:
mstat = os.stat_result((0,id(mask.hdu)) + 8*(0,))
fi.append_mask(masklist[k], mextlist[k], mask_stat=mstat)
else:
fi.append_mask(masklist[k], mextlist[k])
if masklist[k]:
masklist[k].release()
fi.finalize()
new_fi.append(fi)
# Run skymatch algorithm:
skymatch(new_fi,
skymethod = paramDict['skymethod'],
skystat = paramDict['skystat'],
lower = paramDict['skylower'],
upper = paramDict['skyupper'],
nclip = paramDict['skyclip'],
lsigma = paramDict['skylsigma'],
usigma = paramDict['skyusigma'],
binwidth = paramDict['skywidth'],
skyuser_kwd = skyKW,
units_kwd = 'BUNIT',
readonly = not paramDict['skysub'],
dq_bits = None,
optimize = 'inmemory' if in_memory else 'balanced',
clobber = True,
clean = clean,
verbose = True,
flog = MultiFileLog(console = True, enableBold = False),
_taskname4history = 'AstroDrizzle')
# Populate 'subtractedSky' and 'computedSky' of input image objects:
for i in range(nimg):
assert(not new_fi[i].fnamesOnly and not new_fi[i].image.closed)
image = imageList[i]
skysubimage = new_fi[i].image.hdu
numchips = image._numchips
extname = image.scienceExt
assert(os.path.samefile(image._filename, skysubimage.filename()))
for extver in range(1,numchips+1,1):
chip = image[extname,extver]
if not chip.group_member:
continue
subtracted_sky = skysubimage[extname,extver].header[skyKW]
chip.subtractedSky = subtracted_sky
chip.computedSky = subtracted_sky
# clean-up:
for fi in new_fi:
fi.release_all_images()
def _skymatch(imageList, paramDict, in_memory, clean, logfile):
# '_skymatch' converts input imageList and other parameters to
# data structures accepted by the "skymatch" package.
# It also creates a temporary mask by combining 'static' mask,
# DQ image, and user-supplied mask. The combined mask is then
# passed to 'skymatch' to be used for excluding "bad" pixels.
#header keyword that contains the sky that's been subtracted
skyKW = "MDRIZSKY"
nimg = len(imageList)
if nimg == 0:
log.info(
"Skymatch needs at least one image to perform{0} \
sky matching. Nothing to be done.", os.linesep)
return
# create a list of input file names as provided by the user:
user_fnames = []
loaded_fnames = []
filemaskinfos = nimg * [None]
for img in imageList:
user_fnames.append(img._original_file_name)
loaded_fnames.append(img._filename)
# parse sky mask catalog file (if any):
catfile = paramDict['skymask_cat']
if catfile:
#extname = imageList[0].scienceExt
#assert(extname is not None and extname != '')
catfile = catfile.strip()
mfindx = parse_at_file(fname=catfile,
default_ext=('SCI', '*'),
default_mask_ext=0,
clobber=False,
fnamesOnly=True,
doNotOpenDQ=True,
match2Images=user_fnames,
im_fmode='update',
dq_fmode='readonly',
msk_fmode='readonly',
logfile=MultiFileLog(console=True),
verbose=True)
for p in mfindx:
filemaskinfos[p[1]] = p[0]
# step through the list of input images and create
# combined (static + DQ + user supplied, if any) mask, and
# create a list of FileExtMaskInfo objects to be passed
# to 'skymatch' function.
#
# This needs to be done in several steps, mostly due to the fact that
# the mask catalogs use "original" (e.g., GEIS, WAIVER FITS) file names
# while ultimately we want to open the version converted to MEF. Second
# reason is that we want to combine user supplied masks with DQ+static
# masks provided by astrodrizzle.
new_fi = []
sky_bits = interpret_bit_flags(paramDict['sky_bits'])
for i in range(nimg):
# extract extension information:
extname = imageList[i].scienceExt
extver = imageList[i].group
if extver is None:
extver = imageList[i].getExtensions()
assert (extname is not None and extname != '')
assert (extver)
# create a new FileExtMaskInfo object
fi = FileExtMaskInfo(default_ext=(extname, '*'),
default_mask_ext=0,
clobber=False,
doNotOpenDQ=True,
fnamesOnly=False,
im_fmode='update',
dq_fmode='readonly',
msk_fmode='readonly')
# set image file and extensions:
fi.image = loaded_fnames[i]
extlist = [(extname, ev) for ev in extver]
fi.append_ext(extlist)
# set user masks if any (this will open the files for a later use):
fi0 = filemaskinfos[i]
if fi0 is not None:
nmask = len(fi0.mask_images)
for m in range(nmask):
mask = fi0.mask_images[m]
ext = fi0.maskext[m]
fi.append_mask(mask, ext)
fi.finalize()
# combine user masks with static masks:
assert (len(extlist) == fi.count
) #TODO: <-- remove after thorough testing
masklist = []
mextlist = []
for k in range(fi.count):
if fi.mask_images[k].closed:
umask = None
else:
umask = fi.mask_images[k].hdu[fi.maskext[k]].data
(mask, mext) = _buildStaticDQUserMask(imageList[i], extlist[k],
sky_bits,
paramDict['use_static'],
fi.mask_images[k],
fi.maskext[k], in_memory)
masklist.append(mask)
mextlist.append(mext)
# replace the original user-supplied masks with the
# newly computed combined static+DQ+user masks:
fi.clear_masks()
for k in range(fi.count):
if in_memory and mask is not None:
# os.stat() on the "original_fname" of the mask will fail
# since this is a "virtual" mask. Therefore we need to compute
# mask_stat ourselves. We will simply use id(data) for this:
mstat = os.stat_result((0, id(mask.hdu)) + 8 * (0, ))
fi.append_mask(masklist[k], mextlist[k], mask_stat=mstat)
else:
fi.append_mask(masklist[k], mextlist[k])
if masklist[k]:
masklist[k].release()
fi.finalize()
new_fi.append(fi)
try:
# Run skymatch algorithm:
skymatch(new_fi,
skymethod=paramDict['skymethod'],
skystat=paramDict['skystat'],
lower=paramDict['skylower'],
upper=paramDict['skyupper'],
nclip=paramDict['skyclip'],
lsigma=paramDict['skylsigma'],
usigma=paramDict['skyusigma'],
binwidth=paramDict['skywidth'],
skyuser_kwd=skyKW,
units_kwd='BUNIT',
readonly=not paramDict['skysub'],
dq_bits=None,
optimize='inmemory' if in_memory else 'balanced',
clobber=True,
clean=clean,
verbose=True,
flog=MultiFileLog(console=True, enableBold=False),
_taskname4history='AstroDrizzle')
except Exception:
if 'match' in paramDict[
'skymethod']: # This catches 'match' and 'globalmin+match'
new_method = 'globalmin' if 'globalmin' in paramDict[
'skymethod'] else 'localmin'
# revert to simpler sky computation algorithm
log.warning(
'Reverting sky computation to "localmin" from "{}'.format(
paramDict['skymethod']))
skymatch(new_fi,
skymethod=new_method,
skystat=paramDict['skystat'],
lower=paramDict['skylower'],
upper=paramDict['skyupper'],
nclip=paramDict['skyclip'],
lsigma=paramDict['skylsigma'],
usigma=paramDict['skyusigma'],
binwidth=paramDict['skywidth'],
skyuser_kwd=skyKW,
units_kwd='BUNIT',
readonly=not paramDict['skysub'],
dq_bits=None,
optimize='inmemory' if in_memory else 'balanced',
clobber=True,
clean=clean,
verbose=True,
flog=MultiFileLog(console=True, enableBold=False),
_taskname4history='AstroDrizzle')
else:
raise
# Populate 'subtractedSky' and 'computedSky' of input image objects:
for i in range(nimg):
assert (not new_fi[i].fnamesOnly and not new_fi[i].image.closed)
image = imageList[i]
skysubimage = new_fi[i].image.hdu
numchips = image._numchips
extname = image.scienceExt
assert (os.path.samefile(image._filename, skysubimage.filename()))
for extver in range(1, numchips + 1, 1):
chip = image[extname, extver]
if not chip.group_member:
continue
subtracted_sky = skysubimage[extname, extver].header.get(skyKW, 0.)
chip.subtractedSky = subtracted_sky
chip.computedSky = subtracted_sky
# clean-up:
for fi in new_fi:
fi.release_all_images()
def reset_dq_bits(input, bits, extver=None, extname='dq'):
""" This function resets bits in the integer array(s) of a FITS file.
Parameters
----------
filename : str
full filename with path
bits : str
sum or list of integers corresponding to all the bits to be reset
extver : int, optional
List of version numbers of the DQ arrays
to be corrected [Default Value: None, will do all]
extname : str, optional
EXTNAME of the DQ arrays in the FITS file
[Default Value: 'dq']
Notes
-----
The default value of None for the 'extver' parameter specifies that all
extensions with EXTNAME matching 'dq' (as specified by the 'extname'
parameter) will have their bits reset.
Examples
--------
1. The following command will reset the 4096 bits in all
the DQ arrays of the file input_file_flt.fits::
reset_dq_bits("input_file_flt.fits", 4096)
2. To reset the 2,32,64 and 4096 bits in the second DQ array,
specified as 'dq,2', in the file input_file_flt.fits::
reset_dq_bits("input_file_flt.fits", "2,32,64,4096", extver=2)
"""
# Interpret bits value
bits = interpret_bit_flags(bits)
flist, fcol = parseinput.parseinput(input)
for filename in flist:
# open input file in write mode to allow updating the DQ array in-place
p = fits.open(filename, mode='update', memmap=False)
# Identify the DQ array to be updated
# If no extver is specified, build a list of all DQ arrays in the file
if extver is None:
extver = []
for hdu in p:
# find only those extensions which match the input extname
# using case-insensitive name comparisons for 'extname'
if 'extver' in hdu.header and \
hdu.header['extname'].lower() == extname.lower():
extver.append(int(hdu.header['extver']))
else:
# Otherwise, insure that input extver values are a list
if not isinstance(extver, list): extver = [extver]
# for each DQ array identified in the file...
for extn in extver:
dqarr = p[extname, extn].data
dqdtype = dqarr.dtype
# reset the desired bits
p[extname, extn].data = (dqarr & ~bits).astype(
dqdtype) # preserve original dtype
log.info('Reset bit values of %s to a value of 0 in %s[%s,%s]' %
(bits, filename, extname, extn))
# close the file with the updated DQ array(s)
p.close()
def reset_dq_bits(input,bits,extver=None,extname='dq'):
""" This function resets bits in the integer array(s) of a FITS file.
Parameters
----------
filename : str
full filename with path
bits : str
sum or list of integers corresponding to all the bits to be reset
extver : int, optional
List of version numbers of the DQ arrays
to be corrected [Default Value: None, will do all]
extname : str, optional
EXTNAME of the DQ arrays in the FITS file
[Default Value: 'dq']
Notes
-----
The default value of None for the 'extver' parameter specifies that all
extensions with EXTNAME matching 'dq' (as specified by the 'extname'
parameter) will have their bits reset.
Examples
--------
1. The following command will reset the 4096 bits in all
the DQ arrays of the file input_file_flt.fits::
reset_dq_bits("input_file_flt.fits", 4096)
2. To reset the 2,32,64 and 4096 bits in the second DQ array,
specified as 'dq,2', in the file input_file_flt.fits::
reset_dq_bits("input_file_flt.fits", "2,32,64,4096", extver=2)
"""
# Interpret bits value
bits = interpret_bit_flags(bits)
flist, fcol = parseinput.parseinput(input)
for filename in flist:
# open input file in write mode to allow updating the DQ array in-place
p = fits.open(filename, mode='update', memmap=False)
# Identify the DQ array to be updated
# If no extver is specified, build a list of all DQ arrays in the file
if extver is None:
extver = []
for hdu in p:
# find only those extensions which match the input extname
# using case-insensitive name comparisons for 'extname'
if 'extver' in hdu.header and \
hdu.header['extname'].lower() == extname.lower():
extver.append(int(hdu.header['extver']))
else:
# Otherwise, insure that input extver values are a list
if not isinstance(extver, list): extver = [extver]
# for each DQ array identified in the file...
for extn in extver:
dqarr = p[extname,extn].data
dqdtype = dqarr.dtype
# reset the desired bits
p[extname,extn].data = (dqarr & ~bits).astype(dqdtype) # preserve original dtype
log.info('Reset bit values of %s to a value of 0 in %s[%s,%s]' %
(bits, filename, extname, extn))
# close the file with the updated DQ array(s)
p.close()
def test_interpret_allow_single_value_str_nonflags():
assert bitmask.interpret_bit_flags(bit_flags=str(3)) == 3
def test_interpret_bad_str_syntax(flag):
with pytest.raises(ValueError):
bitmask.interpret_bit_flags(bit_flags=flag)
def process(self, input1, input2):
cube_models = datamodels.ModelContainer(input1, persist=False)
models2d = datamodels.ModelContainer(input2, persist=False)
dqbits = interpret_bit_flags(self.dqbits)
# set sky stattistics:
self._skystat = SkyStats(skystat=self.skystat,
lower=self.lower,
upper=self.upper,
nclip=self.nclip,
lsig=self.lsigma,
usig=self.usigma,
binwidth=self.binwidth)
# At this moment running 'cube_skymatch' on images whose
# background has been previously subtracted is not supported.
# Raise an exception if background was subtracted:
self._check_background(cube_models)
self._check_background(models2d)
self._reset_background(cube_models)
self._reset_background(models2d)
# create a list of SkyCubes:
skycubes = []
for cm in cube_models:
# process weights and combine with DQ:
if not hasattr(cm, 'weightmap') or cm.weightmap is None:
weights = np.ones_like(cm.data, dtype=np.float64)
else:
weights = cm.weightmap.copy()
if dqbits is not None:
dq = bitfield_to_boolean_mask(cm.dq,
self._dqbits,
good_mask_value=0,
dtype=np.bool)
weights[dq] = 0.0
wcs = cm.meta.wcs if hasattr(cm.meta, 'wcs') else None
wcsinfo = cm.meta.wcsinfo if hasattr(cm.meta, 'wcsinfo') else None
exptime = cm.meta.exposure.exposure_time
if exptime is None:
exptime = 1.0
sc = SkyCube(data=cm.data,
wcs=wcs,
wcsinfo=wcsinfo,
weights=weights,
cube_weight=exptime,
bkg_deg=self.bkg_degree,
bkg_center=None,
id=None,
meta={'original_cube_model': cm})
skycubes.append(sc)
skymatch_cubes, nsubspace = match(skycubes, subtract=self.subtract)
if nsubspace > 1:
self.log.warning("Not all cubes have been sky matched as "
"some of them do not overlap.")
# save background info in 'meta' and subtract sky from 2D images
# if requested:
##### model.meta.instrument.channel
for c in skymatch_cubes:
self._set_cube_bkg_meta(c.meta['original_cube_model'], c)
model2d, channel = _find_associated_2d_image(
c.meta['original_cube_model'], models2d)
if model2d is None:
continue
self._set_model2d_bkg_meta(c.meta['original_cube_model'], model2d,
channel)
if self.subtract2d:
self._apply_sky_2d(model2d, channel)
return cube_models, models2d
def test_interpret_none_bit_flags_as_None(flag):
assert bitmask.interpret_bit_flags(bit_flags=flag) is None
def test_interpret_None_or_str_and_flip_incompatibility(flag, flip):
with pytest.raises(TypeError):
bitmask.interpret_bit_flags(bit_flags=flag, flip_bits=flip)
def destripe_plus(inputfile,
suffix='strp',
stat='pmode1',
maxiter=15,
sigrej=2.0,
lower=None,
upper=None,
binwidth=0.3,
scimask1=None,
scimask2=None,
dqbits=None,
rpt_clean=0,
atol=0.01,
cte_correct=True,
clobber=False,
verbose=True):
r"""Calibrate post-SM4 ACS/WFC exposure(s) and use
standalone :ref:`acsdestripe`.
This takes a RAW image and generates a FLT file containing
its calibrated and destriped counterpart.
If CTE correction is performed, FLC will also be present.
Parameters
----------
inputfile : str or list of str
Input filenames in one of these formats:
* a Python list of filenames
* a partial filename with wildcards ('\*raw.fits')
* filename of an ASN table ('j12345670_asn.fits')
* an at-file (``@input``)
suffix : str
The string to use to add to each input file name to
indicate an output product of ``acs_destripe``.
This only affects the intermediate output file that will
be automatically renamed to ``*blv_tmp.fits`` during the processing.
stat : { 'pmode1', 'pmode2', 'mean', 'mode', 'median', 'midpt' } (Default = 'pmode1')
Specifies the statistics to be used for computation of the
background in image rows:
* 'pmode1' - SEXTRACTOR-like mode estimate based on a
modified `Pearson's rule <https://en.wikipedia.org/wiki/Nonparametric_skew#Pearson.27s_rule>`_:
``2.5*median-1.5*mean``;
* 'pmode2' - mode estimate based on
`Pearson's rule <https://en.wikipedia.org/wiki/Nonparametric_skew#Pearson.27s_rule>`_:
``3*median-2*mean``;
* 'mean' - the mean of the distribution of the "good" pixels (after
clipping, masking, etc.);
* 'mode' - the mode of the distribution of the "good" pixels;
* 'median' - the median of the distribution of the "good" pixels;
* 'midpt' - estimate of the median of the distribution of the "good"
pixels based on an algorithm similar to IRAF's `imagestats` task
(``CDF(midpt)=1/2``).
.. note::
The midpoint and mode are computed in two passes through the
image. In the first pass the standard deviation of the pixels
is calculated and used with the *binwidth* parameter to compute
the resolution of the data histogram. The midpoint is estimated
by integrating the histogram and computing by interpolation
the data value at which exactly half the pixels are below that
data value and half are above it. The mode is computed by
locating the maximum of the data histogram and fitting the peak
by parabolic interpolation.
maxiter : int
This parameter controls the maximum number of iterations
to perform when computing the statistics used to compute the
row-by-row corrections.
sigrej : float
This parameters sets the sigma level for the rejection applied
during each iteration of statistics computations for the
row-by-row corrections.
lower : float, None (Default = None)
Lower limit of usable pixel values for computing the background.
This value should be specified in the units of the input image(s).
upper : float, None (Default = None)
Upper limit of usable pixel values for computing the background.
This value should be specified in the units of the input image(s).
binwidth : float (Default = 0.1)
Histogram's bin width, in sigma units, used to sample the
distribution of pixel brightness values in order to compute the
background statistics. This parameter is aplicable *only* to *stat*
parameter values of `'mode'` or `'midpt'`.
clobber : bool
Specify whether or not to 'clobber' (delete then replace)
previously generated products with the same names.
scimask1 : str or list of str
Mask images for *calibrated* ``SCI,1``, one for each input file.
Pixels with zero values will be masked out, in addition to clipping.
scimask2 : str or list of str
Mask images for *calibrated* ``SCI,2``, one for each input file.
Pixels with zero values will be masked out, in addition to clipping.
This is not used for subarrays.
dqbits : int, str, None (Default = None)
Integer sum of all the DQ bit values from the input image's DQ array
that should be considered "good" when building masks for de-striping
computations. For example, if pixels in the DQ array can be
combinations of 1, 2, 4, and 8 flags and one wants to consider
DQ "defects" having flags 2 and 4 as being acceptable for de-striping
computations, then `dqbits` should be set to 2+4=6. Then a DQ pixel
having values 2,4, or 6 will be considered a good pixel, while a DQ
pixel with a value, e.g., 1+2=3, 4+8=12, etc. will be flagged
as a "bad" pixel.
Alternatively, one can enter a comma- or '+'-separated list of
integer bit flags that should be added to obtain the final
"good" bits. For example, both ``4,8`` and ``4+8`` are equivalent to
setting `dqbits` to 12.
| Set `dqbits` to 0 to make *all* non-zero pixels in the DQ
mask to be considered "bad" pixels, and the corresponding image
pixels not to be used for de-striping computations.
| Default value (`None`) will turn off the use of image's DQ array
for de-striping computations.
| In order to reverse the meaning of the `dqbits`
parameter from indicating values of the "good" DQ flags
to indicating the "bad" DQ flags, prepend '~' to the string
value. For example, in order not to use pixels with
DQ flags 4 and 8 for sky computations and to consider
as "good" all other pixels (regardless of their DQ flag),
set `dqbits` to ``~4+8``, or ``~4,8``. To obtain the
same effect with an `int` input value (except for 0),
enter -(4+8+1)=-9. Following this convention,
a `dqbits` string value of ``'~0'`` would be equivalent to
setting ``dqbits=None``.
.. note::
DQ masks (if used), *will be* combined with user masks specified
in the `scimask1` and `scimask2` parameters (if any).
rpt_clean : int
An integer indicating how many *additional* times stripe cleaning
should be performed on the input image. Default = 0.
atol : float, None
The threshold for maximum absolute value of bias stripe correction
below which repeated cleanings can stop. When `atol` is `None`
cleaning will be repeated `rpt_clean` number of times.
Default = 0.01 [e].
cte_correct : bool
Perform CTE correction.
verbose : bool
Print informational messages. Default = True.
Raises
------
ImportError
``stsci.tools`` not found.
IOError
Input file does not exist.
ValueError
Invalid header values or CALACS version.
"""
# Optional package dependencies
from stsci.tools import parseinput
try:
from stsci.tools.bitmask import interpret_bit_flags
except ImportError:
from stsci.tools.bitmask import (interpret_bits_value as
interpret_bit_flags)
# process input file(s) and if we have multiple input files - recursively
# call acs_destripe_plus for each input image:
flist = parseinput.parseinput(inputfile)[0]
if isinstance(scimask1, str):
mlist1 = parseinput.parseinput(scimask1)[0]
elif isinstance(scimask1, np.ndarray):
mlist1 = [scimask1.copy()]
elif scimask1 is None:
mlist1 = []
elif isinstance(scimask1, list):
mlist1 = []
for m in scimask1:
if isinstance(m, np.ndarray):
mlist1.append(m.copy())
elif isinstance(m, str):
mlist1 += parseinput.parseinput(m)[0]
else:
raise TypeError("'scimask1' must be a list of str or "
"numpy.ndarray values.")
else:
raise TypeError("'scimask1' must be either a str, or a "
"numpy.ndarray, or a list of the two type of "
"values.")
if isinstance(scimask2, str):
mlist2 = parseinput.parseinput(scimask2)[0]
elif isinstance(scimask2, np.ndarray):
mlist2 = [scimask2.copy()]
elif scimask2 is None:
mlist2 = []
elif isinstance(scimask2, list):
mlist2 = []
for m in scimask2:
if isinstance(m, np.ndarray):
mlist2.append(m.copy())
elif isinstance(m, str):
mlist2 += parseinput.parseinput(m)[0]
else:
raise TypeError("'scimask2' must be a list of str or "
"numpy.ndarray values.")
else:
raise TypeError("'scimask2' must be either a str, or a "
"numpy.ndarray, or a list of the two type of "
"values.")
n_input = len(flist)
n_mask1 = len(mlist1)
n_mask2 = len(mlist2)
if n_input == 0:
raise ValueError(
'No input file(s) provided or the file(s) do not exist')
if n_mask1 == 0:
mlist1 = [None] * n_input
elif n_mask1 != n_input:
raise ValueError('Insufficient masks for [SCI,1]')
if n_mask2 == 0:
mlist2 = [None] * n_input
elif n_mask2 != n_input:
raise ValueError('Insufficient masks for [SCI,2]')
if n_input > 1:
for img, mf1, mf2 in zip(flist, mlist1, mlist2):
destripe_plus(inputfile=img,
suffix=suffix,
stat=stat,
lower=lower,
upper=upper,
binwidth=binwidth,
maxiter=maxiter,
sigrej=sigrej,
scimask1=scimask1,
scimask2=scimask2,
dqbits=dqbits,
cte_correct=cte_correct,
clobber=clobber,
verbose=verbose)
return
inputfile = flist[0]
scimask1 = mlist1[0]
scimask2 = mlist2[0]
# verify that the RAW image exists in cwd
cwddir = os.getcwd()
if not os.path.exists(os.path.join(cwddir, inputfile)):
raise IOError(f"{inputfile} does not exist.")
# get image's primary header:
header = fits.getheader(inputfile)
# verify masks defined (or not) simultaneously:
if (header['CCDAMP'] == 'ABCD'
and ((scimask1 is not None and scimask2 is None) or
(scimask1 is None and scimask2 is not None))):
raise ValueError("Both 'scimask1' and 'scimask2' must be specified "
"or not specified together.")
calacs_str = subprocess.check_output(['calacs.e', '--version'
]).split()[0] # nosec # noqa
calacs_ver = [int(x) for x in calacs_str.decode().split('.')]
if calacs_ver < [8, 3, 1]:
raise ValueError(f'CALACS {calacs_str} is incomptible. '
'Must be 8.3.1 or later.')
# check date for post-SM4 and if supported subarray or full frame
is_subarray = False
ctecorr = header['PCTECORR']
aperture = header['APERTURE']
detector = header['DETECTOR']
date_obs = Time(header['DATE-OBS'])
# intermediate filenames
blvtmp_name = inputfile.replace('raw', 'blv_tmp')
blctmp_name = inputfile.replace('raw', 'blc_tmp')
# output filenames
tra_name = inputfile.replace('_raw.fits', '.tra')
flt_name = inputfile.replace('raw', 'flt')
flc_name = inputfile.replace('raw', 'flc')
if detector != 'WFC':
raise ValueError(f"{inputfile} is not a WFC image, please check the "
"'DETECTOR' keyword.")
if date_obs < SM4_DATE:
raise ValueError(f"{inputfile} is a pre-SM4 image.")
if header['SUBARRAY'] and cte_correct:
if aperture in SUBARRAY_LIST:
is_subarray = True
else:
LOG.warning('Using non-supported subarray, '
'turning CTE correction off')
cte_correct = False
# delete files from previous CALACS runs
if clobber:
for tmpfilename in [
blvtmp_name, blctmp_name, flt_name, flc_name, tra_name
]:
if os.path.exists(tmpfilename):
os.remove(tmpfilename)
# run ACSCCD on RAW
acsccd.acsccd(inputfile)
# modify user mask with DQ masks if requested
dqbits = interpret_bit_flags(dqbits)
if dqbits is not None:
# save 'tra' file in memory to trick the log file
# not to save first acs2d log as this is done only
# for the purpose of obtaining DQ masks.
# WISH: it would have been nice is there was an easy way of obtaining
# just the DQ masks as if data were calibrated but without
# having to recalibrate them with acs2d.
if os.path.isfile(tra_name):
with open(tra_name) as fh:
tra_lines = fh.readlines()
else:
tra_lines = None
# apply flats, etc.
acs2d.acs2d(blvtmp_name, verbose=False, quiet=True)
# extract DQ arrays from the FLT image:
dq1, dq2 = _read_DQ_arrays(flt_name)
mask1 = _get_mask(scimask1, 1)
scimask1 = acs_destripe._mergeUserMaskAndDQ(dq1, mask1, dqbits)
mask2 = _get_mask(scimask2, 2)
if dq2 is not None:
scimask2 = acs_destripe._mergeUserMaskAndDQ(dq2, mask2, dqbits)
elif mask2 is None:
scimask2 = None
# reconstruct trailer file:
if tra_lines is not None:
with open(tra_name, mode='w') as fh:
fh.writelines(tra_lines)
# delete temporary FLT image:
if os.path.isfile(flt_name):
os.remove(flt_name)
# execute destriping (post-SM4 data only)
acs_destripe.clean(blvtmp_name,
suffix,
stat=stat,
maxiter=maxiter,
sigrej=sigrej,
lower=lower,
upper=upper,
binwidth=binwidth,
mask1=scimask1,
mask2=scimask2,
dqbits=dqbits,
rpt_clean=rpt_clean,
atol=atol,
clobber=clobber,
verbose=verbose)
blvtmpsfx = f'blv_tmp_{suffix}'
os.rename(inputfile.replace('raw', blvtmpsfx), blvtmp_name)
# update subarray header
if is_subarray and cte_correct:
fits.setval(blvtmp_name, 'PCTECORR', value='PERFORM')
ctecorr = 'PERFORM'
# perform CTE correction on destriped image
if cte_correct:
if ctecorr == 'PERFORM':
acscte.acscte(blvtmp_name)
else:
LOG.warning(f"PCTECORR={ctecorr}, cannot run CTE correction")
cte_correct = False
# run ACS2D to get FLT and FLC images
acs2d.acs2d(blvtmp_name)
if cte_correct:
acs2d.acs2d(blctmp_name)
# delete intermediate files
os.remove(blvtmp_name)
if cte_correct and os.path.isfile(blctmp_name):
os.remove(blctmp_name)
info_str = f'Done.\nFLT: {flt_name}\n'
if cte_correct:
info_str += f'FLC: {flc_name}\n'
LOG.info(info_str)
def test_interpret_wrong_flag_type(flag):
with pytest.raises(TypeError):
bitmask.interpret_bit_flags(bit_flags=flag)
def destripe_plus(inputfile, suffix='strp', stat='pmode1', maxiter=15,
sigrej=2.0, lower=None, upper=None, binwidth=0.3,
scimask1=None, scimask2=None,
dqbits=None, rpt_clean=0, atol=0.01,
cte_correct=True, clobber=False, verbose=True):
"""Calibrate post-SM4 ACS/WFC exposure(s) and use
standalone :ref:`acsdestripe`.
This takes a RAW image and generates a FLT file containing
its calibrated and destriped counterpart.
If CTE correction is performed, FLC will also be present.
Parameters
----------
inputfile : str or list of str
Input filenames in one of these formats:
* a Python list of filenames
* a partial filename with wildcards ('\*raw.fits')
* filename of an ASN table ('j12345670_asn.fits')
* an at-file (``@input``)
suffix : str
The string to use to add to each input file name to
indicate an output product of ``acs_destripe``.
This only affects the intermediate output file that will
be automatically renamed to ``*blv_tmp.fits`` during the processing.
stat : { 'pmode1', 'pmode2', 'mean', 'mode', 'median', 'midpt' } (Default = 'pmode1')
Specifies the statistics to be used for computation of the
background in image rows:
* 'pmode1' - SEXTRACTOR-like mode estimate based on a
modified `Pearson's rule <http://en.wikipedia.org/wiki/Nonparametric_skew#Pearson.27s_rule>`_:
``2.5*median-1.5*mean``;
* 'pmode2' - mode estimate based on
`Pearson's rule <http://en.wikipedia.org/wiki/Nonparametric_skew#Pearson.27s_rule>`_:
``3*median-2*mean``;
* 'mean' - the mean of the distribution of the "good" pixels (after
clipping, masking, etc.);
* 'mode' - the mode of the distribution of the "good" pixels;
* 'median' - the median of the distribution of the "good" pixels;
* 'midpt' - estimate of the median of the distribution of the "good"
pixels based on an algorithm similar to IRAF's `imagestats` task
(``CDF(midpt)=1/2``).
.. note::
The midpoint and mode are computed in two passes through the
image. In the first pass the standard deviation of the pixels
is calculated and used with the *binwidth* parameter to compute
the resolution of the data histogram. The midpoint is estimated
by integrating the histogram and computing by interpolation
the data value at which exactly half the pixels are below that
data value and half are above it. The mode is computed by
locating the maximum of the data histogram and fitting the peak
by parabolic interpolation.
maxiter : int
This parameter controls the maximum number of iterations
to perform when computing the statistics used to compute the
row-by-row corrections.
sigrej : float
This parameters sets the sigma level for the rejection applied
during each iteration of statistics computations for the
row-by-row corrections.
lower : float, None (Default = None)
Lower limit of usable pixel values for computing the background.
This value should be specified in the units of the input image(s).
upper : float, None (Default = None)
Upper limit of usable pixel values for computing the background.
This value should be specified in the units of the input image(s).
binwidth : float (Default = 0.1)
Histogram's bin width, in sigma units, used to sample the
distribution of pixel brightness values in order to compute the
background statistics. This parameter is aplicable *only* to *stat*
parameter values of `'mode'` or `'midpt'`.
clobber : bool
Specify whether or not to 'clobber' (delete then replace)
previously generated products with the same names.
scimask1 : str or list of str
Mask images for *calibrated* ``SCI,1``, one for each input file.
Pixels with zero values will be masked out, in addition to clipping.
scimask2 : str or list of str
Mask images for *calibrated* ``SCI,2``, one for each input file.
Pixels with zero values will be masked out, in addition to clipping.
This is not used for subarrays.
dqbits : int, str, None (Default = None)
Integer sum of all the DQ bit values from the input image's DQ array
that should be considered "good" when building masks for de-striping
computations. For example, if pixels in the DQ array can be
combinations of 1, 2, 4, and 8 flags and one wants to consider
DQ "defects" having flags 2 and 4 as being acceptable for de-striping
computations, then `dqbits` should be set to 2+4=6. Then a DQ pixel
having values 2,4, or 6 will be considered a good pixel, while a DQ
pixel with a value, e.g., 1+2=3, 4+8=12, etc. will be flagged
as a "bad" pixel.
Alternatively, one can enter a comma- or '+'-separated list of
integer bit flags that should be added to obtain the final
"good" bits. For example, both ``4,8`` and ``4+8`` are equivalent to
setting `dqbits` to 12.
| Set `dqbits` to 0 to make *all* non-zero pixels in the DQ
mask to be considered "bad" pixels, and the corresponding image
pixels not to be used for de-striping computations.
| Default value (`None`) will turn off the use of image's DQ array
for de-striping computations.
| In order to reverse the meaning of the `dqbits`
parameter from indicating values of the "good" DQ flags
to indicating the "bad" DQ flags, prepend '~' to the string
value. For example, in order not to use pixels with
DQ flags 4 and 8 for sky computations and to consider
as "good" all other pixels (regardless of their DQ flag),
set `dqbits` to ``~4+8``, or ``~4,8``. To obtain the
same effect with an `int` input value (except for 0),
enter -(4+8+1)=-9. Following this convention,
a `dqbits` string value of ``'~0'`` would be equivalent to
setting ``dqbits=None``.
.. note::
DQ masks (if used), *will be* combined with user masks specified
in the `scimask1` and `scimask2` parameters (if any).
rpt_clean : int
An integer indicating how many *additional* times stripe cleaning
should be performed on the input image. Default = 0.
atol : float, None
The threshold for maximum absolute value of bias stripe correction
below which repeated cleanings can stop. When `atol` is `None`
cleaning will be repeated `rpt_clean` number of times.
Default = 0.01 [e].
cte_correct : bool
Perform CTE correction.
verbose : bool
Print informational messages. Default = True.
Raises
------
ImportError
``stsci.tools`` not found.
IOError
Input file does not exist.
ValueError
Invalid header values or CALACS version.
"""
# Optional package dependencies
from stsci.tools import parseinput
try:
from stsci.tools.bitmask import interpret_bit_flags
except ImportError:
from stsci.tools.bitmask import (
interpret_bits_value as interpret_bit_flags
)
# process input file(s) and if we have multiple input files - recursively
# call acs_destripe_plus for each input image:
flist = parseinput.parseinput(inputfile)[0]
if isinstance(scimask1, str):
mlist1 = parseinput.parseinput(scimask1)[0]
elif isinstance(scimask1, np.ndarray):
mlist1 = [ scimask1.copy() ]
elif scimask1 is None:
mlist1 = []
elif isinstance(scimask1, list):
mlist1 = []
for m in scimask1:
if isinstance(m, np.ndarray):
mlist1.append(m.copy())
elif isinstance(m, str):
mlist1 += parseinput.parseinput(m)[0]
else:
raise TypeError("'scimask1' must be a list of str or "
"numpy.ndarray values.")
else:
raise TypeError("'scimask1' must be either a str, or a "
"numpy.ndarray, or a list of the two type of "
"values.")
if isinstance(scimask2, str):
mlist2 = parseinput.parseinput(scimask2)[0]
elif isinstance(scimask2, np.ndarray):
mlist2 = [ scimask2.copy() ]
elif scimask2 is None:
mlist2 = []
elif isinstance(scimask2, list):
mlist2 = []
for m in scimask2:
if isinstance(m, np.ndarray):
mlist2.append(m.copy())
elif isinstance(m, str):
mlist2 += parseinput.parseinput(m)[0]
else:
raise TypeError("'scimask2' must be a list of str or "
"numpy.ndarray values.")
else:
raise TypeError("'scimask2' must be either a str, or a "
"numpy.ndarray, or a list of the two type of "
"values.")
n_input = len(flist)
n_mask1 = len(mlist1)
n_mask2 = len(mlist2)
if n_input == 0:
raise ValueError(
'No input file(s) provided or the file(s) do not exist')
if n_mask1 == 0:
mlist1 = [None] * n_input
elif n_mask1 != n_input:
raise ValueError('Insufficient masks for [SCI,1]')
if n_mask2 == 0:
mlist2 = [None] * n_input
elif n_mask2 != n_input:
raise ValueError('Insufficient masks for [SCI,2]')
if n_input > 1:
for img, mf1, mf2 in zip(flist, mlist1, mlist2):
destripe_plus(
inputfile=img, suffix=suffix, stat=stat,
lower=lower, upper=upper, binwidth=binwidth,
maxiter=maxiter, sigrej=sigrej,
scimask1=scimask1, scimask2=scimask2, dqbits=dqbits,
cte_correct=cte_correct, clobber=clobber, verbose=verbose
)
return
inputfile = flist[0]
scimask1 = mlist1[0]
scimask2 = mlist2[0]
# verify that the RAW image exists in cwd
cwddir = os.getcwd()
if not os.path.exists(os.path.join(cwddir, inputfile)):
raise IOError("{0} does not exist.".format(inputfile))
# get image's primary header:
header = fits.getheader(inputfile)
# verify masks defined (or not) simultaneously:
if header['CCDAMP'] == 'ABCD' and \
((scimask1 is not None and scimask2 is None) or \
(scimask1 is None and scimask2 is not None)):
raise ValueError("Both 'scimask1' and 'scimask2' must be specified "
"or not specified together.")
calacs_str = subprocess.check_output(['calacs.e', '--version']).split()[0]
calacs_ver = [int(x) for x in calacs_str.decode().split('.')]
if calacs_ver < [8, 3, 1]:
raise ValueError('CALACS {0} is incomptible. '
'Must be 8.3.1 or later.'.format(calacs_str))
# check date for post-SM4 and if 2K subarray or full frame
is_sub2K = False
ctecorr = header['PCTECORR']
aperture = header['APERTURE']
detector = header['DETECTOR']
date_obs = Time(header['DATE-OBS'])
# intermediate filenames
blvtmp_name = inputfile.replace('raw', 'blv_tmp')
blctmp_name = inputfile.replace('raw', 'blc_tmp')
# output filenames
tra_name = inputfile.replace('_raw.fits', '.tra')
flt_name = inputfile.replace('raw', 'flt')
flc_name = inputfile.replace('raw', 'flc')
if detector != 'WFC':
raise ValueError("{0} is not a WFC image, please check the 'DETECTOR'"
" keyword.".format(inputfile))
if date_obs < SM4_DATE:
raise ValueError(
"{0} is a pre-SM4 image.".format(inputfile))
if header['SUBARRAY'] and cte_correct:
if aperture in SUBARRAY_LIST:
is_sub2K = True
else:
LOG.warning('Using non-2K subarray, turning CTE correction off')
cte_correct = False
# delete files from previous CALACS runs
if clobber:
for tmpfilename in [blvtmp_name, blctmp_name, flt_name, flc_name,
tra_name]:
if os.path.exists(tmpfilename):
os.remove(tmpfilename)
# run ACSCCD on RAW subarray
acsccd.acsccd(inputfile)
# modify user mask with DQ masks if requested
dqbits = interpret_bit_flags(dqbits)
if dqbits is not None:
# save 'tra' file in memory to trick the log file
# not to save first acs2d log as this is done only
# for the purpose of obtaining DQ masks.
# WISH: it would have been nice is there was an easy way of obtaining
# just the DQ masks as if data were calibrated but without
# having to recalibrate them with acs2d.
if os.path.isfile(tra_name):
fh = open(tra_name)
tra_lines = fh.readlines()
fh.close()
else:
tra_lines = None
# apply flats, etc.
acs2d.acs2d(blvtmp_name, verbose=False, quiet=True)
# extract DQ arrays from the FLT image:
dq1, dq2 = _read_DQ_arrays(flt_name)
if isinstance(scimask1, str):
if scimask1.strip() is '':
mask1 = None
scimask1 = None
else:
mask1 = fits.getdata(scimask1)
elif isinstance(scimask1, np.ndarray):
mask1 = scimask1.copy()
elif scimask1 is None:
mask1 = None
else:
raise TypeError("'scimask1' must be either a str file name, "
"a numpy.ndarray, or None.")
scimask1 = acs_destripe._mergeUserMaskAndDQ(dq1, mask1, dqbits)
if isinstance(scimask2, str):
if scimask2.strip() is '':
mask2 = None
scimask2 = None
else:
mask2 = fits.getdata(scimask2)
elif isinstance(scimask2, np.ndarray):
mask2 = scimask2.copy()
elif scimask2 is None:
mask2 = None
else:
raise TypeError("'scimask2' must be either a str file name, "
"a numpy.ndarray, or None.")
if dq2 is not None:
scimask2 = acs_destripe._mergeUserMaskAndDQ(dq2, mask2, dqbits)
# reconstruct trailer file:
if tra_lines is not None:
fh = open(tra_name, mode='w')
fh.writelines(tra_lines)
fh.close()
# delete temporary FLT image:
if os.path.isfile(flt_name):
os.remove(flt_name)
# execute destriping of the subarray (post-SM4 data only)
acs_destripe.clean(
blvtmp_name, suffix, stat=stat, maxiter=maxiter, sigrej=sigrej,
lower=lower, upper=upper, binwidth=binwidth,
mask1=scimask1, mask2=scimask2, dqbits=dqbits,
rpt_clean=rpt_clean, atol=atol, clobber=clobber, verbose=verbose)
blvtmpsfx = 'blv_tmp_{0}'.format(suffix)
os.rename(inputfile.replace('raw', blvtmpsfx), blvtmp_name)
# update subarray header
if is_sub2K and cte_correct:
fits.setval(blvtmp_name, 'PCTECORR', value='PERFORM')
ctecorr = 'PERFORM'
# perform CTE correction on destriped image
if cte_correct:
if ctecorr == 'PERFORM':
acscte.acscte(blvtmp_name)
else:
LOG.warning(
"PCTECORR={0}, cannot run CTE correction".format(ctecorr))
cte_correct = False
# run ACS2D to get FLT and FLC images
acs2d.acs2d(blvtmp_name)
if cte_correct:
acs2d.acs2d(blctmp_name)
# delete intermediate files
os.remove(blvtmp_name)
if cte_correct and os.path.isfile(blctmp_name):
os.remove(blctmp_name)
info_str = 'Done.\nFLT: {0}\n'.format(flt_name)
if cte_correct:
info_str += 'FLC: {0}\n'.format(flc_name)
LOG.info(info_str)
def test_interpret_wrong_string_int_format(flag):
with pytest.raises(ValueError):
bitmask.interpret_bit_flags(bit_flags=flag)
def process(self, input1, input2):
cube_models = datamodels.ModelContainer(input1, persist=False)
models2d = datamodels.ModelContainer(input2, persist=False)
dqbits = interpret_bit_flags(self.dqbits)
# set sky stattistics:
self._skystat = SkyStats(
skystat=self.skystat,
lower=self.lower,
upper=self.upper,
nclip=self.nclip,
lsig=self.lsigma,
usig=self.usigma,
binwidth=self.binwidth
)
# At this moment running 'cube_skymatch' on images whose
# background has been previously subtracted is not supported.
# Raise an exception if background was subtracted:
self._check_background(cube_models)
self._check_background(models2d)
self._reset_background(cube_models)
self._reset_background(models2d)
# create a list of SkyCubes:
skycubes = []
for cm in cube_models:
# process weights and combine with DQ:
if not hasattr(cm, 'weightmap') or cm.weightmap is None:
weights = np.ones_like(cm.data, dtype=np.float64)
else:
weights = cm.weightmap.copy()
if dqbits is not None:
dq = bitfield_to_boolean_mask(
cm.dq,
self._dqbits,
good_mask_value=0,
dtype=np.bool
)
weights[dq] = 0.0
wcs = cm.meta.wcs if hasattr(cm.meta, 'wcs') else None
wcsinfo = cm.meta.wcsinfo if hasattr(cm.meta, 'wcsinfo') else None
exptime = cm.meta.exposure.exposure_time
if exptime is None:
exptime = 1.0
sc = SkyCube(
data=cm.data,
wcs=wcs,
wcsinfo=wcsinfo,
weights=weights,
cube_weight=exptime,
bkg_deg=self.bkg_degree,
bkg_center=None,
id=None,
meta={'original_cube_model': cm}
)
skycubes.append(sc)
skymatch_cubes, nsubspace = match(skycubes, subtract=self.subtract)
if nsubspace > 1:
self.log.warning("Not all cubes have been sky matched as "
"some of them do not overlap.")
# save background info in 'meta' and subtract sky from 2D images
# if requested:
##### model.meta.instrument.channel
for c in skymatch_cubes:
self._set_cube_bkg_meta(c.meta['original_cube_model'], c)
model2d, channel = _find_associated_2d_image(
c.meta['original_cube_model'], models2d
)
if model2d is None:
continue
self._set_model2d_bkg_meta(
c.meta['original_cube_model'],
model2d,
channel
)
if self.subtract2d:
self._apply_sky_2d(model2d, channel)
return cube_models, models2d
def process(self, input):
self.log.setLevel(logging.DEBUG)
img = datamodels.ModelContainer(input)
self._dqbits = interpret_bit_flags(self.dqbits)
# set sky stattistics:
self._skystat = SkyStats(
skystat=self.skystat,
lower=self.lower,
upper=self.upper,
nclip=self.nclip,
lsig=self.lsigma,
usig=self.usigma,
binwidth=self.binwidth
)
# group images by their "group id":
grp_img = img.models_grouped
# create a list of "Sky" Images and/or Groups:
images = []
grp_id = 1
for g in grp_img:
if len(g) > 1:
images.append(
SkyGroup(
list(map(self._imodel2skyim, g)),
id=grp_id
)
)
grp_id += 1
elif len(g) == 1:
images.append(self._imodel2skyim(g[0]))
else:
raise AssertionError("Logical error in the pipeline code.")
# match/compute sky values:
match(images, skymethod=self.skymethod, match_down=self.match_down,
subtract=self.subtract)
# set sky background value in each image's meta:
for im in images:
if isinstance(im, SkyImage):
if im.is_sky_valid:
self._set_sky_background(im.meta['image_model'], im.sky)
im.meta['image_model'].meta.cal_step.skymatch = "COMPLETE"
else:
im.meta['image_model'].meta.cal_step.skymatch = "SKIPPED"
else:
for gim in im:
if gim.is_sky_valid:
self._set_sky_background(
gim.meta['image_model'],
gim.sky
)
gim.meta['image_model'].meta.cal_step.skymatch = "COMPLETE"
else:
gim.meta['image_model'].meta.cal_step.skymatch = "SKIPPED"
return img
