def test_opening_ramp_ref(tmp_path):
# First make test reference file
file_path = tmp_path / 'testramp.asdf'
utils.mk_ramp(filepath=file_path)
ramp = datamodels.open(file_path)
assert ramp.meta.instrument.optical_element == 'F062'
assert isinstance(ramp, datamodels.RampModel)
def test_err():
"""Check that a 4-D ERR array is initialized and all values are zero."""
# size of integration
instrument = 'WFI'
ngroups = 5
xsize = 1032
ysize = 1024
csize = (ngroups, ysize, xsize)
# create raw input data for step
dm_ramp = testutil.mk_ramp((ngroups, ysize, xsize))
dm_ramp.meta.instrument.name = instrument
# create a MaskModel elements for the dq input mask
ref_data = testutil.mk_mask(csize[1:])
ref_data['meta']['instrument']['name'] = instrument
# Filter out validation warnings from ref_data
warnings.filterwarnings("ignore", category=ValidationWarning)
# run correction step
outfile = do_dqinit(dm_ramp, ref_data)
# check that ERR array was created and initialized to zero
errarr = outfile.err
assert(errarr.ndim == 2) # check that output err array is 2-D
assert(np.all(errarr == 0)) # check that values are 0
def test_groupdq():
"""Check that GROUPDQ extension is added to the data and all values are initialized to zero."""
# size of integration
instrument = 'WFI'
ngroups = 5
xsize = 1032
ysize = 1024
csize = (ngroups, ysize, xsize)
# create raw input data for step
dm_ramp = testutil.mk_ramp(csize)
dm_ramp.meta.instrument.name = instrument
# create a MaskModel elements for the dq input mask
ref_data = testutil.mk_mask(csize[1:])
ref_data['meta']['instrument']['name'] = instrument
# run the correction step
outfile = do_dqinit(dm_ramp, ref_data)
# check that GROUPDQ was created and initialized to zero
groupdq = outfile.groupdq
np.testing.assert_array_equal(np.full((ngroups, ysize, xsize), 0, dtype=int),
groupdq, err_msg='groupdq not initialized to zero')
def _setup(ngroups=10,
readnoise=10,
nrows=20,
ncols=20,
nframes=1,
grouptime=1.0,
gain=1,
deltatime=1):
err = np.ones(shape=(nrows, ncols), dtype=np.float32)
data = np.zeros(shape=(ngroups, nrows, ncols), dtype=np.float32)
gdq = np.zeros(shape=(ngroups, nrows, ncols), dtype=np.uint8)
pixdq = np.zeros(shape=(nrows, ncols), dtype=np.uint32)
csize = (ngroups, nrows, ncols)
dm_ramp = rdm.RampModel(testutil.mk_ramp(csize))
dm_ramp.meta.instrument.name = 'WFI'
dm_ramp.meta.instrument.optical_element = 'F158'
dm_ramp.data = data + 6.
dm_ramp.pixeldq = pixdq
dm_ramp.groupdq = gdq
dm_ramp.err = err
dm_ramp.meta.exposure.type = 'WFI_IMAGE'
dm_ramp.meta.exposure.group_time = deltatime
dm_ramp.meta.exposure.frame_time = deltatime
dm_ramp.meta.exposure.ngroups = ngroups
dm_ramp.meta.exposure.nframes = 1
dm_ramp.meta.exposure.groupgap = 0
dm_ramp.meta.cal_step['dq_init'] = 'INCOMPLETE'
dm_ramp.meta.cal_step['jump'] = 'INCOMPLETE'
return dm_ramp
def _models(ngroups, nrows, ncols):
# Create ramp data
data_model = testutil.mk_ramp(arrays=(ngroups, nrows, ncols))
# Create saturation reference data
saturation_model = testutil.mk_saturation(shape=(nrows, ncols))
return data_model, saturation_model
def test_make_ramp():
ramp = utils.mk_ramp(shape=(2, 20, 20))
assert ramp.meta.exposure.type == 'WFI_IMAGE'
assert ramp.data.dtype == np.float32
assert ramp.pixeldq.dtype == np.uint32
assert ramp.pixeldq.shape == (20, 20)
assert ramp.groupdq.dtype == np.uint8
assert ramp.err.dtype == np.float32
assert ramp.err.shape == (20, 20)
# Test validation
ramp = datamodels.RampModel(ramp)
assert ramp.validate() is None
def test_dq_im(xstart, ystart, xsize, ysize, ngroups, instrument, exp_type):
""" Check that PIXELDQ is initialized with the information from the reference file.
test that a flagged value in the reference file flags the PIXELDQ array"""
csize = (ngroups, ysize, xsize)
# create raw input data for step
dm_ramp = testutil.mk_ramp(csize)
dm_ramp.meta.instrument.name = instrument
# create a MaskModel elements for the dq input mask
dq = np.zeros(csize[1:], dtype=np.uint32)
# edit reference file with known bad pixel values
dq[100, 100] = 2 # Saturated pixel
dq[200, 100] = 4 # Jump detected pixel
dq[300, 100] = 8 # Dropout
dq[400, 100] = 32 # Persistence
dq[500, 100] = 1 # Do_not_use
dq[100, 200] = 3 # Saturated pixel + do not use
dq[200, 200] = 5 # Jump detected pixel + do not use
dq[300, 200] = 9 # Dropout + do not use
dq[400, 200] = 33 # Persistence + do not use
# write mask model
ref_data = testutil.mk_mask(csize[1:])
# Copy in maskmodel elemnts
ref_data['dq'] = dq
ref_data['meta']['instrument']['name'] = instrument
# run do_dqinit
outfile = do_dqinit(dm_ramp, ref_data)
dqdata = outfile['pixeldq']
# assert that the pixels read back in match the mapping from ref data to science data
assert (dqdata[100, 100] == dqflags.pixel['SATURATED'])
assert (dqdata[200, 100] == dqflags.pixel['JUMP_DET'])
assert (dqdata[300, 100] == dqflags.pixel['DROPOUT'])
assert (dqdata[400, 100] == dqflags.pixel['PERSISTENCE'])
assert (dqdata[500, 100] == dqflags.pixel['DO_NOT_USE'])
assert (dqdata[100, 200] == dqflags.pixel['SATURATED'] +
dqflags.pixel['DO_NOT_USE'])
assert (dqdata[200, 200] == dqflags.pixel['JUMP_DET'] +
dqflags.pixel['DO_NOT_USE'])
assert (dqdata[300, 200] == dqflags.pixel['DROPOUT'] +
dqflags.pixel['DO_NOT_USE'])
assert (dqdata[400, 200] == dqflags.pixel['PERSISTENCE'] +
dqflags.pixel['DO_NOT_USE'])
def create_ramp_and_dark(shape, instrument, exptype):
"""Helper function to create test ramp and dark models"""
# Create test ramp model
ramp = testutil.mk_ramp(shape)
ramp.meta.instrument.name = instrument
ramp.meta.instrument.detector = 'WFI01'
ramp.meta.instrument.optical_element = 'F158'
ramp.meta.exposure.type = exptype
ramp.data = np.ones(shape, dtype=np.float32)
ramp_model = RampModel(ramp)
# Create dark model
darkref = testutil.mk_dark(shape)
darkref_model = DarkRefModel(darkref)
return ramp_model, darkref_model
def test_dq_add1_groupdq():
"""
Test if the dq_init code set the groupdq flag on the first
group to 'do_not_use' by adding 1 to the flag, not overwriting to 1
Also test whether two flags on the same pixel are added together.
"""
# size of integration
instrument = 'WFI'
ngroups = 5
xsize = 1032
ysize = 1024
csize = (ngroups, ysize, xsize)
# create raw input data for step
dm_ramp = testutil.mk_ramp((ngroups, ysize, xsize))
dm_ramp.meta.instrument.name = instrument
# create a MaskModel elements for the dq input mask
dq = np.zeros(csize[1:], dtype=np.uint32)
# write reference file with known bad pixel values
dq[505, 505] = 1 # Do_not_use
dq[400, 500] = 3 # do_not_use and saturated pixel
# write mask model
ref_data = testutil.mk_mask(csize[1:])
ref_data['meta']['instrument']['name'] = instrument
# Copy in maskmodel elemnts
ref_data['dq'] = dq
# set a flag in the pixel dq
dm_ramp.pixeldq[505, 505] = 4 # Jump detected pixel
# run correction step
outfile = do_dqinit(dm_ramp, ref_data)
# test if pixels in pixeldq were incremented in value by 1
# check that previous dq flag is added to mask value
assert (outfile.pixeldq[505, 505] == dqflags.pixel['JUMP_DET'] +
dqflags.pixel['DO_NOT_USE'])
# check two flags propagate correctly
assert (outfile.pixeldq[400, 500] == dqflags.pixel['SATURATED'] +
dqflags.pixel['DO_NOT_USE'])
def _setup(ngroups=10, nrows=20, ncols=20, deltatime=1):
data = np.zeros(shape=(ngroups, nrows, ncols), dtype=np.float32)
err = np.ones(shape=(nrows, ncols), dtype=np.float32)
pixdq = np.zeros(shape=(nrows, ncols), dtype=np.uint32)
gdq = np.zeros(shape=(ngroups, nrows, ncols), dtype=np.uint8)
csize = (ngroups, nrows, ncols)
dm_ramp = testutil.mk_ramp(csize)
dm_ramp.data = data
dm_ramp.pixeldq = pixdq
dm_ramp.groupdq = gdq
dm_ramp.err = err
dm_ramp.meta.exposure.frame_time = deltatime
dm_ramp.meta.exposure.ngroups = ngroups
dm_ramp.meta.exposure.nframes = 1
dm_ramp.meta.exposure.groupgap = 0
return dm_ramp
def process(self, input):
"""Perform the dq_init calibration step
Parameters
----------
input : Roman datamodel
input roman datamodel
Returns
-------
output_model : Roman datamodel
result roman datamodel
"""
# Open datamodel
input_model = self.open_model(input)
# Convert to RampModel if needed
if not (isinstance(input_model, RampModel)):
# Create base ramp node with dummy values (for validation)
input_ramp = testutil.mk_ramp(input_model.shape)
# Copy input_model contents into RampModel
for key in input_model.keys():
# If a dictionary (like meta), overwrite entires (but keep
# required dummy entries that may not be in input_model)
if isinstance(input_ramp[key], dict):
input_ramp[key].update(input_model.__getattr__(key))
elif isinstance(input_ramp[key], np.ndarray):
# Cast input ndarray as RampModel dtype
input_ramp[key] = input_model.__getattr__(key).astype(
input_ramp[key].dtype)
else:
input_ramp[key] = input_model.__getattr__(key)
# Create model from node
init_model = RampModel(input_ramp)
else:
init_model = input_model
# Get reference file paths
reference_file_names = {}
reffile = self.get_reference_file(init_model, "mask")
reference_file_names['mask'] = reffile if reffile != 'N/A' else None
# Open the relevant reference files as datamodels
reference_file_models = {}
# Test for reference files
if reffile is not None:
# If there are mask files, perform dq step
reference_file_models['mask'] = rdm.open(reffile)
self.log.debug(f'Using MASK ref file: {reffile}')
# Apply the DQ step
output_model = dq_initialization.do_dqinit(
init_model,
**reference_file_models,
)
else:
# Skip DQ step if no mask files
reference_file_models['mask'] = None
self.log.warning('No MASK reference file found.')
self.log.warning('DQ initialization step will be skipped.')
output_model = init_model
output_model.meta.cal_step.dq_init = 'SKIPPED'
# Close the input and reference files
input_model.close()
init_model.close()
try:
for model in reference_file_models.values():
model.close()
except AttributeError:
pass
return output_model
