def test_all_groups_saturated(setup_nrc_cube):
'''Check case where all groups are saturated.'''
# Create inputs, data, and saturation maps
ngroups = 5
nrows = 20
ncols = 20
satvalue = 60000
data, satmap = setup_nrc_cube(ngroups, nrows, ncols)
# Add ramp values at or above saturation limit
data.data[0, 0, 5, 5] = 60000
data.data[0, 1, 5, 5] = 62000
data.data[0, 2, 5, 5] = 62000
data.data[0, 3, 5, 5] = 60000
data.data[0, 4, 5, 5] = 62000
# Set saturation value in the saturation model
satmap.data[5, 5] = satvalue
# Run the pipeline
output = do_correction(data, satmap)
# Make sure all groups are flagged
assert np.all(output.groupdq[0, :, 5, 5] == dqflags.group['SATURATED'])
def test_ad_floor_flagging(setup_nrc_cube):
"""Check that the ad_floor flag is set when a pixel value is zero or
negative."""
# Create inputs, data, and saturation maps
ngroups = 5
nrows = 20
ncols = 20
satvalue = 60000
data, satmap = setup_nrc_cube(ngroups, nrows, ncols)
# Add ramp values up to the saturation limit
data.data[0, 0, 5, 5] = 0 # Signal at bottom rail - low saturation
data.data[0, 1, 5, 5] = 0 # Signal at bottom rail - low saturation
data.data[0, 2, 5, 5] = 20
data.data[0, 3, 5, 5] = 40
data.data[0, 4, 5, 5] = 60
# frames that should be flagged as saturation (low)
satindxs = [0, 1]
# Set saturation value in the saturation model
satmap.data[5, 5] = satvalue
# Run the pipeline
output = do_correction(data, satmap)
# Check if the right frames are flagged as saturated
assert np.all(output.groupdq[0, satindxs, 5,
5] == dqflags.group['DO_NOT_USE']
| dqflags.group['AD_FLOOR'])
def test_nans_in_mask(setup_nrc_cube):
'''Check that pixels in the reference files that have value NaN are not
flagged as saturated in the data and that in the PIXELDQ array the
pixel is set to NO_SAT_CHECK.'''
# Create inputs, data, and saturation maps
ngroups = 5
nrows = 20
ncols = 20
data, satmap = setup_nrc_cube(ngroups, nrows, ncols)
# Add ramp values up to the saturation limit
data.data[0, 0, 5, 5] = 0
data.data[0, 1, 5, 5] = 20000
data.data[0, 2, 5, 5] = 40000
data.data[0, 3, 5, 5] = 60000
data.data[0, 4, 5, 5] = 62000
# Set saturation value for pixel to NaN
satmap.data[5, 5] = np.nan
# Run the pipeline
output = do_correction(data, satmap)
# Check that output GROUPDQ is not flagged as saturated
assert np.all(output.groupdq[0, :, 5, 5] != dqflags.group['SATURATED'])
# Check that output PIXELDQ is set to NO_SAT_CHECK
assert output.pixeldq[5, 5] == dqflags.pixel['NO_SAT_CHECK']
def test_basic_saturation_flagging(setup_nrc_cube):
'''Check that the saturation flag is set when a pixel value is above the
threshold given by the reference file.'''
# Create inputs, data, and saturation maps
ngroups = 5
nrows = 20
ncols = 20
satvalue = 60000
data, satmap = setup_nrc_cube(ngroups, nrows, ncols)
# Add ramp values up to the saturation limit
data.data[0, 0, 5, 5] = 0
data.data[0, 1, 5, 5] = 20000
data.data[0, 2, 5, 5] = 40000
data.data[0, 3, 5, 5] = 60000 # Signal reaches saturation limit
data.data[0, 4, 5, 5] = 62000
# Set saturation value in the saturation model
satmap.data[5, 5] = satvalue
# Run the pipeline
output = do_correction(data, satmap)
# Make sure that groups with signal > saturation limit get flagged
satindex = np.argmax(output.data[0, :, 5, 5] == satvalue)
assert np.all(output.groupdq[0, satindex:, 5, 5] == dqflags.group['SATURATED'])
def test_basic_saturation_flagging(setup_nrc_cube):
'''Check that the saturation flag is set when a pixel value is above the
threshold given by the reference file.'''
# Create inputs, data, and saturation maps
ngroups = 5
nrows = 2048
ncols = 2048
satvalue = 60000
data, satmap = setup_nrc_cube(ngroups, nrows, ncols)
# Add ramp values up to the saturation limit
data.data[0, 0, 500, 500] = 0
data.data[0, 1, 500, 500] = 20000
data.data[0, 2, 500, 500] = 40000
data.data[0, 3, 500, 500] = 60000 # Signal reaches saturation limit
data.data[0, 4, 500, 500] = 62000
# Set saturation value in the saturation model
satmap.data[500, 500] = satvalue
# Run the pipeline
output = do_correction(data, satmap)
# Make sure that groups with signal > saturation limit get flagged
satindex = np.argmax(output.data[0, :, 500, 500] == satvalue)
assert np.all(output.groupdq[0, satindex:, 500, 500] == dqflags.group['SATURATED'])
def test_subarray_extraction(setup_miri_cube):
'''Check the step correctly handles subarrays.'''
# Create input data
# Create model of data with 0 value array
ngroups = 50
nrows = 224
ncols = 288
data, satmap = setup_miri_cube(1, 467, ngroups, nrows, ncols)
# Place DQ flags in DQ array that would be in subarray
# MASK1550 file has colstart=1, rowstart=467
satmap.dq[542, 100:105] = dqflags.pixel['DO_NOT_USE']
# Test a value of NaN in the reference file with an existing DQ flag
satmap.data[550, 100] = np.nan
satmap.dq[550, 100] = dqflags.pixel['NONLINEAR']
# Run the pipeline
output = do_correction(data, satmap)
# Check for DQ flag in PIXELDQ of subarray image
assert(output.pixeldq[76, 100] == dqflags.pixel['DO_NOT_USE'])
assert(output.pixeldq[76, 104] == dqflags.pixel['DO_NOT_USE'])
# Pixel 84, 100 in subarray maps to 550, 100 in reference file
# Check that pixel was flagged 'NO_SAT_CHECK' and that original
# DQ flag persists (i.e. did not get overwritten)
assert(output.pixeldq[84, 100] ==
dqflags.pixel['NO_SAT_CHECK'] + dqflags.pixel['NONLINEAR'])
def test_signal_fluctuation_flagging(setup_nrc_cube):
'''Check that once a pixel is flagged as saturated in a group, all
subsequent groups should also be flagged as saturated, even if the
signal value drops back below saturation.'''
# Create inputs, data, and saturation maps
ngroups = 5
nrows = 20
ncols = 20
satvalue = 60000
data, satmap = setup_nrc_cube(ngroups, nrows, ncols)
# Add ramp values up to the saturation limit
data.data[0, 0, 5, 5] = 0
data.data[0, 1, 5, 5] = 20000
data.data[0, 2, 5, 5] = 40000
data.data[0, 3, 5, 5] = 60000 # Signal reaches saturation limit
data.data[0, 4, 5, 5] = 40000 # Signal drops below saturation limit
# Set saturation value in the saturation model
satmap.data[5, 5] = satvalue
# Run the pipeline
output = do_correction(data, satmap)
# Make sure that all groups after first saturated group are flagged
satindex = np.argmax(output.data[0, :, 5, 5] == satvalue)
assert np.all(output.groupdq[0, satindex:, 5, 5] == dqflags.group['SATURATED'])
def test_all_groups_saturated(setup_nrc_cube):
'''Check case where all groups are saturated.'''
# Create inputs, data, and saturation maps
ngroups = 5
nrows = 2048
ncols = 2048
satvalue = 60000
data, satmap = setup_nrc_cube(ngroups, nrows, ncols)
# Add ramp values at or above saturation limit
data.data[0, 0, 500, 500] = 60000
data.data[0, 1, 500, 500] = 62000
data.data[0, 2, 500, 500] = 62000
data.data[0, 3, 500, 500] = 60000
data.data[0, 4, 500, 500] = 62000
# Set saturation value in the saturation model
satmap.data[500, 500] = satvalue
# Run the pipeline
output = do_correction(data, satmap)
# Make sure all groups are flagged
assert np.all(output.groupdq[0, :, 500, 500] == dqflags.group['SATURATED'])
def test_subarray_extraction(setup_miri_cube):
'''Check the step correctly handles subarrays.'''
# Create input data
# Create model of data with 0 value array
ngroups = 50
nrows = 224
ncols = 288
data, satmap = setup_miri_cube(1, 467, ngroups, nrows, ncols)
# Place DQ flags in DQ array that would be in subarray
# MASK1550 file has colstart=1, rowstart=467
satmap.dq[542, 100:105] = 1
# Test a value of NaN in the reference file
satmap.data[550, 100] = np.nan
# Run the pipeline
output = do_correction(data, satmap)
# Check for DQ flag in PIXELDQ of subarray image
assert(output.pixeldq[76, 100] == dqflags.pixel['DO_NOT_USE'])
assert(output.pixeldq[76, 104] == dqflags.pixel['DO_NOT_USE'])
# Pixel 84, 100 in subarray maps to 550, 100 in reference file
# Check that pixel was flagged 'NO_SAT_CHECK'
assert(output.pixeldq[84, 100] == dqflags.pixel['NO_SAT_CHECK'])
def test_signal_fluctuation_flagging(setup_nrc_cube):
'''Check that once a pixel is flagged as saturated in a group, all
subsequent groups should also be flagged as saturated, even if the
signal value drops back below saturation.'''
# Create inputs, data, and saturation maps
ngroups = 5
nrows = 2048
ncols = 2048
satvalue = 60000
data, satmap = setup_nrc_cube(ngroups, nrows, ncols)
# Add ramp values up to the saturation limit
data.data[0, 0, 500, 500] = 0
data.data[0, 1, 500, 500] = 20000
data.data[0, 2, 500, 500] = 40000
data.data[0, 3, 500, 500] = 60000 # Signal reaches saturation limit
data.data[0, 4, 500, 500] = 40000 # Signal drops below saturation limit
# Set saturation value in the saturation model
satmap.data[500, 500] = satvalue
# Run the pipeline
output = do_correction(data, satmap)
# Make sure that all groups after first saturated group are flagged
satindex = np.argmax(output.data[0, :, 500, 500] == satvalue)
assert np.all(output.groupdq[0, satindex:, 500, 500] == dqflags.group['SATURATED'])
def test_nans_in_mask(setup_nrc_cube):
'''Check that pixels in the reference files that have value NaN are not
flagged as saturated in the data and that in the PIXELDQ array the
pixel is set to NO_SAT_CHECK.'''
# Create inputs, data, and saturation maps
ngroups = 5
nrows = 2048
ncols = 2048
huge_num = 100000.
data, satmap = setup_nrc_cube(ngroups, nrows, ncols)
# Add ramp values up to the saturation limit
data.data[0, 0, 500, 500] = 0
data.data[0, 1, 500, 500] = 20000
data.data[0, 2, 500, 500] = 40000
data.data[0, 3, 500, 500] = 60000
data.data[0, 4, 500, 500] = 62000
# Set saturation value for pixel to NaN
satmap.data[500, 500] = np.nan
# Run the pipeline
correct_for_NaN(satmap.data, satmap.dq)
output = do_correction(data, satmap)
# Check that NaN reference value gets reset to HUGE_NUM
# Check that reference DQ is set to NO_SAT_CHECK
# Check that output GROUPDQ is not flagged as saturated
# Check that output PIXELDQ is set to NO_SAT_CHECK
assert satmap.data[500, 500] == huge_num
assert satmap.dq[500, 500] == dqflags.pixel['NO_SAT_CHECK']
assert np.all(output.groupdq[0, :, 500, 500] != dqflags.group['SATURATED'])
assert output.pixeldq[500, 500] == dqflags.pixel['NO_SAT_CHECK']
def test_no_sat_check(setup_nrc_cube):
'''Check that pixels flagged with NO_SAT_CHECK in the reference file get
added to the DQ mask and are not flagged as saturated.'''
# Create inputs, data, and saturation maps
ngroups = 5
nrows = 2048
ncols = 2048
satvalue = 60000
data, satmap = setup_nrc_cube(ngroups, nrows, ncols)
# Add ramp values up to the saturation limit
data.data[0, 0, 500, 500] = 0
data.data[0, 1, 500, 500] = 20000
data.data[0, 2, 500, 500] = 40000
data.data[0, 3, 500, 500] = 60000
data.data[0, 4, 500, 500] = 62000 # Signal reaches saturation limit
# Set saturation value in the saturation model & DQ value for NO_SAT_CHECK
satmap.data[500, 500] = satvalue
satmap.dq[500, 500] = dqflags.pixel['NO_SAT_CHECK']
# Run the pipeline
output = do_correction(data, satmap)
# Make sure output GROUPDQ does not get flagged as saturated
# Make sure PIXELDQ is set to NO_SAT_CHECK
assert np.all(output.groupdq[0, :, 500, 500] != dqflags.group['SATURATED'])
assert output.pixeldq[500, 500] == dqflags.pixel['NO_SAT_CHECK']
def test_no_sat_check(setup_nrc_cube):
'''Check that pixels flagged with NO_SAT_CHECK in the reference file get
added to the DQ mask and are not flagged as saturated.'''
# Create inputs, data, and saturation maps
ngroups = 5
nrows = 20
ncols = 20
satvalue = 60000
data, satmap = setup_nrc_cube(ngroups, nrows, ncols)
# Add ramp values up to the saturation limit
data.data[0, 0, 5, 5] = 0
data.data[0, 1, 5, 5] = 20000
data.data[0, 2, 5, 5] = 40000
data.data[0, 3, 5, 5] = 60000
data.data[0, 4, 5, 5] = 62000 # Signal reaches saturation limit
# Set saturation value in the saturation model & DQ value for NO_SAT_CHECK
satmap.data[5, 5] = satvalue
satmap.dq[5, 5] = dqflags.pixel['NO_SAT_CHECK']
# Also set an existing DQ flag in input science data
data.pixeldq[5, 5] = dqflags.pixel['RC']
# Run the pipeline
output = do_correction(data, satmap)
# Make sure output GROUPDQ does not get flagged as saturated
# Make sure PIXELDQ is set to NO_SAT_CHECK and original flag
assert np.all(output.groupdq[0, :, 5, 5] != dqflags.group['SATURATED'])
assert output.pixeldq[5, 5] == (dqflags.pixel['NO_SAT_CHECK'] +
dqflags.pixel['RC'])
def test_dq_propagation():
'''Test pixel DQ propagation.'''
ngroups = 5
nrows = 2048
ncols = 2048
dqval1 = 5
dqval2 = 10
data, satmap = setup_cube(ngroups, nrows, ncols)
data.pixeldq[5, 5] = dqval1
satmap.dq[5, 5] = dqval2
output = do_correction(data, satmap)
assert output.pixeldq[5, 5] == dqval1 + dqval2
def test_basic_saturation_flagging():
'''Test basic saturation flagging.'''
ngroups = 5
nrows = 2048
ncols = 2048
satvalue = 60000
data, satmap = setup_cube(ngroups, nrows, ncols)
data.data[0,0,500,500] = 0
data.data[0,1,500,500] = 20000
data.data[0,2,500,500] = 40000
data.data[0,3,500,500] = 60000 # signal reaches saturation limit
data.data[0,4,500,500] = 80000
satmap.data[500,500] = satvalue
output = do_correction(data, satmap)
satindex = np.argmax(output.data[0,:,500,500] == satvalue)
assert np.all(output.groupdq[0,satindex:,500,500] == dqflags.group['SATURATED'])
def test_no_sat_check():
'''Test pixels with NO_SAT_CHECK are not flagged.'''
ngroups = 5
nrows = 2048
ncols = 2048
satvalue = 60000
data, satmap = setup_cube(ngroups, nrows, ncols)
satmap.dq[500,500] = dqflags.pixel['NO_SAT_CHECK']
data.data[0,0,500,500] = 0
data.data[0,1,500,500] = 20000
data.data[0,2,500,500] = 40000
data.data[0,3,500,500] = 60000
data.data[0,4,500,500] = 80000
output = do_correction(data, satmap)
assert np.all(output.groupdq[0,:,500,500] != dqflags.group['SATURATED'])
assert output.pixeldq[500,500] == dqflags.pixel['NO_SAT_CHECK']
def test_first_group_saturation():
'''Test case where all groups are saturated.'''
ngroups = 5
nrows = 2048
ncols = 2048
satvalue = 60000
data, satmap = setup_cube(ngroups, nrows, ncols)
data.data[0,0,500,500] = 60000
data.data[0,1,500,500] = 70000
data.data[0,2,500,500] = 70000
data.data[0,3,500,500] = 60000
data.data[0,4,500,500] = 70000
satmap.data[500,500] = satvalue
output = do_correction(data, satmap)
satindex = np.argmax(output.data[0,:,500,500] == satvalue)
assert np.all(output.groupdq[0,satindex:,500,500] == dqflags.group['SATURATED'])
def test_signal_fluctuation_flagging():
'''Test case where signal drops after reaching saturation.'''
ngroups = 5
nrows = 2048
ncols = 2048
satvalue = 60000
data, satmap = setup_cube(ngroups, nrows, ncols)
data.data[0,0,500,500] = 0
data.data[0,1,500,500] = 20000
data.data[0,2,500,500] = 40000
data.data[0,3,500,500] = 60000
data.data[0,4,500,500] = 40000 # signal dips back below saturation limit
satmap.data[500,500] = satvalue
output = do_correction(data, satmap)
satindex = np.argmax(output.data[0,:,500,500] == satvalue)
assert np.all(output.groupdq[0,satindex:,500,500] == dqflags.group['SATURATED'])
def test_subarray_flagging():
'''Test subarray extraction and saturation flagging.'''
ngroups = 5
nrows = 200
ncols = 200
satvalue = 60000
data, satmap = setup_cube(ngroups, nrows, ncols)
print(data.data.shape,satmap.data.shape)
data.data[0,0,150,150] = 0
data.data[0,1,150,150] = 20000
data.data[0,2,150,150] = 40000
data.data[0,3,150,150] = 60000
data.data[0,4,150,150] = 80000
satmap.data[150,150] = satvalue
output = do_correction(data, satmap)
satindex = np.argmax(output.data[0,:,150,150] == satvalue)
assert np.all(output.groupdq[0,satindex:,150,150] == dqflags.group['SATURATED'])
def test_dq_propagation(setup_nrc_cube):
'''Check PIXELDQ propagation.'''
# Create inputs, data, and saturation maps
ngroups = 5
nrows = 2048
ncols = 2048
dqval1 = 5
dqval2 = 10
data, satmap = setup_nrc_cube(ngroups, nrows, ncols)
# Add DQ values to the data and reference file
data.pixeldq[5, 5] = dqval1
satmap.dq[5, 5] = dqval2
# Run the pipeline
output = do_correction(data, satmap)
# Make sure DQ values from data and reference file are added in the output
assert output.pixeldq[5, 5] == dqval1 + dqval2
def test_dq_propagation(setup_nrc_cube):
'''Check PIXELDQ propagation.'''
# Create inputs, data, and saturation maps
ngroups = 5
nrows = 20
ncols = 20
dqval1 = 5
dqval2 = 10
data, satmap = setup_nrc_cube(ngroups, nrows, ncols)
# Add DQ values to the data and reference file
data.pixeldq[5, 5] = dqval1
satmap.dq[5, 5] = dqval2
# Run the pipeline
output = do_correction(data, satmap)
# Make sure DQ values from data and reference file are added in the output
assert output.pixeldq[5, 5] == dqval1 + dqval2
