def test_proc(setup_inputs):
""""
A single CR in a 10 group exposure. Verify that the pixels flagged using
multiprocessing are identical to the pixels flagged when no
multiprocessing is done.
"""
grouptime = 3.0
ingain = 5
inreadnoise = 7.0
ngroups = 10
model, rnoise, gain = setup_inputs(ngroups=ngroups, nrows=25, ncols=6,
nints=2, gain=ingain,
readnoise=inreadnoise, deltatime=grouptime)
model.data[0, 0, 2, 3] = 15.0
model.data[0, 1, 2, 3] = 21.0
model.data[0, 2, 2, 3] = 25.0
model.data[0, 3, 2, 3] = 30.2
model.data[0, 4, 2, 3] = 35.0
model.data[0, 5, 2, 3] = 140.0
model.data[0, 6, 2, 3] = 151.0
model.data[0, 7, 2, 3] = 160.0
model.data[0, 8, 2, 3] = 170.0
model.data[0, 9, 2, 3] = 180.0
out_model_a = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, None, 200, 4, True)
out_model_b = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'half', 200, 4, True)
assert_array_equal(out_model_a.groupdq, out_model_b.groupdq)
out_model_c = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'all', 200, 4, True)
assert_array_equal(out_model_a.groupdq, out_model_c.groupdq)
def test_single_CR_neighbor_flag(setup_inputs):
""""
A single CR in a 10 group exposure. Tests that:
- if neighbor-flagging is set, the 4 neighboring pixels *ARE* flagged, and
- if neighbor-flagging is *NOT* set, the 4 neighboring pixels are *NOT* flagged
"""
grouptime = 3.0
ingain = 5
inreadnoise = 7.0
ngroups = 10
model, rnoise, gain = setup_inputs(ngroups=ngroups,
nrows=5,
ncols=6,
gain=ingain,
readnoise=inreadnoise,
deltatime=grouptime)
# two segments perfect fit, second segment has twice the slope
model.data[0, 0, 3, 3] = 15.0
model.data[0, 1, 3, 3] = 20.0
model.data[0, 2, 3, 3] = 25.0
model.data[0, 3, 3, 3] = 30.0
model.data[0, 4, 3, 3] = 35.0
model.data[0, 5, 3, 3] = 140.0
model.data[0, 6, 3, 3] = 150.0
model.data[0, 7, 3, 3] = 160.0
model.data[0, 8, 3, 3] = 170.0
model.data[0, 9, 3, 3] = 180.0
# Flag neighbors
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none',
200, 4, True)
assert np.max(out_model.groupdq[0, 5, 3, 3]) == JUMP_DET
assert out_model.groupdq[0, 5, 3, 4] == JUMP_DET
assert out_model.groupdq[0, 5, 3, 2] == JUMP_DET
assert out_model.groupdq[0, 5, 2, 3] == JUMP_DET
assert out_model.groupdq[0, 5, 4, 3] == JUMP_DET
# Do not flag neighbors
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none',
200, 4, False)
assert np.max(out_model.groupdq[0, 5, 3, 3]) == JUMP_DET
assert out_model.groupdq[0, 5, 3, 4] == GOOD
assert out_model.groupdq[0, 5, 3, 2] == GOOD
assert out_model.groupdq[0, 5, 2, 3] == GOOD
assert out_model.groupdq[0, 5, 4, 3] == GOOD
def test_exec_time_many_crs(setup_inputs):
""""
Set up with dimension similar to simulated MIRI datasets, Dataset has
many cosmic rays; approximately one CR per 4 groups. Test only the execution
time of run_detect_jumps() for comparison with nominal time; hopefully
indicative of faults with newly added code.
"""
nrows = 350
ncols = 400
model, rnoise, gain = setup_inputs(ngroups=10,
nrows=nrows,
ncols=ncols,
nints=2,
readnoise=6.5,
gain=5.5,
grouptime=2.775,
deltatime=2.775)
crs_frac = 0.25 # fraction of groups having a CR
model = add_crs(model, crs_frac) # add desired fraction of CRs
tstart = time.time()
# using dummy variable in next to prevent "F841-variable is assigned to but never used"
_ = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 4,
True)
tstop = time.time()
t_elapsed = tstop - tstart
MAX_TIME = 600 # takes ~100 sec on my Mac
assert t_elapsed < MAX_TIME
def test_exec_time_0_crs(setup_inputs):
""""
Set up with dimension similar to simulated MIRI datasets, Dataset has no
cosmic rays. Test only the execution time of run_detect_jumps() for
comparison with nominal time; hopefully indicative of faults with newly
added code.
"""
model, rnoise, gain = setup_inputs(ngroups=10,
nrows=1024,
ncols=1032,
nints=2,
readnoise=6.5,
gain=5.5,
grouptime=2.775,
deltatime=2.775)
tstart = time.time()
# using dummy variable in next to prevent "F841-variable is assigned to but never used"
_ = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none', 200, 4,
True)
tstop = time.time()
t_elapsed = tstop - tstart
MAX_TIME = 10 # takes 1.6 sec on my Mac
assert t_elapsed < MAX_TIME
def test_every_pixel_CR_neighbors_flagged(setup_inputs):
""""
A multiprocessing test that has a jump in every pixel. This is used
to test the performance gain from multiprocessing.
"""
grouptime = 3.0
ingain = 200
inreadnoise = 7.0
ngroups = 10
model, rnoise, gain = setup_inputs(ngroups=ngroups,
gain=ingain,
nrows=100,
ncols=100,
readnoise=inreadnoise,
deltatime=grouptime)
# two segments perfect fit, second segment has twice the slope
model.data[0, 0, :, :] = 15.0
model.data[0, 1, :, :] = 20.0
model.data[0, 2, :, :] = 25.0
model.data[0, 3, :, :] = 30.0
model.data[0, 4, :, :] = 35.0
model.data[0, 5, :, :] = 140.0
model.data[0, 6, :, :] = 150.0
model.data[0, 7, :, :] = 160.0
model.data[0, 8, :, :] = 170.0
model.data[0, 9, :, :] = 180.0
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'half',
200, 4, True)
assert np.max(out_model.groupdq[0, 5, 5, 5]) == JUMP_DET
assert out_model.groupdq[0, 5, 5, 6] == JUMP_DET
assert out_model.groupdq[0, 5, 5, 4] == JUMP_DET
assert out_model.groupdq[0, 5, 6, 5] == JUMP_DET
assert out_model.groupdq[0, 5, 4, 5] == JUMP_DET
def test_nocr_100_groups_nframes1(setup_inputs):
""""
NO CR in a 100 group exposure to make sure that frames_per_group is passed correctly to
twopoint_difference. This test recreates the problem found in issue #4571.
"""
grouptime = 3.0
ingain = 1
inreadnoise = 7.0
ngroups = 100
model, rnoise, gain = setup_inputs(ngroups=ngroups,
gain=ingain,
nframes=1,
nrows=10,
ncols=10,
readnoise=inreadnoise,
deltatime=grouptime)
# two segments perfect fit, second segment has twice the slope
model.data[0, 0, 5, 5] = 14.0
model.data[0, 1, 5, 5] = 20.0
model.data[0, 2, 5, 5] = 27.0
model.data[0, 3, 5, 5] = 30.0
model.data[0, 4, 5, 5] = 38.0
model.data[0, 5, 5, 5] = 40.0
model.data[0, 6, 5, 5] = 50.0
model.data[0, 7, 5, 5] = 52.0
model.data[0, 8, 5, 5] = 63.0
model.data[0, 9, 5, 5] = 68.0
for i in range(10, 100):
model.data[0, i, 5, 5] = i * 5
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none',
200, 4, True)
assert np.max(out_model.groupdq) == GOOD
def test_onecr_10_groups_neighbors_flagged(setup_inputs):
""""
A single CR in a 10 group exposure
"""
grouptime = 3.0
ingain = 200
inreadnoise = 7.0
ngroups = 10
model, rnoise, gain = setup_inputs(ngroups=ngroups,
gain=ingain,
nrows=10,
ncols=10,
readnoise=inreadnoise,
deltatime=grouptime)
# two segments perfect fit, second segment has twice the slope
model.data[0, 0, 5, 5] = 15.0
model.data[0, 1, 5, 5] = 20.0
model.data[0, 2, 5, 5] = 25.0
model.data[0, 3, 5, 5] = 30.0
model.data[0, 4, 5, 5] = 35.0
model.data[0, 5, 5, 5] = 140.0
model.data[0, 6, 5, 5] = 150.0
model.data[0, 7, 5, 5] = 160.0
model.data[0, 8, 5, 5] = 170.0
model.data[0, 9, 5, 5] = 180.0
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none',
200, 4, True)
assert np.max(out_model.groupdq[0, 5, 5, 5]) == JUMP_DET
assert out_model.groupdq[0, 5, 5, 6] == JUMP_DET
assert out_model.groupdq[0, 5, 5, 4] == JUMP_DET
assert out_model.groupdq[0, 5, 6, 5] == JUMP_DET
assert out_model.groupdq[0, 5, 4, 5] == JUMP_DET
def test_onecr_10_groups(setup_inputs):
""""
A test to make sure that neighbors are not flagged when they are not requested to be flagged.
"""
grouptime = 3.0
ingain = 200
inreadnoise = 7.0
ngroups = 10
model, rnoise, gain = setup_inputs(ngroups=ngroups,
gain=ingain,
nrows=20,
ncols=20,
readnoise=inreadnoise,
deltatime=grouptime)
# two segments perfect fit, second segment has twice the slope
model.data[0, 0, 5, 5] = 15.0
model.data[0, 1, 5, 5] = 20.0
model.data[0, 2, 5, 5] = 25.0
model.data[0, 3, 5, 5] = 30.0
model.data[0, 4, 5, 5] = 35.0
model.data[0, 5, 5, 5] = 140.0
model.data[0, 6, 5, 5] = 150.0
model.data[0, 7, 5, 5] = 160.0
model.data[0, 8, 5, 5] = 170.0
model.data[0, 9, 5, 5] = 180.0
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none',
200, 10, False)
assert out_model.groupdq[0, 5, 5, 5] == JUMP_DET
assert out_model.groupdq[0, 5, 4, 5] == GOOD
assert out_model.groupdq[0, 5, 6, 5] == GOOD
assert out_model.groupdq[0, 5, 5, 6] == GOOD
assert out_model.groupdq[0, 5, 5, 4] == GOOD
def test_nocrs_noflux(setup_inputs):
""""
All pixel values are zero. So slope should be zero
"""
model, rnoise, gain = setup_inputs(ngroups=5)
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, True)
assert np.max(out_model.groupdq) == GOOD
def test_onecr_50_groups(setup_inputs):
""""
A test with a fifty group integration. There are two jumps in pixel 5,5. One in group 5 and
one in group 30.
"""
grouptime = 3.0
ingain = 5
inreadnoise = 7.0
ngroups = 50
model, rnoise, gain = setup_inputs(ngroups=ngroups, gain=ingain, nrows=10, ncols=10,
readnoise=inreadnoise, deltatime=grouptime)
model.data[0, 0, 5, 5] = 15.0
model.data[0, 1, 5, 5] = 20.0
model.data[0, 2, 5, 5] = 25.0
model.data[0, 3, 5, 5] = 30.0
model.data[0, 4, 5, 5] = 35.0
model.data[0, 5, 5, 5] = 140.0
model.data[0, 6, 5, 5] = 150.0
model.data[0, 7, 5, 5] = 160.0
model.data[0, 8, 5, 5] = 170.0
model.data[0, 9, 5, 5] = 180.0
model.data[0, 10:30, 5, 5] = np.arange(190, 290, 5)
model.data[0, 30:50, 5, 5] = np.arange(500, 600, 5)
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 10, False)
# CR in group 5
assert out_model.groupdq[0, 5, 5, 5] == JUMP_DET
# CR in group 30
assert out_model.groupdq[0, 30, 5, 5] == JUMP_DET
# groups in between are not flagged
assert_array_equal(out_model.groupdq[0, 6:30, 5, 5], GOOD)
def test_nirspec_saturated_pix(setup_inputs):
"""
This test is based on an actual NIRSpec exposure that has some pixels
flagged as saturated in one or more groups, which the jump step is
supposed to ignore, but an old version of the code was setting JUMP flags
for some of the saturated groups. This is to verify that the saturated
groups are no longer flagged with jumps.
"""
grouptime = 3.0
ingain = 1.0
inreadnoise = 10.7
ngroups = 7
nrows = 6
ncols = 6
model, rnoise, gain = setup_inputs(ngroups=ngroups,
nints=1,
nrows=nrows,
ncols=ncols,
gain=ingain,
readnoise=inreadnoise,
deltatime=grouptime)
# Setup the needed input pixel and DQ values
model.data[0, :, 1, 1] = [
639854.75, 4872.451, -17861.791, 14022.15, 22320.176, 1116.3828,
1936.9746
]
model.groupdq[0, :, 1, 1] = [0, 0, 0, 0, 0, 2, 2]
model.data[0, :, 2, 2] = [
8.25666812e+05, -1.10471914e+05, 1.95755371e+02, 1.83118457e+03,
1.72250879e+03, 1.81733496e+03, 1.65188281e+03
]
# 2 non-sat groups means only 1 non-sat diff, so no jumps should be flagged
model.groupdq[0, :, 2, 2] = [0, 0, 2, 2, 2, 2, 2]
model.data[0, :, 3, 3] = [
1228767., 46392.234, -3245.6553, 7762.413, 37190.76, 266611.62,
5072.4434
]
model.groupdq[0, :, 3, 3] = [0, 0, 0, 0, 0, 0, 2]
# run jump detection
out_model = run_detect_jumps(model,
gain,
rnoise,
rejection_thresh=200.0,
three_grp_thresh=200,
four_grp_thresh=200,
max_cores='none',
max_jump_to_flag_neighbors=200,
min_jump_to_flag_neighbors=10,
flag_4_neighbors=True)
# Check the results. There should not be any pixels with DQ values of 6, which
# is saturated (2) plus jump (4). All the DQ's should be either just 2 or just 4.
assert_array_equal(out_model.groupdq[0, :, 1, 1], [0, 4, 4, 4, 0, 2, 2])
# assert that no groups are flagged when theres only 1 non-sat. grp
assert_array_equal(out_model.groupdq[0, :, 2, 2], [0, 0, 2, 2, 2, 2, 2])
assert_array_equal(out_model.groupdq[0, :, 3, 3], [0, 4, 4, 0, 0, 4, 2])
def test_flagging_of_CRs_across_slice_boundaries(setup_inputs):
""""
A multiprocessing test that has two CRs on the boundary between two slices.
This makes sure that we are correctly flagging neighbors in different slices.
"""
grouptime = 3.0
ingain = 200
inreadnoise = 7.0
ngroups = 10
model, rnoise, gain = setup_inputs(ngroups=ngroups,
nints=2,
gain=ingain,
readnoise=inreadnoise,
deltatime=grouptime)
nrows = model.data.shape[3]
num_cores = multiprocessing.cpu_count()
max_cores = 'half'
numslices = num_cores // 2
if numslices > 1:
yincrement = int(nrows / numslices)
# two segments perfect fit, second segment has twice the slope
# add a CR on the last row of the first slice
model.data[0, 0, yincrement - 1, 5] = 15.0
model.data[0, 1, yincrement - 1, 5] = 20.0
model.data[0, 2, yincrement - 1, 5] = 25.0
model.data[0, 3, yincrement - 1, 5] = 30.0
model.data[0, 4, yincrement - 1, 5] = 35.0
model.data[0, 5, yincrement - 1, 5] = 140.0
model.data[0, 6, yincrement - 1, 5] = 150.0
model.data[0, 7, yincrement - 1, 5] = 160.0
model.data[0, 8, yincrement - 1, 5] = 170.0
model.data[0, 9, yincrement - 1, 5] = 180.0
# add a CR on the first row of the second slice
model.data[1, 0, yincrement, 25] = 15.0
model.data[1, 1, yincrement, 25] = 20.0
model.data[1, 2, yincrement, 25] = 25.0
model.data[1, 3, yincrement, 25] = 30.0
model.data[1, 4, yincrement, 25] = 35.0
model.data[1, 5, yincrement, 25] = 40.0
model.data[1, 6, yincrement, 25] = 50.0
model.data[1, 7, yincrement, 25] = 160.0
model.data[1, 8, yincrement, 25] = 170.0
model.data[1, 9, yincrement, 25] = 180.0
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0,
max_cores, 200, 4, True)
# check that the neighbors of the CR on the last row were flagged
assert out_model.groupdq[0, 5, yincrement - 1, 5] == JUMP_DET
assert out_model.groupdq[0, 5, yincrement - 1, 6] == JUMP_DET
assert out_model.groupdq[0, 5, yincrement - 1, 4] == JUMP_DET
assert out_model.groupdq[0, 5, yincrement, 5] == JUMP_DET
assert out_model.groupdq[0, 5, yincrement - 2, 5] == JUMP_DET
# check that the neighbors of the CR on the first row were flagged
assert out_model.groupdq[1, 7, yincrement, 25] == JUMP_DET
assert out_model.groupdq[1, 7, yincrement, 26] == JUMP_DET
assert out_model.groupdq[1, 7, yincrement, 24] == JUMP_DET
assert out_model.groupdq[1, 7, yincrement + 1, 25] == JUMP_DET
assert out_model.groupdq[1, 7, yincrement - 1, 25] == JUMP_DET
def test_nocrs_noflux_subarray(setup_inputs):
""""
All pixel values are zero. This shows that the subarray reference files get
extracted from the full frame versions.
"""
model, rnoise, gain = setup_inputs(ngroups=5, subarray=True)
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, True)
assert np.max(out_model.groupdq) == GOOD
def test_twoints_onecr_10_groups_neighbors_flagged_multi(setup_inputs):
""""
A multiprocessing test that has two CRs on the boundary between two slices
in different integrations. This makes sure that we are correctly flagging
neighbors in different slices and that we are parsing the integrations correctly.
"""
grouptime = 3.0
ingain = 200
inreadnoise = 7.0
ngroups = 10
model, rnoise, gain = setup_inputs(ngroups=ngroups,
nints=2,
gain=ingain,
nrows=40,
ncols=10,
readnoise=inreadnoise,
deltatime=grouptime)
# two segments perfect fit, second segment has twice the slope
model.data[0, 0, 5, 5] = 15.0
model.data[0, 1, 5, 5] = 20.0
model.data[0, 2, 5, 5] = 25.0
model.data[0, 3, 5, 5] = 30.0
model.data[0, 4, 5, 5] = 35.0
model.data[0, 5, 5, 5] = 140.0
model.data[0, 6, 5, 5] = 150.0
model.data[0, 7, 5, 5] = 160.0
model.data[0, 8, 5, 5] = 170.0
model.data[0, 9, 5, 5] = 180.0
model.data[1, 0, 15, 5] = 15.0
model.data[1, 1, 15, 5] = 20.0
model.data[1, 2, 15, 5] = 25.0
model.data[1, 3, 15, 5] = 30.0
model.data[1, 4, 15, 5] = 35.0
model.data[1, 5, 15, 5] = 40.0
model.data[1, 6, 15, 5] = 45.0
model.data[1, 7, 15, 5] = 160.0
model.data[1, 8, 15, 5] = 170.0
model.data[1, 9, 15, 5] = 180.0
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'half',
200, 4, True)
assert np.max(out_model.groupdq[0, 5, 5, 5]) == JUMP_DET
assert out_model.groupdq[0, 5, 5, 6] == JUMP_DET
assert out_model.groupdq[0, 5, 5, 4] == JUMP_DET
assert out_model.groupdq[0, 5, 6, 5] == JUMP_DET
assert out_model.groupdq[0, 5, 4, 5] == JUMP_DET
assert out_model.groupdq[1, 7, 15, 5] == JUMP_DET
assert out_model.groupdq[1, 7, 15, 6] == JUMP_DET
assert out_model.groupdq[1, 7, 15, 4] == JUMP_DET
assert out_model.groupdq[1, 7, 16, 5] == JUMP_DET
assert out_model.groupdq[1, 7, 14, 5] == JUMP_DET
def test_twoints_onecr_each_10_groups_neighbors_flagged(setup_inputs):
""""
Two integrations with CRs in different locations. This makes sure we are correctly
dealing with integrations.
"""
grouptime = 3.0
ingain = 200
inreadnoise = 7.0
ngroups = 10
model, rnoise, gain = setup_inputs(ngroups=ngroups,
nints=2,
gain=ingain,
nrows=20,
ncols=20,
readnoise=inreadnoise,
deltatime=grouptime)
# two segments perfect fit, second segment has twice the slope
model.data[0, 0, 5, 5] = 15.0
model.data[0, 1, 5, 5] = 20.0
model.data[0, 2, 5, 5] = 25.0
model.data[0, 3, 5, 5] = 30.0
model.data[0, 4, 5, 5] = 35.0
model.data[0, 5, 5, 5] = 140.0
model.data[0, 6, 5, 5] = 150.0
model.data[0, 7, 5, 5] = 160.0
model.data[0, 8, 5, 5] = 170.0
model.data[0, 9, 5, 5] = 180.0
model.data[1, 0, 15, 5] = 15.0
model.data[1, 1, 15, 5] = 20.0
model.data[1, 2, 15, 5] = 25.0
model.data[1, 3, 15, 5] = 30.0
model.data[1, 4, 15, 5] = 35.0
model.data[1, 5, 15, 5] = 40.0
model.data[1, 6, 15, 5] = 45.0
model.data[1, 7, 15, 5] = 160.0
model.data[1, 8, 15, 5] = 170.0
model.data[1, 9, 15, 5] = 180.0
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none',
200, 4, True)
assert np.max(out_model.groupdq[0, 5, 5, 5]) == JUMP_DET
assert out_model.groupdq[0, 5, 5, 6] == JUMP_DET
assert out_model.groupdq[0, 5, 5, 4] == JUMP_DET
assert out_model.groupdq[0, 5, 6, 5] == JUMP_DET
assert out_model.groupdq[0, 5, 4, 5] == JUMP_DET
assert out_model.groupdq[1, 7, 15, 5] == JUMP_DET
assert out_model.groupdq[1, 7, 15, 6] == JUMP_DET
assert out_model.groupdq[1, 7, 15, 4] == JUMP_DET
assert out_model.groupdq[1, 7, 16, 5] == JUMP_DET
assert out_model.groupdq[1, 7, 14, 5] == JUMP_DET
def test_nocrs_noflux_badgain_pixel(setup_inputs):
""""
all pixel values are zero. So slope should be zero, pixel with bad gain should
have pixel dq set to 'NO_GAIN_VALUE' and 'DO_NOT_USE'
"""
model, rnoise, gain = setup_inputs(ngroups=5, nrows=20, ncols=20)
gain.data[7, 7] = -10 # bad gain
gain.data[17, 17] = np.nan # bad gain
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, True)
# 2 bits are set for each pixel, so use bitwise_and to check is set
assert np.bitwise_and(out_model.pixeldq[7, 7], NO_GAIN_VALUE) == NO_GAIN_VALUE
assert np.bitwise_and(out_model.pixeldq[7, 7], DO_NOT_USE) == DO_NOT_USE
assert np.bitwise_and(out_model.pixeldq[17, 17], NO_GAIN_VALUE) == NO_GAIN_VALUE
assert np.bitwise_and(out_model.pixeldq[17, 17], DO_NOT_USE) == DO_NOT_USE
def test_onecr_10_groups_fullarray(setup_inputs):
""""
A test that has a cosmic ray in the 5th group for all pixels except column 10. In column
10 the jump is in the 7th group.
"""
grouptime = 3.0
ingain = 5
inreadnoise = 7.0
ngroups = 10
model, rnoise, gain = setup_inputs(ngroups=ngroups,
gain=ingain,
nrows=20,
ncols=20,
readnoise=inreadnoise,
deltatime=grouptime)
model.data[0, 0, 5, :] = 15.0
model.data[0, 1, 5, :] = 20.0
model.data[0, 2, 5, :] = 25.0
model.data[0, 3, 5, :] = 30.0
model.data[0, 4, 5, :] = 35.0
model.data[0, 5, 5, :] = 140.0
model.data[0, 6, 5, :] = 150.0
model.data[0, 7, 5, :] = 160.0
model.data[0, 8, 5, :] = 170.0
model.data[0, 9, 5, :] = 180.0
# move the CR to group 7 for row 10 and make difference be 300
model.data[0, 3, 5, 10] = 100
model.data[0, 4, 5, 10] = 130
model.data[0, 5, 5, 10] = 160
model.data[0, 6, 5, 10] = 190
model.data[0, 7, 5, 10] = 400
model.data[0, 8, 5, 10] = 410
model.data[0, 9, 5, 10] = 420
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none',
200, 10, False)
# The jump is in group 5 for columns 0-9
assert_array_equal(out_model.groupdq[0, 5, 5, 0:10], JUMP_DET)
# The jump is in group 7 for column 10
assert out_model.groupdq[0, 7, 5, 10] == JUMP_DET
# The jump is in group 5 for columns 11+
assert_array_equal(out_model.groupdq[0, 5, 5, 11:], JUMP_DET)
def test_adjacent_CRs(setup_inputs):
"""
Three CRs in a 10 group exposure; the CRs have overlapping neighboring
pixels. This test makes sure that the correct pixels are flagged.
"""
grouptime = 3.0
ingain = 5
inreadnoise = 7.0
ngroups = 10
model, rnoise, gain = setup_inputs(ngroups=ngroups,
nrows=15,
ncols=6,
gain=ingain,
readnoise=inreadnoise,
deltatime=grouptime)
# Populate arrays for 1st CR, centered at (x=2, y=3)
x = 2
y = 3
model.data[0, 0, y, x] = 15.0
model.data[0, 1, y, x] = 20.0
model.data[0, 2, y, x] = 26.0
model.data[0, 3, y, x] = 30.0
model.data[0, 4, y, x] = 35.0
model.data[0, 5, y, x] = 140.0
model.data[0, 6, y, x] = 150.0
model.data[0, 7, y, x] = 161.0
model.data[0, 8, y, x] = 170.0
model.data[0, 9, y, x] = 180.0
# Populate arrays for 2nd CR, centered at (x=2, y=2)
x = 2
y = 2
model.data[0, 0, y, x] = 20.0
model.data[0, 1, y, x] = 30.0
model.data[0, 2, y, x] = 41.0
model.data[0, 3, y, x] = 51.0
model.data[0, 4, y, x] = 62.0
model.data[0, 5, y, x] = 170.0
model.data[0, 6, y, x] = 200.0
model.data[0, 7, y, x] = 231.0
model.data[0, 8, y, x] = 260.0
model.data[0, 9, y, x] = 290.0
# Populate arrays for 3rd CR, centered at (x=3, y=2)
x = 3
y = 2
model.data[0, 0, y, x] = 120.0
model.data[0, 1, y, x] = 140.0
model.data[0, 2, y, x] = 161.0
model.data[0, 3, y, x] = 181.0
model.data[0, 4, y, x] = 202.0
model.data[0, 5, y, x] = 70.0
model.data[0, 6, y, x] = 100.0
model.data[0, 7, y, x] = 131.0
model.data[0, 8, y, x] = 160.0
model.data[0, 9, y, x] = 190.0
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'half',
200, 4, True)
# 1st CR (centered at x=2, y=3)
assert out_model.groupdq[0, 5, 2, 2] == JUMP_DET
assert out_model.groupdq[0, 5, 3, 1] == JUMP_DET
assert out_model.groupdq[0, 5, 3, 2] == JUMP_DET
assert out_model.groupdq[0, 5, 3, 3] == JUMP_DET
assert out_model.groupdq[0, 5, 4, 2] == JUMP_DET
# 2nd CR (centered at x=2, y=2)
assert out_model.groupdq[0, 5, 1, 2] == JUMP_DET
assert out_model.groupdq[0, 5, 2, 1] == JUMP_DET
assert out_model.groupdq[0, 5, 2, 3] == JUMP_DET
# 3rd CR (centered at x=3, y=2)
assert out_model.groupdq[0, 5, 1, 3] == JUMP_DET
assert out_model.groupdq[0, 5, 2, 4] == JUMP_DET
def test_crs_on_edge_with_neighbor_flagging(setup_inputs):
""""
A test to make sure that the neighbors of CRs on the edges of the
array are flagged correctly.
"""
grouptime = 3.0
ingain = 200
inreadnoise = 7.0
ngroups = 10
model, rnoise, gain = setup_inputs(ngroups=ngroups,
nrows=20,
ncols=20,
gain=ingain,
readnoise=inreadnoise,
deltatime=grouptime)
# two segments perfect fit, second segment has twice the slope
# CR on 1st row
model.data[0, 0, 0, 15] = 15.0
model.data[0, 1, 0, 15] = 20.0
model.data[0, 2, 0, 15] = 25.0
model.data[0, 3, 0, 15] = 30.0
model.data[0, 4, 0, 15] = 35.0
model.data[0, 5, 0, 15] = 140.0
model.data[0, 6, 0, 15] = 150.0
model.data[0, 7, 0, 15] = 160.0
model.data[0, 8, 0, 15] = 170.0
model.data[0, 9, 0, 15] = 180.0
# CR on last row
model.data[0, 0, -1, 5] = 15.0
model.data[0, 1, -1, 5] = 20.0
model.data[0, 2, -1, 5] = 25.0
model.data[0, 3, -1, 5] = 30.0
model.data[0, 4, -1, 5] = 35.0
model.data[0, 5, -1, 5] = 140.0
model.data[0, 6, -1, 5] = 150.0
model.data[0, 7, -1, 5] = 160.0
model.data[0, 8, -1, 5] = 170.0
model.data[0, 9, -1, 5] = 180.0
# CR on 1st column
model.data[0, 0, 5, 0] = 15.0
model.data[0, 1, 5, 0] = 20.0
model.data[0, 2, 5, 0] = 25.0
model.data[0, 3, 5, 0] = 30.0
model.data[0, 4, 5, 0] = 35.0
model.data[0, 5, 5, 0] = 140.0
model.data[0, 6, 5, 0] = 150.0
model.data[0, 7, 5, 0] = 160.0
model.data[0, 8, 5, 0] = 170.0
model.data[0, 9, 5, 0] = 180.0
# CR on last column
model.data[0, 0, 15, -1] = 15.0
model.data[0, 1, 15, -1] = 20.0
model.data[0, 2, 15, -1] = 25.0
model.data[0, 3, 15, -1] = 30.0
model.data[0, 4, 15, -1] = 35.0
model.data[0, 5, 15, -1] = 140.0
model.data[0, 6, 15, -1] = 150.0
model.data[0, 7, 15, -1] = 160.0
model.data[0, 8, 15, -1] = 170.0
model.data[0, 9, 15, -1] = 180.0
out_model = run_detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'none',
200, 10, True)
# flag CR and three neighbors of first row CR
assert out_model.groupdq[0, 5, 0, 15] == JUMP_DET
assert out_model.groupdq[0, 5, 1, 15] == JUMP_DET
assert out_model.groupdq[0, 5, 0, 14] == JUMP_DET
assert out_model.groupdq[0, 5, 0, 16] == JUMP_DET
assert out_model.groupdq[0, 5, -1, 15] == 0 # The one not to flag
# flag CR and three neighbors of last row CR
assert out_model.groupdq[0, 5, -1, 5] == JUMP_DET
assert out_model.groupdq[0, 5, -2, 5] == JUMP_DET
assert out_model.groupdq[0, 5, -1, 4] == JUMP_DET
assert out_model.groupdq[0, 5, -1, 6] == JUMP_DET
# flag CR and three neighbors of first column CR
assert out_model.groupdq[0, 5, 5, 0] == JUMP_DET
assert out_model.groupdq[0, 5, 6, 0] == JUMP_DET
assert out_model.groupdq[0, 5, 4, 0] == JUMP_DET
assert out_model.groupdq[0, 5, 5, 1] == JUMP_DET
assert out_model.groupdq[0, 5, 5, -1] == 0 # The one not to flag
# flag CR and three neighbors of last column CR
assert out_model.groupdq[0, 5, 15, -1] == JUMP_DET
assert out_model.groupdq[0, 5, 15, -2] == JUMP_DET
assert out_model.groupdq[0, 5, 16, -1] == JUMP_DET
assert out_model.groupdq[0, 5, 14, -1] == JUMP_DET
def test_multiple_neighbor_jumps_firstlastbad(setup_inputs):
"""
This test is based on actual MIRI data that was having the incorrect
group flagged with JUMP_DET (it was flagging group 2 instead of group 5).
This makes sure that group 5 is getting flagged.
Note that the first and last frames/groups are all flagged with DO_NOT_USE,
due to the application of the first/last frame steps.
"""
grouptime = 3.0
ingain = 5.5
inreadnoise = 6.5
ngroups = 10
nrows = 10
ncols = 10
model, rnoise, gain = setup_inputs(ngroups=ngroups,
nints=1,
nrows=nrows,
ncols=ncols,
gain=ingain,
readnoise=inreadnoise,
deltatime=grouptime)
# Setup the desired pixel values
model.data[0, :, 1, 1] = [
10019.966, 10057.298, 10078.248, 10096.01, 20241.627, 20248.752,
20268.047, 20284.895, 20298.705, 20314.25
]
model.data[0, :, 1, 2] = [
10016.457, 10053.907, 10063.568, 10076.166, 11655.773, 11654.063,
11681.795, 11693.763, 11712.788, 11736.994
]
model.data[0, :, 1, 3] = [
10013.259, 10050.348, 10070.398, 10097.658, 10766.534, 10787.84,
10802.418, 10818.872, 10832.695, 10861.175
]
model.data[0, :, 1, 4] = [
10016.422, 10053.959, 10070.934, 10090.381, 10104.014, 10127.665,
10143.687, 10172.227, 10178.138, 10199.59
]
model.data[0, :, 2, 1] = [
10021.067, 10042.973, 10059.062, 10069.323, 18732.406, 18749.602,
18771.908, 18794.695, 18803.223, 18819.523
]
model.data[0, :, 2, 2] = [
10019.651, 10043.371, 10056.423, 10085.121, 40584.703, 40606.08,
40619.51, 40629.574, 40641.9, 40660.145
]
model.data[0, :, 2, 3] = [
10021.223, 10042.112, 10052.958, 10067.142, 28188.316, 28202.922,
28225.557, 28243.79, 28253.883, 28273.586
]
model.data[0, :, 2, 4] = [
10022.608, 10037.174, 10069.476, 10081.729, 11173.748, 11177.344,
11201.127, 11219.607, 11229.468, 11243.174
]
model.data[0, :, 2, 5] = [
10011.095, 10047.422, 10061.066, 10079.375, 10106.405, 10116.071,
10129.348, 10136.305, 10161.373, 10181.479
]
model.data[0, :, 3, 1] = [
10011.877, 10052.809, 10075.108, 10085.111, 10397.106, 10409.291,
10430.475, 10445.3, 10462.004, 10484.906
]
model.data[0, :, 3, 2] = [
10012.124, 10059.202, 10078.984, 10092.74, 11939.488, 11958.45,
11977.5625, 11991.776, 12025.897, 12027.326
]
model.data[0, :, 3, 3] = [
10013.282, 10046.887, 10062.308, 10085.447, 28308.426, 28318.957,
28335.55, 28353.832, 28371.746, 28388.848
]
model.data[0, :, 3, 4] = [
10016.784, 10048.249, 10060.097, 10074.606, 21506.082, 21522.027,
21542.309, 21558.34, 21576.365, 21595.58
]
model.data[0, :, 3, 5] = [
10014.916, 10052.995, 10063.7705, 10092.866, 10538.075, 10558.318,
10570.754, 10597.343, 10608.488, 10628.104
]
model.data[0, :, 4, 1] = [
10017.438, 10038.94, 10057.657, 10069.987, 10090.22, 10114.296,
10133.543, 10148.657, 10158.109, 10172.842
]
model.data[0, :, 4, 2] = [
10011.129, 10037.982, 10054.445, 10079.703, 10097.964, 10110.593,
10135.701, 10149.448, 10171.771, 10185.874
]
model.data[0, :, 4, 3] = [
10021.109, 10043.658, 10063.909, 10072.364, 10766.232, 10774.402,
10790.677, 10809.337, 10833.65, 10849.55
]
model.data[0, :, 4, 4] = [
10023.877, 10035.997, 10052.321, 10077.937, 10529.645, 10541.947,
10571.127, 10577.249, 10599.716, 10609.544
]
# Flag first and last frame as DO_NOT_USE
model.groupdq[0, 0, :, :] = 1
model.groupdq[0, -1, :, :] = 1
# run jump detection
out_model = run_detect_jumps(model,
gain,
rnoise,
rejection_thresh=200.0,
three_grp_thresh=200,
four_grp_thresh=200,
max_cores='none',
max_jump_to_flag_neighbors=200,
min_jump_to_flag_neighbors=10,
flag_4_neighbors=True)
# Verify that the correct groups have been flagged. The entries for pixels
# 2,2 and 3,3 are the ones that had previously been flagged in group 2 instead
# of group 5.
assert_array_equal(out_model.groupdq[0, :, 1, 1],
[1, 0, 0, 0, 4, 0, 0, 0, 0, 1])
assert_array_equal(out_model.groupdq[0, :, 1, 2],
[1, 0, 0, 0, 4, 0, 0, 0, 0, 1])
assert_array_equal(out_model.groupdq[0, :, 1, 3],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 1])
assert_array_equal(out_model.groupdq[0, :, 1, 4],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 1])
assert_array_equal(out_model.groupdq[0, :, 2, 1],
[1, 0, 0, 0, 4, 0, 0, 0, 0, 1])
assert_array_equal(out_model.groupdq[0, :, 2, 2],
[1, 0, 0, 0, 4, 0, 0, 0, 0, 1])
assert_array_equal(out_model.groupdq[0, :, 2, 3],
[1, 0, 0, 0, 4, 0, 0, 0, 0, 1])
assert_array_equal(out_model.groupdq[0, :, 2, 4],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 1])
assert_array_equal(out_model.groupdq[0, :, 3, 1],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 1])
assert_array_equal(out_model.groupdq[0, :, 3, 2],
[1, 0, 0, 0, 4, 0, 0, 0, 0, 1])
assert_array_equal(out_model.groupdq[0, :, 3, 3],
[1, 0, 0, 0, 4, 0, 0, 0, 0, 1])
assert_array_equal(out_model.groupdq[0, :, 3, 4],
[1, 0, 0, 0, 4, 0, 0, 0, 0, 1])
assert_array_equal(out_model.groupdq[0, :, 4, 1],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 1])
assert_array_equal(out_model.groupdq[0, :, 4, 2],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 1])
assert_array_equal(out_model.groupdq[0, :, 4, 3],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 1])
assert_array_equal(out_model.groupdq[0, :, 4, 4],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 1])