def test_proc(setup_inputs): """" A single CR in a 10 group exposure. Verify that the pixels flagged using multiprocessing are identical to the pixels flagged when no multiprocessing is done. """ grouptime = 3.0 ingain = 5 inreadnoise = np.float64(7) ngroups = 10 model1, gdq, rnModel, pixdq, err, gain = \ setup_inputs( ngroups=ngroups, nrows=5, ncols=6, nints=2, gain=ingain, readnoise=inreadnoise, deltatime=grouptime ) model1.data[0, 0, 2, 3] = 15.0 model1.data[0, 1, 2, 3] = 21.0 model1.data[0, 2, 2, 3] = 25.0 model1.data[0, 3, 2, 3] = 30.2 model1.data[0, 4, 2, 3] = 35.0 model1.data[0, 5, 2, 3] = 140.0 model1.data[0, 6, 2, 3] = 151.0 model1.data[0, 7, 2, 3] = 160.0 model1.data[0, 8, 2, 3] = 170.0 model1.data[0, 9, 2, 3] = 180.0 out_model_a = detect_jumps(model1, gain, rnModel, 4.0, 'half', 200, 4, True) out_model_b = detect_jumps(model1, gain, rnModel, 4.0, None, 200, 4, True) assert (out_model_a.groupdq == out_model_b.groupdq).all() out_model_c = detect_jumps(model1, gain, rnModel, 4.0, 'All', 200, 4, True) assert (out_model_a.groupdq == out_model_c.groupdq).all()
def test_single_CR_neighbor_flag(setup_inputs): """" A single CR in a 10 group exposure. Tests that: - if neighbor-flagging is set, the 4 neighboring pixels *ARE* flagged, and - if neighbor-flagging is *NOT* set, the 4 neighboring pixels are *NOT* flagged """ grouptime = 3.0 ingain = 5 inreadnoise = 7.0 ngroups = 10 model, rnoise, gain = setup_inputs(ngroups=ngroups, nrows=5, ncols=6, gain=ingain, readnoise=inreadnoise, deltatime=grouptime) # two segments perfect fit, second segment has twice the slope model.data[0, 0, 3, 3] = 15.0 model.data[0, 1, 3, 3] = 20.0 model.data[0, 2, 3, 3] = 25.0 model.data[0, 3, 3, 3] = 30.0 model.data[0, 4, 3, 3] = 35.0 model.data[0, 5, 3, 3] = 140.0 model.data[0, 6, 3, 3] = 150.0 model.data[0, 7, 3, 3] = 160.0 model.data[0, 8, 3, 3] = 170.0 model.data[0, 9, 3, 3] = 180.0 # Flag neighbors out_model = detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, True) assert np.max(out_model.groupdq[0, 5, 3, 3]) == JUMP_DET assert out_model.groupdq[0, 5, 3, 4] == JUMP_DET assert out_model.groupdq[0, 5, 3, 2] == JUMP_DET assert out_model.groupdq[0, 5, 2, 3] == JUMP_DET assert out_model.groupdq[0, 5, 4, 3] == JUMP_DET # Do not flag neighbors out_model = detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, False) assert np.max(out_model.groupdq[0, 5, 3, 3]) == JUMP_DET assert out_model.groupdq[0, 5, 3, 4] == GOOD assert out_model.groupdq[0, 5, 3, 2] == GOOD assert out_model.groupdq[0, 5, 2, 3] == GOOD assert out_model.groupdq[0, 5, 4, 3] == GOOD
def test_nocrs_noflux(setup_inputs): """" All pixel values are zero. So slope should be zero """ model1, rnModel, gain = setup_inputs(ngroups=5) out_model = detect_jumps(model1, gain, rnModel, 4.0, 1, 200, 4, True) assert (0 == np.max(out_model.groupdq))
def test_onecr_10_groups_neighbors_flagged(setup_inputs): """" A single CR in a 10 group exposure """ grouptime = 3.0 ingain = 200 inreadnoise = np.float64(7) ngroups = 10 model1, rnModel, gain = setup_inputs(ngroups=ngroups, gain=ingain, nrows=10, ncols=10, readnoise=inreadnoise, deltatime=grouptime) # two segments perfect fit, second segment has twice the slope model1.data[0, 0, 5, 5] = 15.0 model1.data[0, 1, 5, 5] = 20.0 model1.data[0, 2, 5, 5] = 25.0 model1.data[0, 3, 5, 5] = 30.0 model1.data[0, 4, 5, 5] = 35.0 model1.data[0, 5, 5, 5] = 140.0 model1.data[0, 6, 5, 5] = 150.0 model1.data[0, 7, 5, 5] = 160.0 model1.data[0, 8, 5, 5] = 170.0 model1.data[0, 9, 5, 5] = 180.0 out_model = detect_jumps(model1, gain, rnModel, 4.0, 1, 200, 4, True) assert (4 == np.max(out_model.groupdq[0, 5, 5, 5])) assert (4 == out_model.groupdq[0, 5, 5, 6]) assert (4 == out_model.groupdq[0, 5, 5, 4]) assert (4 == out_model.groupdq[0, 5, 6, 5]) assert (4 == out_model.groupdq[0, 5, 4, 5])
def test_every_pixel_CR_neighbors_flagged(setup_inputs): """" A multiprocessing test that has a jump in every pixel. This is used to test the performance gain from multiprocessing. """ grouptime = 3.0 ingain = 200 inreadnoise = np.float64(7) ngroups = 10 model1, rnModel, gain = setup_inputs(ngroups=ngroups, gain=ingain, readnoise=inreadnoise, deltatime=grouptime) # two segments perfect fit, second segment has twice the slope model1.data[0, 0, :, :] = 15.0 model1.data[0, 1, :, :] = 20.0 model1.data[0, 2, :, :] = 25.0 model1.data[0, 3, :, :] = 30.0 model1.data[0, 4, :, :] = 35.0 model1.data[0, 5, :, :] = 140.0 model1.data[0, 6, :, :] = 150.0 model1.data[0, 7, :, :] = 160.0 model1.data[0, 8, :, :] = 170.0 model1.data[0, 9, :, :] = 180.0 out_model = detect_jumps(model1, gain, rnModel, 4.0, 'half', 200, 4, True) assert (4 == np.max(out_model.groupdq[0, 5, 5, 5])) assert (4 == out_model.groupdq[0, 5, 5, 6]) assert (4 == out_model.groupdq[0, 5, 5, 4]) assert (4 == out_model.groupdq[0, 5, 6, 5]) assert (4 == out_model.groupdq[0, 5, 4, 5])
def test_onecr_50_groups(setup_inputs): grouptime = 3.0 ingain = 5 # use large gain to show that Poisson noise doesn't affect the recombination inreadnoise = np.float64(7) ngroups = 50 model1, gdq, rnModel, pixdq, err, gain = setup_inputs( ngroups=ngroups, gain=ingain, readnoise=inreadnoise, deltatime=grouptime) # two segments perfect fit, second segment has twice the slope model1.data[0, 0, 5, 5] = 15.0 model1.data[0, 1, 5, 5] = 20.0 model1.data[0, 2, 5, 5] = 25.0 model1.data[0, 3, 5, 5] = 30.0 model1.data[0, 4, 5, 5] = 35.0 model1.data[0, 5, 5, 5] = 140.0 model1.data[0, 6, 5, 5] = 150.0 model1.data[0, 7, 5, 5] = 160.0 model1.data[0, 8, 5, 5] = 170.0 model1.data[0, 9, 5, 5] = 180.0 model1.data[0, 10:29, 5, 5] = 190.0 model1.data[0, 30:49, 5, 5] = 490.0 out_model = detect_jumps(model1, gain, rnModel, 4.0, False, 4.0) assert (4 == np.max(out_model.groupdq[0, 5, 5, 5])) outdqcr = out_model.groupdq[0, 5, 5, 5] np.testing.assert_allclose(4, outdqcr)
def test_onecr_10_groups_fullarray(setup_inputs): grouptime = 3.0 ingain = 5 # use large gain to show that Poisson noise doesn't affect the recombination inreadnoise = np.float64(7) ngroups = 10 model1, gdq, rnModel, pixdq, err, gain = setup_inputs( ngroups=ngroups, gain=ingain, readnoise=inreadnoise, deltatime=grouptime) # two segments perfect fit, second segment has twice the slope model1.data[0, 0, 5, :] = 15.0 model1.data[0, 1, 5, :] = 20.0 model1.data[0, 2, 5, :] = 25.0 model1.data[0, 3, 5, :] = 30.0 model1.data[0, 4, 5, :] = 35.0 model1.data[0, 5, 5, :] = 140.0 model1.data[0, 6, 5, :] = 150.0 model1.data[0, 7, 5, :] = 160.0 model1.data[0, 8, 5, :] = 170.0 model1.data[0, 9, 5, :] = 180.0 # move the CR to group 3 for row 10 and make difference be 30 model1.data[0, 3, 5, 10] = 100 model1.data[0, 4, 5, 10] = 130 model1.data[0, 5, 5, 10] = 160 model1.data[0, 6, 5, 10] = 190 model1.data[0, 7, 5, 10] = 220 model1.data[0, 8, 5, 10] = 250 model1.data[0, 9, 5, 10] = 280 out_model = detect_jumps(model1, gain, rnModel, 4.0, False, 4.0) assert (4 == np.max(out_model.groupdq[0, 5, :, :]))
def test_onecr_10_groups(setup_inputs): """" A test to make sure that neighbors are not flagged when they are not requested to be flagged. """ grouptime = 3.0 ingain = 200 inreadnoise = np.float64(7) ngroups = 10 model1, rnModel, gain = setup_inputs(ngroups=ngroups, nrows=20, ncols=20, gain=ingain, readnoise=inreadnoise, deltatime=grouptime) # two segments perfect fit, second segment has twice the slope model1.data[0, 0, 5, 5] = 15.0 model1.data[0, 1, 5, 5] = 20.0 model1.data[0, 2, 5, 5] = 25.0 model1.data[0, 3, 5, 5] = 30.0 model1.data[0, 4, 5, 5] = 35.0 model1.data[0, 5, 5, 5] = 140.0 model1.data[0, 6, 5, 5] = 150.0 model1.data[0, 7, 5, 5] = 160.0 model1.data[0, 8, 5, 5] = 170.0 model1.data[0, 9, 5, 5] = 180.0 out_model = detect_jumps(model1, gain, rnModel, 4.0, 1, 200, 10, False) assert (out_model.groupdq[0, 5, 5, 5] == 4) assert (out_model.groupdq[0, 5, 4, 5] == 0) assert (out_model.groupdq[0, 5, 6, 5] == 0) assert (out_model.groupdq[0, 5, 5, 6] == 0) assert (out_model.groupdq[0, 5, 5, 4] == 0)
def test_onecr_50_groups(setup_inputs): """" A test with a fifty group integration. There are two jumps in pixel 5,5. One in group 5 and one in group 30. """ grouptime = 3.0 ingain = 5 inreadnoise = np.float64(7) ngroups = 50 model1, rnModel, gain = setup_inputs(ngroups=ngroups, nrows=10, ncols=10, gain=ingain, readnoise=inreadnoise, deltatime=grouptime) model1.data[0, 0, 5, 5] = 15.0 model1.data[0, 1, 5, 5] = 20.0 model1.data[0, 2, 5, 5] = 25.0 model1.data[0, 3, 5, 5] = 30.0 model1.data[0, 4, 5, 5] = 35.0 model1.data[0, 5, 5, 5] = 140.0 model1.data[0, 6, 5, 5] = 150.0 model1.data[0, 7, 5, 5] = 160.0 model1.data[0, 8, 5, 5] = 170.0 model1.data[0, 9, 5, 5] = 180.0 model1.data[0, 10:30, 5, 5] = np.arange(190, 290, 5) model1.data[0, 30:50, 5, 5] = np.arange(500, 600, 5) out_model = detect_jumps(model1, gain, rnModel, 4.0, 1, 200, 10, False) assert (out_model.groupdq[0, 5, 5, 5] == 4) # CR in group 5 assert (out_model.groupdq[0, 30, 5, 5] == 4) # CR in group 30 assert (np.all(out_model.groupdq[0, 6:30, 5, 5] == 0) ) # groups in between are not flagged
def test_flagging_of_CRs_across_slice_boundaries(setup_inputs): """" A multiprocessing test that has two CRs on the boundary between two slices. This makes sure that we are correctly flagging neighbors in different slices. """ grouptime = 3.0 ingain = 200 inreadnoise = np.float64(7) ngroups = 10 model1, rnModel, gain = setup_inputs(ngroups=ngroups, nints=2, gain=ingain, readnoise=inreadnoise, deltatime=grouptime) nrows = model1.data.shape[3] num_cores = multiprocessing.cpu_count() max_cores = 'half' numslices = num_cores // 2 if numslices > 1: yincrement = int(nrows / numslices) # two segments perfect fit, second segment has twice the slope # add a CR on the last row of the first slice model1.data[0, 0, yincrement - 1, 5] = 15.0 model1.data[0, 1, yincrement - 1, 5] = 20.0 model1.data[0, 2, yincrement - 1, 5] = 25.0 model1.data[0, 3, yincrement - 1, 5] = 30.0 model1.data[0, 4, yincrement - 1, 5] = 35.0 model1.data[0, 5, yincrement - 1, 5] = 140.0 model1.data[0, 6, yincrement - 1, 5] = 150.0 model1.data[0, 7, yincrement - 1, 5] = 160.0 model1.data[0, 8, yincrement - 1, 5] = 170.0 model1.data[0, 9, yincrement - 1, 5] = 180.0 # add a CR on the first row of the second slice model1.data[1, 0, yincrement, 25] = 15.0 model1.data[1, 1, yincrement, 25] = 20.0 model1.data[1, 2, yincrement, 25] = 25.0 model1.data[1, 3, yincrement, 25] = 30.0 model1.data[1, 4, yincrement, 25] = 35.0 model1.data[1, 5, yincrement, 25] = 40.0 model1.data[1, 6, yincrement, 25] = 50.0 model1.data[1, 7, yincrement, 25] = 160.0 model1.data[1, 8, yincrement, 25] = 170.0 model1.data[1, 9, yincrement, 25] = 180.0 out_model = detect_jumps(model1, gain, rnModel, 4.0, max_cores, 200, 4, True) # check that the neighbors of the CR on the last row were flagged assert (4 == out_model.groupdq[0, 5, yincrement - 1, 5]) assert (4 == out_model.groupdq[0, 5, yincrement - 1, 6]) assert (4 == out_model.groupdq[0, 5, yincrement - 1, 4]) assert (4 == out_model.groupdq[0, 5, yincrement, 5]) assert (4 == out_model.groupdq[0, 5, yincrement - 2, 5]) # check that the neighbors of the CR on the first row were flagged assert (4 == out_model.groupdq[1, 7, yincrement, 25]) assert (4 == out_model.groupdq[1, 7, yincrement, 26]) assert (4 == out_model.groupdq[1, 7, yincrement, 24]) assert (4 == out_model.groupdq[1, 7, yincrement + 1, 25]) assert (4 == out_model.groupdq[1, 7, yincrement - 1, 25])
def test_nocr_100_groups_nframes1(setup_inputs): """" NO CR in a 100 group exposure to make sure that frames_per_group is passed correctly to twopoint_difference. This test recreates the problem found in issue #4571. """ grouptime = 3.0 ingain = 1 # to make the noise calculation simple inreadnoise = np.float64(7) ngroups = 100 model1, rnModel, gain = setup_inputs(ngroups=ngroups, nrows=10, ncols=10, gain=ingain, readnoise=inreadnoise, deltatime=grouptime) model1.meta.exposure.nframes = 1 # two segments perfect fit, second segment has twice the slope model1.data[0, 0, 5, 5] = 14.0 model1.data[0, 1, 5, 5] = 20.0 model1.data[0, 2, 5, 5] = 27.0 model1.data[0, 3, 5, 5] = 30.0 model1.data[0, 4, 5, 5] = 38.0 model1.data[0, 5, 5, 5] = 40.0 model1.data[0, 6, 5, 5] = 50.0 model1.data[0, 7, 5, 5] = 52.0 model1.data[0, 8, 5, 5] = 63.0 model1.data[0, 9, 5, 5] = 68.0 for i in range(10, 100): model1.data[0, i, 5, 5] = i * 5 out_model = detect_jumps(model1, gain, rnModel, 4.0, 1, 200, 4, True) assert (0 == np.max(out_model.groupdq))
def test_nocrs_noflux_subarray(setup_inputs): """" All pixel values are zero. This shows that the subarray reference files get extracted from the full frame versions. """ model1, rnModel, gain = setup_inputs(ngroups=5, subarray=True) out_model = detect_jumps(model1, gain, rnModel, 4.0, 1, 200, 4, True) assert (0 == np.max(out_model.groupdq))
def test_twoints_onecr_10_groups_neighbors_flagged_multi(setup_inputs): """" A multiprocessing test that has two CRs on the boundary between two slices in different integrations. This makes sure that we are correctly flagging neighbors in different slices and that we are parsing the integrations correctly. """ grouptime = 3.0 ingain = 200 inreadnoise = 7.0 ngroups = 10 model, rnoise, gain = setup_inputs(ngroups=ngroups, nints=2, gain=ingain, nrows=40, ncols=10, readnoise=inreadnoise, deltatime=grouptime) # two segments perfect fit, second segment has twice the slope model.data[0, 0, 5, 5] = 15.0 model.data[0, 1, 5, 5] = 20.0 model.data[0, 2, 5, 5] = 25.0 model.data[0, 3, 5, 5] = 30.0 model.data[0, 4, 5, 5] = 35.0 model.data[0, 5, 5, 5] = 140.0 model.data[0, 6, 5, 5] = 150.0 model.data[0, 7, 5, 5] = 160.0 model.data[0, 8, 5, 5] = 170.0 model.data[0, 9, 5, 5] = 180.0 model.data[1, 0, 15, 5] = 15.0 model.data[1, 1, 15, 5] = 20.0 model.data[1, 2, 15, 5] = 25.0 model.data[1, 3, 15, 5] = 30.0 model.data[1, 4, 15, 5] = 35.0 model.data[1, 5, 15, 5] = 40.0 model.data[1, 6, 15, 5] = 45.0 model.data[1, 7, 15, 5] = 160.0 model.data[1, 8, 15, 5] = 170.0 model.data[1, 9, 15, 5] = 180.0 out_model = detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'half', 200, 4, True) assert np.max(out_model.groupdq[0, 5, 5, 5]) == JUMP_DET assert out_model.groupdq[0, 5, 5, 6] == JUMP_DET assert out_model.groupdq[0, 5, 5, 4] == JUMP_DET assert out_model.groupdq[0, 5, 6, 5] == JUMP_DET assert out_model.groupdq[0, 5, 4, 5] == JUMP_DET assert out_model.groupdq[1, 7, 15, 5] == JUMP_DET assert out_model.groupdq[1, 7, 15, 6] == JUMP_DET assert out_model.groupdq[1, 7, 15, 4] == JUMP_DET assert out_model.groupdq[1, 7, 16, 5] == JUMP_DET assert out_model.groupdq[1, 7, 14, 5] == JUMP_DET
def test_proc(setup_inputs): """" A single CR in a 10 group exposure. Verify that the pixels flagged using multiprocessing are identical to the pixels flagged when no multiprocessing is done. """ grouptime = 3.0 ingain = 5 inreadnoise = 7.0 ngroups = 10 model, rnoise, gain = setup_inputs(ngroups=ngroups, nrows=25, ncols=6, nints=2, gain=ingain, readnoise=inreadnoise, deltatime=grouptime) model.data[0, 0, 2, 3] = 15.0 model.data[0, 1, 2, 3] = 21.0 model.data[0, 2, 2, 3] = 25.0 model.data[0, 3, 2, 3] = 30.2 model.data[0, 4, 2, 3] = 35.0 model.data[0, 5, 2, 3] = 140.0 model.data[0, 6, 2, 3] = 151.0 model.data[0, 7, 2, 3] = 160.0 model.data[0, 8, 2, 3] = 170.0 model.data[0, 9, 2, 3] = 180.0 out_model_a = detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, None, 200, 4, True) out_model_b = detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'half', 200, 4, True) assert_array_equal(out_model_a.groupdq, out_model_b.groupdq) out_model_c = detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 'all', 200, 4, True) assert_array_equal(out_model_a.groupdq, out_model_c.groupdq)
def test_twoints_onecr_each_10_groups_neighbors_flagged(setup_inputs): """" Two integrations with CRs in different locations. This makes sure we are correctly dealing with integrations. """ grouptime = 3.0 ingain = 200 inreadnoise = 7.0 ngroups = 10 model, rnoise, gain = setup_inputs(ngroups=ngroups, nints=2, gain=ingain, nrows=20, ncols=20, readnoise=inreadnoise, deltatime=grouptime) # two segments perfect fit, second segment has twice the slope model.data[0, 0, 5, 5] = 15.0 model.data[0, 1, 5, 5] = 20.0 model.data[0, 2, 5, 5] = 25.0 model.data[0, 3, 5, 5] = 30.0 model.data[0, 4, 5, 5] = 35.0 model.data[0, 5, 5, 5] = 140.0 model.data[0, 6, 5, 5] = 150.0 model.data[0, 7, 5, 5] = 160.0 model.data[0, 8, 5, 5] = 170.0 model.data[0, 9, 5, 5] = 180.0 model.data[1, 0, 15, 5] = 15.0 model.data[1, 1, 15, 5] = 20.0 model.data[1, 2, 15, 5] = 25.0 model.data[1, 3, 15, 5] = 30.0 model.data[1, 4, 15, 5] = 35.0 model.data[1, 5, 15, 5] = 40.0 model.data[1, 6, 15, 5] = 45.0 model.data[1, 7, 15, 5] = 160.0 model.data[1, 8, 15, 5] = 170.0 model.data[1, 9, 15, 5] = 180.0 out_model = detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 4, True) assert np.max(out_model.groupdq[0, 5, 5, 5]) == JUMP_DET assert out_model.groupdq[0, 5, 5, 6] == JUMP_DET assert out_model.groupdq[0, 5, 5, 4] == JUMP_DET assert out_model.groupdq[0, 5, 6, 5] == JUMP_DET assert out_model.groupdq[0, 5, 4, 5] == JUMP_DET assert out_model.groupdq[1, 7, 15, 5] == JUMP_DET assert out_model.groupdq[1, 7, 15, 6] == JUMP_DET assert out_model.groupdq[1, 7, 15, 4] == JUMP_DET assert out_model.groupdq[1, 7, 16, 5] == JUMP_DET assert out_model.groupdq[1, 7, 14, 5] == JUMP_DET
def test_nocrs_noflux_badgain_pixel(setup_inputs): """" all pixel values are zero. So slope should be zero, pixel with bad gain should have pixel dq set to 'NO_GAIN_VALUE' and 'DO_NOT_USE' """ model, rnoise, gain = setup_inputs(ngroups=5, nrows=20, ncols=20) gain.data[7, 7] = -10 # bad gain gain.data[17, 17] = np.nan # bad gain out_model = detect_jumps(model, gain, rnoise, 4.0, 1, 200, 4, True) # 2 bits are set for each pixel, so use bitwise_and to check is set assert np.bitwise_and(out_model.pixeldq[7, 7], NO_GAIN_VALUE) == NO_GAIN_VALUE assert np.bitwise_and(out_model.pixeldq[7, 7], DO_NOT_USE) == DO_NOT_USE assert np.bitwise_and(out_model.pixeldq[17, 17], NO_GAIN_VALUE) == NO_GAIN_VALUE assert np.bitwise_and(out_model.pixeldq[17, 17], DO_NOT_USE) == DO_NOT_USE
def test_nocrs_noflux_badgain_pixel(setup_inputs): """" all pixel values are zero. So slope should be zero, pixel with bad gain should have pixel dq set to 'NO_GAIN_VALUE' and 'DO_NOT_USE' """ model1, rnModel, gain = setup_inputs(ngroups=5, nrows=20, ncols=20) gain.data[7, 7] = -10 # bad gain gain.data[17, 17] = np.nan # bad gain out_model = detect_jumps(model1, gain, rnModel, 4.0, 1, 200, 4, True) assert (np.bitwise_and(out_model.pixeldq[7, 7], dqflags.pixel['NO_GAIN_VALUE'])) assert (np.bitwise_and(out_model.pixeldq[7, 7], dqflags.pixel['DO_NOT_USE'])) assert (np.bitwise_and(out_model.pixeldq[17, 17], dqflags.pixel['NO_GAIN_VALUE'])) assert (np.bitwise_and(out_model.pixeldq[17, 17], dqflags.pixel['DO_NOT_USE']))
def test_onecr_10_groups_fullarray(setup_inputs): """" A test that has a cosmic ray in the 5th group for all pixels except column 10. In column 10 the jump is in the 7th group. """ grouptime = 3.0 ingain = 5 inreadnoise = np.float64(7) ngroups = 10 model1, rnModel, gain = setup_inputs(ngroups=ngroups, gain=ingain, nrows=20, ncols=20, readnoise=inreadnoise, deltatime=grouptime) model1.data[0, 0, 5, :] = 15.0 model1.data[0, 1, 5, :] = 20.0 model1.data[0, 2, 5, :] = 25.0 model1.data[0, 3, 5, :] = 30.0 model1.data[0, 4, 5, :] = 35.0 model1.data[0, 5, 5, :] = 140.0 model1.data[0, 6, 5, :] = 150.0 model1.data[0, 7, 5, :] = 160.0 model1.data[0, 8, 5, :] = 170.0 model1.data[0, 9, 5, :] = 180.0 # move the CR to group 7 for row 10 and make difference be 300 model1.data[0, 3, 5, 10] = 100 model1.data[0, 4, 5, 10] = 130 model1.data[0, 5, 5, 10] = 160 model1.data[0, 6, 5, 10] = 190 model1.data[0, 7, 5, 10] = 400 model1.data[0, 8, 5, 10] = 410 model1.data[0, 9, 5, 10] = 420 out_model = detect_jumps(model1, gain, rnModel, 4.0, 1, 200, 10, False) assert (np.all(out_model.groupdq[0, 5, 5, 0:10] == 4) ) # The jump is in group 5 for columns 0-9 assert (out_model.groupdq[0, 7, 5, 10] == 4) # The jump is in group 7 for column 10 assert (np.all(out_model.groupdq[0, 5, 5, 11:] == 4) ) # The jump is in group 5 for columns 11+
def process(self, input): with models.open(input) as input_model: # Check for an input model with NGROUPS <=2 if input_model.meta.exposure.ngroups<=2: self.log.warn('Can not apply jump detection when NGROUPS<=2;') self.log.warn('Jump step will be skipped') result = input_model.copy() result.meta.cal_step.jump = 'SKIPPED' return result # Retrieve the parameter values rej_thresh = self.rejection_threshold do_yint = self.do_yintercept sig_thresh = self.yint_threshold self.log.info('CR rejection threshold = %g sigma', rej_thresh) if do_yint: self.log.info('Y-intercept signal threshold = %g', sig_thresh) # Get the gain and readnoise reference files gain_filename = self.get_reference_file( input_model, 'gain') self.log.info('Using GAIN reference file: %s', gain_filename) gain_model = models.GainModel( gain_filename ) readnoise_filename = self.get_reference_file( input_model, 'readnoise') self.log.info('Using READNOISE reference file: %s', readnoise_filename) readnoise_model = models.ReadnoiseModel( readnoise_filename ) # Call the jump detection routine result = detect_jumps( input_model, gain_model, readnoise_model, rej_thresh, do_yint, sig_thresh ) gain_model.close() readnoise_model.close() result.meta.cal_step.jump = 'COMPLETE' return result
def process(self, input): with models.open(input) as input_model: # Check for an input model with NGROUPS <=2 if input_model.meta.exposure.ngroups <= 2: self.log.warn('Can not apply jump detection when NGROUPS<=2;') self.log.warn('Jump step will be skipped') result = input_model.copy() result.meta.cal_step.jump = 'SKIPPED' return result # Retrieve the parameter values rej_thresh = self.rejection_threshold do_yint = self.do_yintercept sig_thresh = self.yint_threshold self.log.info('CR rejection threshold = %g sigma', rej_thresh) if do_yint: self.log.info('Y-intercept signal threshold = %g', sig_thresh) # Get the gain and readnoise reference files gain_filename = self.get_reference_file(input_model, 'gain') self.log.info('Using GAIN reference file: %s', gain_filename) gain_model = models.GainModel(gain_filename) readnoise_filename = self.get_reference_file( input_model, 'readnoise') self.log.info('Using READNOISE reference file: %s', readnoise_filename) readnoise_model = models.ReadnoiseModel(readnoise_filename) # Call the jump detection routine result = detect_jumps(input_model, gain_model, readnoise_model, rej_thresh, do_yint, sig_thresh) gain_model.close() readnoise_model.close() result.meta.cal_step.jump = 'COMPLETE' return result
def test_onecr_50_groups(setup_inputs): """" A test with a fifty group integration. There are two jumps in pixel 5,5. One in group 5 and one in group 30. """ grouptime = 3.0 ingain = 5 inreadnoise = 7.0 ngroups = 50 model, rnoise, gain = setup_inputs(ngroups=ngroups, gain=ingain, nrows=10, ncols=10, readnoise=inreadnoise, deltatime=grouptime) model.data[0, 0, 5, 5] = 15.0 model.data[0, 1, 5, 5] = 20.0 model.data[0, 2, 5, 5] = 25.0 model.data[0, 3, 5, 5] = 30.0 model.data[0, 4, 5, 5] = 35.0 model.data[0, 5, 5, 5] = 140.0 model.data[0, 6, 5, 5] = 150.0 model.data[0, 7, 5, 5] = 160.0 model.data[0, 8, 5, 5] = 170.0 model.data[0, 9, 5, 5] = 180.0 model.data[0, 10:30, 5, 5] = np.arange(190, 290, 5) model.data[0, 30:50, 5, 5] = np.arange(500, 600, 5) out_model = detect_jumps(model, gain, rnoise, 4.0, 5.0, 6.0, 1, 200, 10, False) # CR in group 5 assert out_model.groupdq[0, 5, 5, 5] == JUMP_DET # CR in group 30 assert out_model.groupdq[0, 30, 5, 5] == JUMP_DET # groups in between are not flagged assert_array_equal(out_model.groupdq[0, 6:30, 5, 5], GOOD)
def test_multiple_neighbor_jumps_firstlastbad(setup_inputs): """ This test is based on actual MIRI data that was having the incorrect group flagged with JUMP_DET (it was flagging group 2 instead of group 5). This makes sure that group 5 is getting flagged. Note that the first and last frames/groups are all flagged with DO_NOT_USE, due to the application of the first/last frame steps. """ grouptime = 3.0 ingain = 5.5 inreadnoise = 6.5 ngroups = 10 nrows = 10 ncols = 10 model1, rnModel, gain = setup_inputs(ngroups=ngroups, nints=1, nrows=nrows, ncols=ncols, gain=ingain, readnoise=inreadnoise, deltatime=grouptime) # Setup the desired pixel values model1.data[0, :, 1, 1] = [ 10019.966, 10057.298, 10078.248, 10096.01, 20241.627, 20248.752, 20268.047, 20284.895, 20298.705, 20314.25 ] model1.data[0, :, 1, 2] = [ 10016.457, 10053.907, 10063.568, 10076.166, 11655.773, 11654.063, 11681.795, 11693.763, 11712.788, 11736.994 ] model1.data[0, :, 1, 3] = [ 10013.259, 10050.348, 10070.398, 10097.658, 10766.534, 10787.84, 10802.418, 10818.872, 10832.695, 10861.175 ] model1.data[0, :, 1, 4] = [ 10016.422, 10053.959, 10070.934, 10090.381, 10104.014, 10127.665, 10143.687, 10172.227, 10178.138, 10199.59 ] model1.data[0, :, 2, 1] = [ 10021.067, 10042.973, 10059.062, 10069.323, 18732.406, 18749.602, 18771.908, 18794.695, 18803.223, 18819.523 ] model1.data[0, :, 2, 2] = [ 10019.651, 10043.371, 10056.423, 10085.121, 40584.703, 40606.08, 40619.51, 40629.574, 40641.9, 40660.145 ] model1.data[0, :, 2, 3] = [ 10021.223, 10042.112, 10052.958, 10067.142, 28188.316, 28202.922, 28225.557, 28243.79, 28253.883, 28273.586 ] model1.data[0, :, 2, 4] = [ 10022.608, 10037.174, 10069.476, 10081.729, 11173.748, 11177.344, 11201.127, 11219.607, 11229.468, 11243.174 ] model1.data[0, :, 2, 5] = [ 10011.095, 10047.422, 10061.066, 10079.375, 10106.405, 10116.071, 10129.348, 10136.305, 10161.373, 10181.479 ] model1.data[0, :, 3, 1] = [ 10011.877, 10052.809, 10075.108, 10085.111, 10397.106, 10409.291, 10430.475, 10445.3, 10462.004, 10484.906 ] model1.data[0, :, 3, 2] = [ 10012.124, 10059.202, 10078.984, 10092.74, 11939.488, 11958.45, 11977.5625, 11991.776, 12025.897, 12027.326 ] model1.data[0, :, 3, 3] = [ 10013.282, 10046.887, 10062.308, 10085.447, 28308.426, 28318.957, 28335.55, 28353.832, 28371.746, 28388.848 ] model1.data[0, :, 3, 4] = [ 10016.784, 10048.249, 10060.097, 10074.606, 21506.082, 21522.027, 21542.309, 21558.34, 21576.365, 21595.58 ] model1.data[0, :, 3, 5] = [ 10014.916, 10052.995, 10063.7705, 10092.866, 10538.075, 10558.318, 10570.754, 10597.343, 10608.488, 10628.104 ] model1.data[0, :, 4, 1] = [ 10017.438, 10038.94, 10057.657, 10069.987, 10090.22, 10114.296, 10133.543, 10148.657, 10158.109, 10172.842 ] model1.data[0, :, 4, 2] = [ 10011.129, 10037.982, 10054.445, 10079.703, 10097.964, 10110.593, 10135.701, 10149.448, 10171.771, 10185.874 ] model1.data[0, :, 4, 3] = [ 10021.109, 10043.658, 10063.909, 10072.364, 10766.232, 10774.402, 10790.677, 10809.337, 10833.65, 10849.55 ] model1.data[0, :, 4, 4] = [ 10023.877, 10035.997, 10052.321, 10077.937, 10529.645, 10541.947, 10571.127, 10577.249, 10599.716, 10609.544 ] model1.groupdq[0, 0, :, :] = 1 # Flag first frame as DO_NOT_USE model1.groupdq[0, -1, :, :] = 1 # Flag last frame as DO_NOT_USE # run jump detection out_model = detect_jumps(model1, gain, rnModel, rejection_threshold=200.0, max_cores=None, max_jump_to_flag_neighbors=200, min_jump_to_flag_neighbors=10, flag_4_neighbors=True) # Verify that the correct groups have been flagged. The entries for pixels # 2,2 and 3,3 are the ones that had previously been flagged in group 2 instead # of group 5. assert_array_equal(out_model.groupdq[0, :, 1, 1], [1, 0, 0, 0, 4, 0, 0, 0, 0, 1]) assert_array_equal(out_model.groupdq[0, :, 1, 2], [1, 0, 0, 0, 4, 0, 0, 0, 0, 1]) assert_array_equal(out_model.groupdq[0, :, 1, 3], [1, 0, 0, 0, 0, 0, 0, 0, 0, 1]) assert_array_equal(out_model.groupdq[0, :, 1, 4], [1, 0, 0, 0, 0, 0, 0, 0, 0, 1]) assert_array_equal(out_model.groupdq[0, :, 2, 1], [1, 0, 0, 0, 4, 0, 0, 0, 0, 1]) assert_array_equal(out_model.groupdq[0, :, 2, 2], [1, 0, 0, 0, 4, 0, 0, 0, 0, 1]) # <-- assert_array_equal(out_model.groupdq[0, :, 2, 3], [1, 0, 0, 0, 4, 0, 0, 0, 0, 1]) assert_array_equal(out_model.groupdq[0, :, 2, 4], [1, 0, 0, 0, 0, 0, 0, 0, 0, 1]) assert_array_equal(out_model.groupdq[0, :, 3, 1], [1, 0, 0, 0, 0, 0, 0, 0, 0, 1]) assert_array_equal(out_model.groupdq[0, :, 3, 2], [1, 0, 0, 0, 4, 0, 0, 0, 0, 1]) assert_array_equal(out_model.groupdq[0, :, 3, 3], [1, 0, 0, 0, 4, 0, 0, 0, 0, 1]) # <-- assert_array_equal(out_model.groupdq[0, :, 3, 4], [1, 0, 0, 0, 4, 0, 0, 0, 0, 1]) assert_array_equal(out_model.groupdq[0, :, 4, 1], [1, 0, 0, 0, 0, 0, 0, 0, 0, 1]) assert_array_equal(out_model.groupdq[0, :, 4, 2], [1, 0, 0, 0, 0, 0, 0, 0, 0, 1]) assert_array_equal(out_model.groupdq[0, :, 4, 3], [1, 0, 0, 0, 0, 0, 0, 0, 0, 1]) assert_array_equal(out_model.groupdq[0, :, 4, 4], [1, 0, 0, 0, 0, 0, 0, 0, 0, 1])
def test_adjacent_CRs(setup_inputs): """ Three CRs in a 10 group exposure; the CRs have overlapping neighboring pixels. This test makes sure that the correct pixels are flagged. """ grouptime = 3.0 ingain = 5 inreadnoise = np.float64(7) ngroups = 10 model1, rnModel, gain = setup_inputs(ngroups=ngroups, nrows=15, ncols=6, gain=ingain, readnoise=inreadnoise, deltatime=grouptime) # Populate arrays for 1st CR, centered at (x=2, y=3) x = 2 y = 3 model1.data[0, 0, y, x] = 15.0 model1.data[0, 1, y, x] = 20.0 model1.data[0, 2, y, x] = 26.0 model1.data[0, 3, y, x] = 30.0 model1.data[0, 4, y, x] = 35.0 model1.data[0, 5, y, x] = 140.0 model1.data[0, 6, y, x] = 150.0 model1.data[0, 7, y, x] = 161.0 model1.data[0, 8, y, x] = 170.0 model1.data[0, 9, y, x] = 180.0 # Populate arrays for 2nd CR, centered at (x=2, y=2) x = 2 y = 2 model1.data[0, 0, y, x] = 20.0 model1.data[0, 1, y, x] = 30.0 model1.data[0, 2, y, x] = 41.0 model1.data[0, 3, y, x] = 51.0 model1.data[0, 4, y, x] = 62.0 model1.data[0, 5, y, x] = 170.0 model1.data[0, 6, y, x] = 200.0 model1.data[0, 7, y, x] = 231.0 model1.data[0, 8, y, x] = 260.0 model1.data[0, 9, y, x] = 290.0 # Populate arrays for 3rd CR, centered at (x=3, y=2) x = 3 y = 2 model1.data[0, 0, y, x] = 120.0 model1.data[0, 1, y, x] = 140.0 model1.data[0, 2, y, x] = 161.0 model1.data[0, 3, y, x] = 181.0 model1.data[0, 4, y, x] = 202.0 model1.data[0, 5, y, x] = 70.0 model1.data[0, 6, y, x] = 100.0 model1.data[0, 7, y, x] = 131.0 model1.data[0, 8, y, x] = 160.0 model1.data[0, 9, y, x] = 190.0 out_model = detect_jumps(model1, gain, rnModel, 4.0, 'half', 200, 4, True) # 1st CR (centered at x=2, y=3) assert (4 == out_model.groupdq[0, 5, 2, 2]) assert (4 == out_model.groupdq[0, 5, 3, 1]) assert (4 == out_model.groupdq[0, 5, 3, 2]) assert (4 == out_model.groupdq[0, 5, 3, 3]) assert (4 == out_model.groupdq[0, 5, 4, 2]) # 2nd CR (centered at x=2, y=2) assert (4 == out_model.groupdq[0, 5, 1, 2]) assert (4 == out_model.groupdq[0, 5, 2, 1]) assert (4 == out_model.groupdq[0, 5, 2, 3]) # 3rd CR (centered at x=3, y=2) assert (4 == out_model.groupdq[0, 5, 1, 3]) assert (4 == out_model.groupdq[0, 5, 2, 4])
def test_crs_on_edge_with_neighbor_flagging(setup_inputs): """" A test to make sure that the neighbors of CRs on the edges of the array are flagged correctly. """ grouptime = 3.0 ingain = 200 inreadnoise = np.float64(7) ngroups = 10 model1, rnModel, gain = setup_inputs(ngroups=ngroups, nrows=20, ncols=20, gain=ingain, readnoise=inreadnoise, deltatime=grouptime) # two segments perfect fit, second segment has twice the slope # CR on 1st row model1.data[0, 0, 0, 15] = 15.0 model1.data[0, 1, 0, 15] = 20.0 model1.data[0, 2, 0, 15] = 25.0 model1.data[0, 3, 0, 15] = 30.0 model1.data[0, 4, 0, 15] = 35.0 model1.data[0, 5, 0, 15] = 140.0 model1.data[0, 6, 0, 15] = 150.0 model1.data[0, 7, 0, 15] = 160.0 model1.data[0, 8, 0, 15] = 170.0 model1.data[0, 9, 0, 15] = 180.0 # CR on last row model1.data[0, 0, 19, 5] = 15.0 model1.data[0, 1, 19, 5] = 20.0 model1.data[0, 2, 19, 5] = 25.0 model1.data[0, 3, 19, 5] = 30.0 model1.data[0, 4, 19, 5] = 35.0 model1.data[0, 5, 19, 5] = 140.0 model1.data[0, 6, 19, 5] = 150.0 model1.data[0, 7, 19, 5] = 160.0 model1.data[0, 8, 19, 5] = 170.0 model1.data[0, 9, 19, 5] = 180.0 # CR on 1st column model1.data[0, 0, 5, 0] = 15.0 model1.data[0, 1, 5, 0] = 20.0 model1.data[0, 2, 5, 0] = 25.0 model1.data[0, 3, 5, 0] = 30.0 model1.data[0, 4, 5, 0] = 35.0 model1.data[0, 5, 5, 0] = 140.0 model1.data[0, 6, 5, 0] = 150.0 model1.data[0, 7, 5, 0] = 160.0 model1.data[0, 8, 5, 0] = 170.0 model1.data[0, 9, 5, 0] = 180.0 # CR on last column model1.data[0, 0, 15, 19] = 15.0 model1.data[0, 1, 15, 19] = 20.0 model1.data[0, 2, 15, 19] = 25.0 model1.data[0, 3, 15, 19] = 30.0 model1.data[0, 4, 15, 19] = 35.0 model1.data[0, 5, 15, 19] = 140.0 model1.data[0, 6, 15, 19] = 150.0 model1.data[0, 7, 15, 19] = 160.0 model1.data[0, 8, 15, 19] = 170.0 model1.data[0, 9, 15, 19] = 180.0 out_model = detect_jumps(model1, gain, rnModel, 4.0, 1, 200, 10, True) # flag CR and three neighbors of first row CR assert (4 == out_model.groupdq[0, 5, 0, 15]) assert (4 == out_model.groupdq[0, 5, 1, 15]) assert (4 == out_model.groupdq[0, 5, 0, 14]) assert (4 == out_model.groupdq[0, 5, 0, 16]) assert (out_model.groupdq[0, 5, -1, 15] == 0) # The one not to flag # flag CR and three neighbors of last row CR assert (4 == out_model.groupdq[0, 5, 19, 5]) assert (4 == out_model.groupdq[0, 5, 18, 5]) assert (4 == out_model.groupdq[0, 5, 19, 4]) assert (4 == out_model.groupdq[0, 5, 19, 6]) # flag CR and three neighbors of first column CR assert (4 == out_model.groupdq[0, 5, 5, 0]) assert (4 == out_model.groupdq[0, 5, 6, 0]) assert (4 == out_model.groupdq[0, 5, 4, 0]) assert (4 == out_model.groupdq[0, 5, 5, 1]) assert (out_model.groupdq[0, 5, 5, -1] == 0) # The one not to flag # flag CR and three neighbors of last column CR assert (4 == out_model.groupdq[0, 5, 15, 19]) assert (4 == out_model.groupdq[0, 5, 15, 18]) assert (4 == out_model.groupdq[0, 5, 16, 19]) assert (4 == out_model.groupdq[0, 5, 14, 19])
def test_nocrs_noflux(setup_inputs): # all pixel values are zero. So slope should be zero model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=5) out_model = detect_jumps(model1, gain, rnModel, 4.0, False, 4.0) assert (0 == np.max(out_model.groupdq))
def test_nirspec_saturated_pix(setup_inputs): """ This test is based on an actual NIRSpec exposure that has some pixels flagged as saturated in one or more groups, which the jump step is supposed to ignore, but an old version of the code was setting JUMP flags for some of the saturated groups. This is to verify that the saturated groups are no longer flagged with jumps. """ grouptime = 3.0 ingain = 1.0 inreadnoise = 10.7 ngroups = 7 nrows = 6 ncols = 6 model1, rnModel, gain = setup_inputs(ngroups=ngroups, nints=1, nrows=nrows, ncols=ncols, gain=ingain, readnoise=inreadnoise, deltatime=grouptime) # Setup the needed input pixel and DQ values model1.data[0, :, 1, 1] = [ 639854.75, 4872.451, -17861.791, 14022.15, 22320.176, 1116.3828, 1936.9746 ] model1.groupdq[0, :, 1, 1] = [0, 0, 0, 0, 0, 2, 2] model1.data[0, :, 2, 2] = [ 8.25666812e+05, -1.10471914e+05, 1.95755371e+02, 1.83118457e+03, 1.72250879e+03, 1.81733496e+03, 1.65188281e+03 ] model1.groupdq[0, :, 2, 2] = [0, 0, 2, 2, 2, 2, 2] model1.data[0, :, 3, 3] = [ 1228767., 46392.234, -3245.6553, 7762.413, 37190.76, 266611.62, 5072.4434 ] model1.groupdq[0, :, 3, 3] = [0, 0, 0, 0, 0, 0, 2] model1.data[0, :, 4, 4] = [ 7.5306038e+05, 1.8269953e+04, 1.8352356e+02, 2.1245061e+03, 2.0628525e+03, 2.1039399e+03, 2.0069873e+03 ] model1.groupdq[0, :, 4, 4] = [0, 0, 2, 2, 2, 2, 2] # run jump detection out_model = detect_jumps(model1, gain, rnModel, rejection_threshold=200.0, max_cores=None, max_jump_to_flag_neighbors=200, min_jump_to_flag_neighbors=10, flag_4_neighbors=True) # Check the results. There should not be any pixels with DQ values of 6, which # is saturated (2) plus jump (4). All the DQ's should be either just 2 or just 4. assert_array_equal(out_model.groupdq[0, :, 1, 1], [0, 4, 4, 4, 0, 2, 2]) assert_array_equal(out_model.groupdq[0, :, 2, 2], [0, 4, 2, 2, 2, 2, 2]) assert_array_equal(out_model.groupdq[0, :, 3, 3], [0, 4, 4, 0, 0, 4, 2]) assert_array_equal(out_model.groupdq[0, :, 4, 4], [0, 4, 2, 2, 2, 2, 2])