def test_pix_8():
"""
CASE H: the segment has a good 1st group and a bad 2nd group.
CASE B: segment has >2 groups, not at end of ramp.
SCI segs are: seg0[15](H, ignored), seg1[25, 35, 54, 55, 65, 75](B)
"""
ngroups, nints, nrows, ncols, deltatime, gain, readnoise = set_scalars()
RampMod, RnMod, GMod, pixdq, groupdq, err = create_mod_arrays( ngroups,
nints, nrows, ncols, deltatime, gain, readnoise )
# Populate pixel-specific SCI and GROUPDQ arrays
RampMod.data[0,:,0,0] = np.array([ 15., 25., 35., 54., 55., 65., 75.,
94., 95., 105.], dtype=np.float32)
RampMod.groupdq[0,:,0,0] = np.array([0, 4, 0, 0, 0, 0, 0, 2, 2, 2])
# Set truth values for PRIMARY results:
p_true =[ 1.0101178, 6, 0.1848883, 0.00363636, 0.03054732 ]
# Set truth values for OPTIONAL results:
o_true = [ 1.0101178, 12.385354, 0.00363636, 0.03054732, 16.508228,
40.81897, 4.898822, 855.78046 ]
new_mod, int_mod, opt_mod, gls_opt_mod = ramp_fit( RampMod, 1024*300., True,
RnMod, GMod, 'OLS', 'optimal' )
assert_pri( p_true, new_mod )
assert_opt( o_true, opt_mod )
def test_pix_11():
"""
CASE F: segment has 2 good groups not at array end.
SCI seg is: seg0[15,25](F)
"""
ngroups, nints, nrows, ncols, deltatime, gain, readnoise = set_scalars()
RampMod, RnMod, GMod, pixdq, groupdq, err = create_mod_arrays( ngroups,
nints, nrows, ncols, deltatime, gain, readnoise )
# Populate pixel-specific SCI and GROUPDQ arrays
RampMod.data[0,:,0,0] = np.array([ 15., 25., 35., 54., 55., 65., 75.,
94., 95., 105.], dtype=np.float32)
RampMod.groupdq[0,:,0,0] = np.array([0, 0, 2, 2, 2, 2, 2, 2, 2, 2])
# Set truth values for PRIMARY results:
p_true = [ 1., 2, 1.042755, 0.01818182, 1.0691562 ]
# Set truth values for OPTIONAL results:
o_true = [1., 56.870003, 0.01818182, 1.0691562, 15., 56.870003, 5.,
0.84580624 ]
new_mod, int_mod, opt_mod, gls_opt_mod = ramp_fit( RampMod, 1024*300., True,
RnMod, GMod, 'OLS', 'optimal' )
assert_pri( p_true, new_mod )
assert_opt( o_true, opt_mod )
def test_pix_1():
"""
CASE H: the segment has a good 1st group and a bad 2nd group, so is a
single group. If there are no later and longer segments in the ramp,
this group's data will be used in the 'fit'. If there are later and
longer segments, this group's data will not be used.
CASE F: segment has 2 good groups not at array end.
SCI segs are: seg0[15] (H, ignored), seg1[35, 54] (F)
"""
ngroups, nints, nrows, ncols, deltatime, gain, readnoise = set_scalars()
RampMod, RnMod, GMod, pixdq, groupdq, err = create_mod_arrays( ngroups,
nints, nrows, ncols, deltatime, gain, readnoise )
# Populate pixel-specific SCI and GROUPDQ arrays
RampMod.data[0,:,0,0] = np.array([ 15., 25., 35., 54., 55., 65., 75.,
94., 95., 105.], dtype=np.float32)
RampMod.groupdq[0,:,0,0] = np.array([0, 4, 4, 0, 2, 2, 2, 2, 2, 2])
# Set truth values for PRIMARY results:
p_true = [1.8999999, 6, 1.046670, 0.02636364, 1.0691562 ]
# Set truth values for OPTIONAL results:
o_true = [ 1.9, 56.870003, 0.02636364, 1.0691562, -3., 56.870003,
-3.999998, 0.83321977]
new_mod, int_mod, opt_mod, gls_opt_mod = ramp_fit( RampMod, 1024*300., True,
RnMod, GMod, 'OLS', 'optimal' )
assert_pri( p_true, new_mod )
assert_opt( o_true, opt_mod )
def test_pix_2():
"""
CASE B: segment has >2 groups, not at end of ramp.
CASE F: (twice) segment has 2 good groups not at array end.
SCI segs are: seg0[15,25,35](B), seg1[54,55](F), seg2[65,75](F)
"""
ngroups, nints, nrows, ncols, deltatime, gain, readnoise = set_scalars()
RampMod, RnMod, GMod, pixdq, groupdq, err = create_mod_arrays( ngroups,
nints, nrows, ncols, deltatime, gain, readnoise )
# Populate pixel-specific SCI and GROUPDQ arrays
RampMod.data[0,:,0,0] = np.array([ 15., 25., 35., 54., 55., 65., 75., 94.,
95., 105.], dtype=np.float32)
RampMod.groupdq[0,:,0,0] = np.array([0, 0, 0, 4, 0, 4, 0, 4, 2, 2])
# Set truth values for PRIMARY results:
p_true = [0.84761256, 6, 0.42986465, 0.00659091, 0.1781927 ]
# Set truth values for OPTIONAL results for all segments
o_true = [[ 1.0000001, 0.1, 1. ], # slopes for 3 segments
[ 28.435, 56.870003, 56.870003], # sigslope
[ 0.01318182, 0.02636364, 0.02636364,], # var_poisson
[ 0.26728904, 1.0691562, 1.0691562 ], # var_rnoise
[ 14.999998, 51., 15. ], # yint
[ 36.709427, 56.870003, 56.870003], # sigyint
[ 6.5238733 ], # pedestal
[ 12.712313, 0.83321977, 0.83321977], # weights
]
new_mod, int_mod, opt_mod, gls_opt_mod = ramp_fit( RampMod, 1024*300., True,
RnMod, GMod, 'OLS', 'optimal' )
assert_pri( p_true, new_mod )
assert_opt( o_true, opt_mod )
def test_pix_7():
"""
CASE B: segment has >2 groups, not at end of ramp.
SCI seg is seg0[15,25,35,54,55,65,75,94](B)
"""
ngroups, nints, nrows, ncols, deltatime, gain, readnoise = set_scalars()
RampMod, RnMod, GMod, pixdq, groupdq, err = create_mod_arrays( ngroups,
nints, nrows, ncols, deltatime, gain, readnoise )
# Populate pixel-specific SCI and GROUPDQ arrays
RampMod.data[0,:,0,0] = np.array([ 15., 25., 35., 54., 55., 65., 75.,
94., 195., 205.], dtype=np.float32)
RampMod.groupdq[0,:,0,0] = np.array([0, 0, 0, 0, 0, 0, 0, 0, 4, 4])
# Set truth values for PRIMARY results:
p_true = [ 1.0757396, 4, 0.12379601, 0.0025974, 0.01272805 ]
# Set truth values for OPTIONAL results:
o_true = [ 1.0757396, 6.450687, 0.0025974, 0.01272805, 14.504951,
27.842508, 4.2426033, 4257.684 ]
new_mod, int_mod, opt_mod, gls_opt_mod = ramp_fit( RampMod, 1024*300., True,
RnMod, GMod, 'OLS', 'optimal' )
assert_pri( p_true, new_mod )
assert_opt( o_true, opt_mod )
def test_pix_0():
"""
CASE A: segment has >2 groups, at end of ramp.
SCI seg is [15., 25., 35., 54., 55., 65., 75., 94., 95., 105.](A)
"""
ngroups, nints, nrows, ncols, deltatime, gain, readnoise = set_scalars()
RampMod, RnMod, GMod, pixdq, groupdq, err = create_mod_arrays( ngroups,
nints, nrows, ncols, deltatime, gain, readnoise )
# Populate pixel-specific SCI and GROUPDQ arrays
RampMod.data[0,:,0,0] = np.array([ 15., 25., 35., 54., 55.,
65., 75., 94., 95., 105.], dtype=np.float32)
RampMod.groupdq[0,:,0,0] = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
# Set truth values for PRIMARY results:
# [data, dq, err, var_p, var_r]
p_true = [1.0117551, 0, 0.0921951, 0.0020202, 0.00647973 ]
# Set truth values for OPTIONAL results:
# [slope, sigslope, var_poisson, var_rnoise, yint, sigyint, ped, weights]
o_true = [1.0117551, 4.874572, 0.0020202, 0.00647973,
15.911023, 27.789335, 4.882449, 13841.038 ]
new_mod, int_mod, opt_mod, gls_opt_mod = ramp_fit( RampMod, 1024*300., True,
RnMod, GMod, 'OLS', 'optimal' )
assert_pri( p_true, new_mod )
assert_opt( o_true, opt_mod )
def test_twenty_groups_two_segments():
''' Test to verify weighting of multiple segments in combination:
a) gdq all 0 ; b) 1 CR (2 segments) c) 1 CR then SAT (2 segments)
'''
(ngroups, nints, nrows, ncols, deltatime) = (20, 1, 1, 3, 6.)
model1, gdq, rnModel, pixdq, err, gain = setup_small_cube( ngroups,
nints, nrows, ncols, deltatime)
# a) ramp having gdq all 0
model1.data[0,:,0,0] = np.arange(ngroups)*10. + 30.
# b) ramp having 1 CR at group 15; 2 segments
model1.data[0,:,0,1] = np.arange(ngroups)*10. + 50.
gdq[0,15,0,1] = dqflags.group['JUMP_DET']
model1.data[0,15:,0,1] += 1000.
# c) ramp having 1 CR at group 2; SAT starting in group 15
model1.data[0,:,0,2] = np.arange(ngroups)*10. + 70.
gdq[0,2,0,2] = dqflags.group['JUMP_DET']
model1.data[0,2:,0,2] += 2000.
gdq[0,15:,0,2] = dqflags.group['SATURATED']
model1.data[0,15:,0,2] = 25000.
new_mod, int_model, opt_model, gls_opt_model = ramp_fit(model1, 1024*30000.,
True, rnModel, gain, 'OLS', 'optimal')
# Check some PRI & OPT output arrays
np.testing.assert_allclose( new_mod.data, 10./deltatime, rtol=1E-4 )
wh_data = opt_model.slope != 0. # only test existing segments
np.testing.assert_allclose(opt_model.slope[wh_data], 10./deltatime, rtol=1E-4)
np.testing.assert_allclose(opt_model.yint[0,0,0,:], model1.data[0,0,0,:], rtol=1E-5)
np.testing.assert_allclose(opt_model.pedestal[0,0,:],
model1.data[0,0,0,:]-10., rtol=1E-5)
def test_error_when_frame_time_not_set():
#all pixel values are zero. So slope should be zero
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=5)
model1.meta.exposure.frame_time = None
slopes = ramp_fit(model1, 64000, False, rnModel, gain, 'OLS', 'optimal')
assert(0 == np.max(slopes[0].data))
assert(0 == np.min(slopes[0].data))
def test_pix_5():
"""
CASE B: segment has >2 groups, not at end of ramp.
CASE A: segment has >2 groups, at end of ramp.
SCI segs are: seg0[15, 25, 35, 54](B), seg1[ 2055, 2065, 2075, 2094, 2095,
2105](A)
"""
ngroups, nints, nrows, ncols, deltatime, gain, readnoise = set_scalars()
RampMod, RnMod, GMod, pixdq, groupdq, err = create_mod_arrays( ngroups,
nints, nrows, ncols, deltatime, gain, readnoise )
# Populate pixel-specific SCI and GROUPDQ arrays
RampMod.data[0,:,0,0] = np.array([ 15., 25., 35., 54., 2055., 2065., 2075.,
2094., 2095., 2105.], dtype=np.float32)
RampMod.groupdq[0,:,0,0] = np.array([0, 0, 0, 0, 4, 0, 0, 0, 0, 0])
# Set truth values for PRIMARY results:
p_true = [ 1.0760875, 4, 0.16134359, 0.00227273, 0.02375903 ]
# Set truth values for OPTIONAL results:
o_true = [[ 1.2799551, 1.0144024 ],
[ 18.312422, 9.920552 ],
[ 0.00606061, 0.00363636 ],
[ 0.10691562, 0.03054732 ],
[ 13.537246, 2015.0737 ],
[ 35.301933, 67.10882 ],
[ 4.2391253],
[ 78.34764, 855.78046 ]
]
new_mod, int_mod, opt_mod, gls_opt_mod = ramp_fit( RampMod, 1024*300., True,
RnMod, GMod, 'OLS', 'optimal' )
assert_pri( p_true, new_mod )
assert_opt( o_true, opt_mod )
def test_five_groups_two_ints_Poisson_noise_only():
grouptime=3.0
deltaDN = 5
ingain = 2000
inreadnoise =7
ngroups=5
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=ngroups,
gain=ingain, readnoise=inreadnoise,deltatime=grouptime, nints=2)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 15.0
model1.data[0, 2, 500, 500] = 25.0
model1.data[0, 3, 500, 500] = 33.0
model1.data[0, 4, 500, 500] = 60.0
model1.data[1, 0, 500, 500] = 10.0
model1.data[1, 1, 500, 500] = 15.0
model1.data[1, 2, 500, 500] = 25.0
model1.data[1, 3, 500, 500] = 33.0
model1.data[1, 4, 500, 500] = 160.0
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
out_slope=slopes[0].data[500, 500]
median_slope=np.median(np.diff(model1.data[:,:,500,500]))/grouptime
deltaDN1 = 50
deltaDN2 = 150
delta_time = (ngroups - 1) * grouptime
delta_electrons = median_slope * ingain *delta_time
single_sample_readnoise = np.float64(inreadnoise/np.sqrt(2))
np.testing.assert_allclose(out_slope, (deltaDN1 + deltaDN2)/2.0, 75.0, 1e-6)
def test_oneCR_10_groups_combination():
grouptime=3.0
deltaDN = 5
ingain = 200 # use large gain to show that Poisson noise doesn't affect the recombination
inreadnoise = np.float64(7)
ngroups=10
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=ngroups,
gain=ingain, readnoise=inreadnoise,deltatime=grouptime)
# two segments perfect fit, second segment has twice the slope
model1.data[0, 0, 500, 500] = 15.0
model1.data[0, 1, 500, 500] = 20.0
model1.data[0, 2, 500, 500] = 25.0
model1.data[0, 3, 500, 500] = 30.0
model1.data[0, 4, 500, 500] = 35.0
model1.data[0, 5, 500, 500] = 140.0
model1.data[0, 6, 500, 500] = 150.0
model1.data[0, 7, 500, 500] = 160.0
model1.data[0, 8, 500, 500] = 170.0
model1.data[0, 9, 500, 500] = 180.0
model1.groupdq[0,5,500,500]=dqflags.group['JUMP_DET']
slopes, int_model, opt_model, gls_opt_model= ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
segment_groups = 5
single_sample_readnoise = np.float64( inreadnoise/np.sqrt(2))
#check that the segment variance is as expected
np.testing.assert_allclose(opt_model.var_rnoise[0,0,500,500],(12.0 * single_sample_readnoise**2/(segment_groups * (segment_groups**2 - 1) * grouptime**2)), rtol=1e-6)
# check the combined slope is the average of the two segments since they have the same number of groups
np.testing.assert_allclose(slopes.data[500, 500], 2.5,rtol=1e-5)
#check that the slopes of the two segments are correct
np.testing.assert_allclose(opt_model.slope[0,0,500, 500], 5/3.0,rtol=1e-5)
np.testing.assert_allclose(opt_model.slope[0,1,500, 500], 10/3.0,rtol=1e-5)
def test_ngroups_doesnot_match_cube_size():
#all pixel values are zero. So slope should be zero
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=5)
model1.meta.exposure.ngroups = 11
slopes = ramp_fit(model1, 64000, False, rnModel, gain, 'OLS', 'optimal')
assert(0 == np.max(slopes[0].data))
assert(0 == np.min(slopes[0].data))
def test_miri_first_last():
''' This is a test of whether ramp fitting correctly handles having all 0th
group dq flagged as DO_NOT_USE, and all final group dq flagged as
DO_NOT_USE for MIRI data. For 1 pixel ([1,1]) the 1st (again, 0-based)
group is flagged as a CR. For such a ramp, the code removes the CR-flag
from such a CR-affected 1st group; so if it initially was 4 it is reset
to 0 ("good"), in which case it's as if that CR was not even there.
'''
(ngroups, nints, nrows, ncols, deltatime) = (10, 1, 2, 2, 3.)
model1, gdq, rnModel, pixdq, err, gain = setup_small_cube(ngroups,
nints, nrows, ncols, deltatime)
# Make smooth ramps having outlier SCI values in the 0th and final groups
# to reveal if they are included in the fit (they shouldn't be, as those
# groups are flagged as DO_NOT_USE)
model1.data[0,:,0,0] = np.array([-200., 30., 40., 50., 60., 70., 80.,
90., 100., -500.], dtype=np.float32)
model1.data[0,:,0,1] = np.array([-300., 80., 90., 100., 110., 120.,
130., 140., 150., -600.], dtype=np.float32)
model1.data[0,:,1,0] = np.array([-200., 40., 50., 60., 70., 80., 90.,
100., 110., 900.], dtype=np.float32)
model1.data[0,:,1,1] = np.array([-600., 140., 150., 160., 170., 180.,
190., 200., 210., -400.], dtype=np.float32)
# For all pixels, set gdq for 0th and final groups to DO_NOT_USE
model1.groupdq[:,0,:,:] = dqflags.group['DO_NOT_USE']
model1.groupdq[:,-1,:,:] = dqflags.group['DO_NOT_USE']
# Put CR in 1st (0-based) group
model1.groupdq[0,1,1,1] = dqflags.group['JUMP_DET']
new_mod, int_model, opt_model, gls_opt_model = ramp_fit(model1, 1024*30000.,
True, rnModel, gain, 'OLS', 'optimal')
np.testing.assert_allclose( new_mod.data, 10./3., rtol=1E-5 )
def test_one_group_two_ints_fit():
#
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=1,gain=1,readnoise=10,nints=2)
model1.data[0, 0, 500, 500] = 10.0
model1.data[1, 0, 500, 500] = 12.0
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
np.testing.assert_allclose(slopes[0].data[500, 500],11.0, 1e-6)
def test_pix_4():
"""
CASE G: segment is the good 1st group of the entire ramp, and no later
groups are good.
SCI seg is seg0[15](G)
"""
ngroups, nints, nrows, ncols, deltatime, gain, readnoise = set_scalars()
RampMod, RnMod, GMod, pixdq, groupdq, err = create_mod_arrays( ngroups,
nints, nrows, ncols, deltatime, gain, readnoise )
# Populate pixel-specific SCI and GROUPDQ arrays
RampMod.data[0,:,0,0] = np.array([ 15., 25., 35., 54., 1055., 1065., 1075.,
2594., 2595., 2605.], dtype=np.float32)
RampMod.groupdq[0,:,0,0] = np.array([0, 2, 2, 2, 2, 2, 2, 2, 2, 2])
# Set truth values for PRIMARY results:
p_true = [1.5, 2, 1.047105, 0.02727273, 1.0691562]
# Set truth values for OPTIONAL results:
o_true = [ 1.5, 0., 0.02727273, 1.0691562, 0., 0., 0., 0.8318386]
new_mod, int_mod, opt_mod, gls_opt_mod = ramp_fit( RampMod, 1024*300., True,
RnMod, GMod, 'OLS', 'optimal' )
assert_pri( p_true, new_mod )
assert_opt( o_true, opt_mod )
def test_pix_3():
"""
CASE B: segment has >2 groups, not at end of ramp.
CASE E: segment has 2 good groups, at end of ramp.
SCI segs are: seg0[15,25,35,54,55,65,75,94](B), seg1[95,105](E)
"""
ngroups, nints, nrows, ncols, deltatime, gain, readnoise = set_scalars()
RampMod, RnMod, GMod, pixdq, groupdq, err = create_mod_arrays( ngroups,
nints, nrows, ncols, deltatime, gain, readnoise )
# Populate pixel-specific SCI and GROUPDQ arrays
RampMod.data[0,:,0,0] = np.array([ 15., 25., 35., 54., 55., 65., 75.,
94., 95., 105.], dtype=np.float32)
RampMod.groupdq[0,:,0,0] = np.array([0, 0, 0, 0, 0, 0, 0, 0, 4, 0])
# Set truth values for PRIMARY results:
p_true = [1.0746869, 4, 0.12186482, 0.00227273, 0.01257831 ]
# Set truth values for OPTIONAL results:
o_true = [[ 1.0757396, 1.],
[ 6.450687, 56.870003],
[ 0.0025974, 0.01818182],
[ 0.01272805, 1.0691562 ],
[ 14.504951, 15.],
[ 27.842508, 56.870003],
[ 4.25313],
[ 4.2576841e+03, 8.458062e-01],
]
new_mod, int_mod, opt_mod, gls_opt_mod = ramp_fit( RampMod, 1024*300., True,
RnMod, GMod, 'OLS', 'optimal' )
assert_pri( p_true, new_mod )
assert_opt( o_true, opt_mod )
def test_miri_all_sat():
''' Test of all groups in all integrations being saturated; all output arrays
(particularly variances) should be 0.
'''
(ngroups, nints, nrows, ncols, deltatime) = (3, 2, 2, 2, 6.)
model1, gdq, rnModel, pixdq, err, gain = setup_small_cube(ngroups,
nints, nrows, ncols, deltatime)
model1.groupdq[:, :, :, :] = dqflags.group['SATURATED']
new_mod, int_model, opt_model, gls_opt_model = ramp_fit(model1, 1024*30000.,
True, rnModel, gain, 'OLS', 'optimal')
# Check PRI output arrays
np.testing.assert_allclose( new_mod.data, 0.0, atol=1E-6 )
np.testing.assert_allclose( new_mod.err, 0.0, atol=1E-6 )
np.testing.assert_allclose( new_mod.var_poisson, 0.0, atol=1E-6 )
np.testing.assert_allclose( new_mod.var_rnoise, 0.0, atol=1E-6 )
# Check INT output arrays
np.testing.assert_allclose( int_model.data, 0.0, atol=1E-6 )
np.testing.assert_allclose( int_model.err, 0.0, atol=1E-6 )
np.testing.assert_allclose( int_model.var_poisson, 0.0, atol=1E-6 )
np.testing.assert_allclose( int_model.var_rnoise, 0.0, atol=1E-6 )
# Check OPT output arrays
np.testing.assert_allclose( opt_model.slope, 0.0, atol=1E-6 )
np.testing.assert_allclose( opt_model.var_poisson, 0.0, atol=1E-6 )
np.testing.assert_allclose( opt_model.var_rnoise, 0.0, atol=1E-6 )
np.testing.assert_allclose( opt_model.sigslope, 0.0, atol=1E-6 )
np.testing.assert_allclose( opt_model.yint, 0.0, atol=1E-6 )
np.testing.assert_allclose( opt_model.sigyint, 0.0, atol=1E-6 )
np.testing.assert_allclose( opt_model.pedestal, 0.0, atol=1E-6 )
np.testing.assert_allclose( opt_model.weights, 0.0, atol=1E-6 )
def test_pix_6():
"""
CASE F: segment has 2 good groups not at array end
CASE A: segment has >2 groups, at end of ramp.
SCI segs are: seg0[15,25](F), seg1[54, 55, 65, 375, 394, 395, 405](A)
"""
ngroups, nints, nrows, ncols, deltatime, gain, readnoise = set_scalars()
RampMod, RnMod, GMod, pixdq, groupdq, err = create_mod_arrays( ngroups,
nints, nrows, ncols, deltatime, gain, readnoise )
# Populate pixel-specific SCI and GROUPDQ arrays
RampMod.data[0,:,0,0] = np.array([ 15., 25., 35., 54., 55., 65., 375.,
394., 395., 405.], dtype=np.float32)
RampMod.groupdq[0,:,0,0] = np.array([0, 0, 4, 4, 0, 0, 0, 0, 0, 0])
# Set truth values for PRIMARY results:
p_true = [ 6.092052, 4, 0.14613187, 0.0025974, 0.01875712 ]
# Set truth values for OPTIONAL results:
o_true = [[ 1., 6.195652 ],
[ 56.870003, 8.8936615 ],
[ 0.01818182, 0.0030303 ],
[ 1.0691562, 0.01909207 ],
[ 15.,-143.2391 ],
[ 56.870003, 58.76999 ],
[ -45.92052 ],
[ 8.4580624e-01, 2.0433204e+03 ]
]
new_mod, int_mod, opt_mod, gls_opt_mod = ramp_fit( RampMod, 1024*300., True,
RnMod, GMod, 'OLS', 'optimal' )
assert_pri( p_true, new_mod )
assert_opt( o_true, opt_mod )
def test_photon_noise_only_bad_last_frame_two_groups():
#
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=2,gain=1000,readnoise=1)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 15.0
model1.groupdq[0,1,:,:] = dqflags.group['DO_NOT_USE']
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
cds_slope = (model1.data[0,1,500,500] - model1.data[0,0,500,500])/ 1.0
np.testing.assert_allclose(slopes[0].data[500, 500], cds_slope, 1e-6)
def test_four_groups_CR_causes_orphan_1st_group():
#
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=4,gain=.01,readnoise=10000)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 125.0
model1.data[0, 2, 500, 500] = 145.0
model1.data[0, 3, 500, 500] = 165.0
model1.groupdq[0,1,500,500]=dqflags.group['JUMP_DET']
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
expected_slope=20.0
np.testing.assert_allclose(slopes[0].data[500, 500],expected_slope, 1e-6)
def test_photon_noise_only_fit():
#
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=5,gain=1000,readnoise=1)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 15.0
model1.data[0, 2, 500, 500] = 25.0
model1.data[0, 3, 500, 500] = 33.0
model1.data[0, 4, 500, 500] = 60.0
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
cds_slope = (model1.data[0,4,500,500] - model1.data[0,0,500,500])/ 4.0
np.testing.assert_allclose(slopes[0].data[500, 500], cds_slope, 1e-2)
def test_one_group_two_ints_fit():
#
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=1,
gain=1,
readnoise=10,
nints=2)
model1.data[0, 0, 500, 500] = 10.0
model1.data[1, 0, 500, 500] = 12.0
slopes = ramp_fit(model1, 1024 * 30000., True, rnModel, gain, 'OLS',
'optimal')
np.testing.assert_allclose(slopes[0].data[500, 500], 11.0, 1e-6)
def test_four_groups_oneCR_orphangroupatend_fit(self, method):
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=4,gain=1,readnoise=10)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 15.0
model1.data[0, 2, 500, 500] = 20.0
model1.data[0, 3, 500, 500] = 145.0
model1.groupdq[0,3,500,500]=dqflags.group['JUMP_DET']
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
cds_slope = (model1.data[0,1,500,500] - model1.data[0,0,500,500])
np.testing.assert_allclose(slopes[0].data[500, 500], cds_slope, 1e-6)
def test_four_groups_CR_causes_orphan_1st_group(self, method):
#
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=4,gain=.01,readnoise=10000)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 125.0
model1.data[0, 2, 500, 500] = 145.0
model1.data[0, 3, 500, 500] = 165.0
model1.groupdq[0,1,500,500]=dqflags.group['JUMP_DET']
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
expected_slope=20.0
np.testing.assert_allclose(slopes[0].data[500, 500],expected_slope, 1e-6)
def test_photon_noise_only_bad_last_frame(self, method):
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=5, gain=1000, readnoise=1)
model1.data[0, 0, 50, 50] = 10.0
model1.data[0, 1, 50, 50] = 15.0
model1.data[0, 2, 50, 50] = 25.0
model1.data[0, 3, 50, 50] = 33.0
model1.data[0, 4, 50, 50] = 60.0
model1.groupdq[0, 4, :, :] = DO_NOT_USE
slopes = ramp_fit(model1, 1024 * 30000., True, rnModel, gain, 'OLS', 'optimal', 'none')
cds_slope = (model1.data[0, 3, 50, 50] - model1.data[0, 0, 50, 50]) / 3.0
np.testing.assert_allclose(slopes[0].data[50, 50], cds_slope, 1e-2)
def test_four_groups_two_CRs_at_end(self, method):
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=4, gain=1, readnoise=10)
model1.data[0, 0, 50, 50] = 10.0
model1.data[0, 1, 50, 50] = 15.0
model1.data[0, 2, 50, 50] = 25.0
model1.data[0, 3, 50, 50] = 145.0
model1.groupdq[0, 2, 50, 50] = JUMP_DET
model1.groupdq[0, 3, 50, 50] = JUMP_DET
slopes = ramp_fit(model1, 1024 * 30000., True, rnModel, gain, 'OLS', 'optimal', 'none')
cds_slope = (model1.data[0, 1, 50, 50] - model1.data[0, 0, 50, 50])
np.testing.assert_allclose(slopes[0].data[50, 50], cds_slope, 1e-6)
def test_bad_gain_values():
#all pixel values are zero. So slope should be zero
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=5)
model1.meta.exposure.ngroups = 11
gain.data[100,100] = -10
gain.data[200,200] = np.nan
slopes = ramp_fit(model1, 64000, False, rnModel, gain, 'OLS', 'optimal')
assert(0 == np.max(slopes[0].data))
assert(0 == np.min(slopes[0].data))
assert slopes[0].dq[100, 100] == 524288 + 1
assert slopes[0].dq[200, 200] == 524288 + 1
def test_bad_gain_values(self, method):
# all pixel values are zero. So slope should be zero
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=5)
model1.meta.exposure.ngroups = 11
gain.data[10, 10] = -10
gain.data[20, 20] = np.nan
slopes = ramp_fit(model1, 64000, False, rnModel, gain, 'OLS', 'optimal', 'none')
assert(0 == np.max(slopes[0].data))
assert(0 == np.min(slopes[0].data))
assert slopes[0].dq[10, 10] == 524288 + 1
assert slopes[0].dq[20, 20] == 524288 + 1
def test_read_noise_only_fit(self, method):
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=5, readnoise=50)
model1.data[0, 0, 50, 50] = 10.0
model1.data[0, 1, 50, 50] = 15.0
model1.data[0, 2, 50, 50] = 25.0
model1.data[0, 3, 50, 50] = 33.0
model1.data[0, 4, 50, 50] = 60.0
slopes = ramp_fit(model1, 1024 * 30000., True, rnModel, gain, 'OLS', 'optimal', 'none')
xvalues = np.arange(5) * 1.0
yvalues = np.array([10, 15, 25, 33, 60])
coeff = np.polyfit(xvalues, yvalues, 1)
np.testing.assert_allclose(slopes[0].data[50, 50], coeff[0], 1e-6)
def test_four_groups_two_CRs_at_end():
#
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=4,gain=1,readnoise=10)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 15.0
model1.data[0, 2, 500, 500] = 25.0
model1.data[0, 3, 500, 500] = 145.0
model1.groupdq[0,2,500,500]=dqflags.group['JUMP_DET']
model1.groupdq[0,3,500,500]=dqflags.group['JUMP_DET']
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
cds_slope = (model1.data[0,1,500,500] - model1.data[0,0,500,500])
np.testing.assert_allclose(slopes[0].data[500, 500], cds_slope, 1e-6)
def test_pix_12():
"""
CASE NGROUPS=2: the segment has a good 1st group and a saturated 2nd group,
so is a single group. Group 1's data will be used in the 'fit'.
"""
ngroups, nints, nrows, ncols, deltatime, gain, readnoise = set_scalars()
ngroups = 2
nints = 1
ncols = 2
ramp_model, rnoise_model, gain_model, pixdq, groupdq, err = create_mod_arrays(
ngroups, nints, nrows, ncols, deltatime, gain, readnoise)
# Populate pixel-specific SCI and GROUPDQ arrays
ramp_model.data[0, :, 0, 0] = np.array([15., 59025.], dtype=np.float32)
ramp_model.groupdq[0, :, 0, 0] = np.array([0, 2])
ramp_model.data[0, :, 0, 1] = np.array([61000., 61000.], dtype=np.float32)
ramp_model.groupdq[0, :, 0, 1] = np.array([2, 2])
# call ramp_fit
new_mod, int_mod, opt_mod, gls_opt_mod = ramp_fit(ramp_model, 1024 * 300.,
True, rnoise_model,
gain_model, 'OLS',
'optimal', 'none')
# Set truth values for PRIMARY results for pixel 1:
# slope, dq, err, var_p, var_r
# slope = group1 / deltatime = 15 / 10 = 1.5
# dq = 2 (saturation) because group2 is saturated, but DO_NOT_USE is *not* set
p_true = [1.5, 2, 1.047105, 0.027273, 1.069156]
# Set truth values for OPTIONAL results:
# slope, sig_slope, var_p, var_r, yint, sig_yint, pedestal, weights
# slope = group1 / deltatime = 15 / 10 = 1.5
# sig_slope, yint, sig_yint, and pedestal are all zero, because only 1 good group
o_true = [1.5, 0., 0.027273, 1.069156, 0., 0., 0., 0.831839]
assert_pri(p_true, new_mod, 0)
assert_opt(o_true, opt_mod, 0)
# Set truth values for PRIMARY results for pixel 2:
# slope, dq, err, var_p, var_r
# slope = zero, because no good data
# dq = 3 (saturation + do_not_use) because both groups are saturated
p_true = [0., 3, 0., 0., 0.]
# Set truth values for OPTIONAL results:
# slope, sig_slope, var_p, var_r, yint, sig_yint, pedestal, weights
# all values zero, because no good data
o_true = [0., 0., 0., 0., 0., 0., 0., 0.]
assert_pri(p_true, new_mod, 1)
assert_opt(o_true, opt_mod, 1)
def test_read_noise_only_fit():
#
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=5,readnoise=50)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 15.0
model1.data[0, 2, 500, 500] = 25.0
model1.data[0, 3, 500, 500] = 33.0
model1.data[0, 4, 500, 500] = 60.0
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
xvalues = np.arange(5)*1.0
yvalues = np.array([10,15,25,33,60])
coeff = np.polyfit(xvalues, yvalues, 1)
np.testing.assert_allclose(slopes[0].data[500, 500], coeff[0], 1e-6)
def test_photon_noise_with_unweighted_fit(self, method):
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=5,gain=1000,readnoise=1)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 15.0
model1.data[0, 2, 500, 500] = 25.0
model1.data[0, 3, 500, 500] = 33.0
model1.data[0, 4, 500, 500] = 60.0
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'unweighted')
# cds_slope = (model1.data[0,4,500,500] - model1.data[0,0,500,500])/ 4.0
xvalues = np.arange(5)*1.0
yvalues = np.array([10,15,25,33,60])
coeff = np.polyfit(xvalues, yvalues, 1)
np.testing.assert_allclose(slopes[0].data[500, 500], coeff[0], 1e-6)
def test_photon_noise_only_bad_last_frame_two_groups(self, method):
#
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=2,
gain=1000,
readnoise=1)
model1.data[0, 0, 50, 50] = 10.0
model1.data[0, 1, 50, 50] = 15.0
model1.groupdq[0, 1, :, :] = dqflags.group['DO_NOT_USE']
slopes = ramp_fit(model1, 1024 * 30000., True, rnModel, gain, 'OLS',
'optimal', 'none')
cds_slope = (model1.data[0, 1, 50, 50] -
model1.data[0, 0, 50, 50]) / 1.0
np.testing.assert_allclose(slopes[0].data[50, 50], cds_slope, 1e-6)
def test_multiprocessing():
nrows = 100
ncols = 100
ngroups = 25
nints = 3
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=ngroups,
gain=1,
readnoise=10,
nints=nints,
nrows=nrows,
ncols=ncols)
delta_plane1 = np.zeros((nrows, ncols), dtype=np.float64)
delta_plane2 = np.zeros((nrows, ncols), dtype=np.float64)
delta_vec = np.asarray([x / 50.0 for x in range(nrows)])
for i in range(ncols):
delta_plane1[i, :] = delta_vec * i
delta_plane2[:, i] = delta_vec * i
model1.data[:, :, :, :] = 0
for j in range(ngroups - 1):
model1.data[
0, j +
1, :, :] = model1.data[0, j, :, :] + delta_plane1 + delta_plane2
model1.data[
1, j +
1, :, :] = model1.data[1, j, :, :] + delta_plane1 + delta_plane2
model1.data[
2, j +
1, :, :] = model1.data[2, j, :, :] + delta_plane1 + delta_plane2
model1.data = np.round(model1.data +
np.random.normal(0, 5, (nints, ngroups, ncols,
nrows)))
slopes, int_model, opt_model, gls_opt_model = ramp_fit(
model1, 1024 * 30000., False, rnModel, gain, 'GLS', 'optimal', 'none')
slopes_multi, int_model_multi, opt_model_multi, gls_opt_model_multi = ramp_fit(
model1, 1024 * 30000., False, rnModel, gain, 'GLS', 'optimal', 'half')
np.testing.assert_allclose(slopes.data, slopes_multi.data, rtol=1e-5)
def test_photon_noise_with_unweighted_fit():
#
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=5,gain=1000,readnoise=1)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 15.0
model1.data[0, 2, 500, 500] = 25.0
model1.data[0, 3, 500, 500] = 33.0
model1.data[0, 4, 500, 500] = 60.0
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'unweighted')
cds_slope = (model1.data[0,4,500,500] - model1.data[0,0,500,500])/ 4.0
xvalues = np.arange(5)*1.0
yvalues = np.array([10,15,25,33,60])
coeff = np.polyfit(xvalues, yvalues, 1)
np.testing.assert_allclose(slopes[0].data[500, 500], coeff[0], 1e-6)
def test_subarray_5groups():
#all pixel values are zero. So slope should be zero
model1, gdq, rnModel, pixdq, err, gain = setup_subarray_inputs(ngroups=5,subxstart=100, subystart=200, subxsize=50, subysize=150, readnoise=50)
model1.meta.exposure.ngroups = 11
model1.data[0, 0, 125, 10] = 10.0
model1.data[0, 1, 125, 10] = 15.0
model1.data[0, 2, 125, 10] = 25.0
model1.data[0, 3, 125, 10] = 33.0
model1.data[0, 4, 125, 10] = 60.0
xvalues = np.arange(5)*1.0
yvalues = np.array([10,15,25,33,60])
coeff = np.polyfit(xvalues, yvalues, 1)
slopes = ramp_fit(model1, 64000, False, rnModel, gain, 'OLS', 'optimal')
np.testing.assert_allclose(slopes[0].data[125,10],coeff[0],1e-6)
def simple_ramp(fit_method='OLS', ngroups=10, cr_group=None):
# Here given a 10 group ramp with an exact slope of 20/group.
# The output slope should be 20.
slope_per_group = 20.
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=ngroups)
for k in range(ngroups):
model1.data[0, k, :, :] = 10.0 + k * slope_per_group
if cr_group:
model1.groupdq[0, cr_group, :, :] = dqflags.group['JUMP_DET']
slopes = ramp_fit(model1, 1024 * 1., False, rnModel, gain, fit_method,
'optimal')
return slopes
def test_subarray_5groups(self, method):
#all pixel values are zero. So slope should be zero
model1, gdq, rnModel, pixdq, err, gain = setup_subarray_inputs(ngroups=5,subxstart=100, subystart=200, subxsize=50, subysize=150, readnoise=50)
model1.meta.exposure.ngroups = 11
model1.data[0, 0, 125, 10] = 10.0
model1.data[0, 1, 125, 10] = 15.0
model1.data[0, 2, 125, 10] = 25.0
model1.data[0, 3, 125, 10] = 33.0
model1.data[0, 4, 125, 10] = 60.0
xvalues = np.arange(5)*1.0
yvalues = np.array([10,15,25,33,60])
coeff = np.polyfit(xvalues, yvalues, 1)
slopes = ramp_fit(model1, 64000, False, rnModel, gain, 'OLS', 'optimal')
np.testing.assert_allclose(slopes[0].data[125,10],coeff[0],1e-6)
def test_photon_noise_only_fit():
#
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=5,
gain=1000,
readnoise=1)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 15.0
model1.data[0, 2, 500, 500] = 25.0
model1.data[0, 3, 500, 500] = 33.0
model1.data[0, 4, 500, 500] = 60.0
slopes = ramp_fit(model1, 1024 * 30000., True, rnModel, gain, 'OLS',
'optimal')
cds_slope = (model1.data[0, 4, 500, 500] -
model1.data[0, 0, 500, 500]) / 4.0
np.testing.assert_allclose(slopes[0].data[500, 500], cds_slope, 1e-2)
def test_simple_gls_ramp():
#Here given a 10 group ramp with an exact slope of 20/group. The output slope should be 20.
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=10)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 30.0
model1.data[0, 2, 500, 500] = 50.0
model1.data[0, 3, 500, 500] = 70.0
model1.data[0, 4, 500, 500] = 90.0
model1.data[0, 5, 500, 500] = 110.0
model1.data[0, 6, 500, 500] = 130.0
model1.data[0, 7, 500, 500] = 150.0
model1.data[0, 8, 500, 500] = 170.0
model1.data[0, 9, 500, 500] = 190.0
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'GLS', 'optimal')
# take the ratio of the slopes to get the relative error
np.testing.assert_allclose(slopes[0].data[500, 500], 20.0, 1e-6)
def test_two_groups_unc(self, method):
grouptime=3.0
deltaDN = 5
ingain = 2
inreadnoise =10
ngroups=2
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=ngroups, gain=ingain, readnoise=inreadnoise,deltatime=grouptime)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 10.0 + deltaDN
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
# delta_electrons = deltaDN * ingain
single_sample_readnoise = inreadnoise/np.sqrt(2)
np.testing.assert_allclose(slopes[0].var_poisson[500,500],((deltaDN/ingain)/grouptime**2), 1e-6)
np.testing.assert_allclose(slopes[0].var_rnoise[500,500],(inreadnoise**2/grouptime**2), 1e-6)
np.testing.assert_allclose(slopes[0].var_rnoise[500,500],(12*single_sample_readnoise**2/(ngroups*(ngroups**2 - 1)*grouptime**2)), 1e-6)
np.testing.assert_allclose(slopes[0].err[500,500],(np.sqrt((deltaDN/ingain)/grouptime**2+(inreadnoise**2/grouptime**2))), 1e-6)
def test_simple_ramp(self, method):
#Here given a 10 group ramp with an exact slope of 20/group. The output slope should be 20.
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=10, deltatime=3)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 30.0
model1.data[0, 2, 500, 500] = 50.0
model1.data[0, 3, 500, 500] = 70.0
model1.data[0, 4, 500, 500] = 90.0
model1.data[0, 5, 500, 500] = 110.0
model1.data[0, 6, 500, 500] = 130.0
model1.data[0, 7, 500, 500] = 150.0
model1.data[0, 8, 500, 500] = 170.0
model1.data[0, 9, 500, 500] = 190.0
slopes = ramp_fit(model1, 64000, True, rnModel, gain, 'OLS', 'optimal')
# take the ratio of the slopes to get the relative error
np.testing.assert_allclose(slopes[0].data[500, 500], (20.0/3), 1e-6)
def test_four_groups_three_CRs_at_end():
#
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=4,
gain=1,
readnoise=10)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 15.0
model1.data[0, 2, 500, 500] = 25.0
model1.data[0, 3, 500, 500] = 145.0
model1.groupdq[0, 1, 500, 500] = dqflags.group['JUMP_DET']
model1.groupdq[0, 2, 500, 500] = dqflags.group['JUMP_DET']
model1.groupdq[0, 3, 500, 500] = dqflags.group['JUMP_DET']
slopes = ramp_fit(model1, 1024 * 30000., True, rnModel, gain, 'OLS',
'optimal')
expected_slope = 10.0
np.testing.assert_allclose(slopes[0].data[500, 500], expected_slope, 1e-6)
def test_two_groups_unc():
grouptime=3.0
deltaDN = 5
ingain = 2
inreadnoise =10
ngroups=2
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=ngroups, gain=ingain, readnoise=inreadnoise,deltatime=grouptime)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 10.0 + deltaDN
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
delta_electrons = deltaDN * ingain
single_sample_readnoise = inreadnoise/np.sqrt(2)
np.testing.assert_allclose(slopes[0].var_poisson[500,500],((deltaDN/ingain)/grouptime**2), 1e-6)
np.testing.assert_allclose(slopes[0].var_rnoise[500,500],(inreadnoise**2/grouptime**2), 1e-6)
np.testing.assert_allclose(slopes[0].var_rnoise[500,500],(12*single_sample_readnoise**2/(ngroups*(ngroups**2 - 1)*grouptime**2)), 1e-6)
np.testing.assert_allclose(slopes[0].err[500,500],(np.sqrt((deltaDN/ingain)/grouptime**2+(inreadnoise**2/grouptime**2))), 1e-6)
def test_four_groups_four_CRs(self, method):
#
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=4,
gain=1,
readnoise=10)
model1.data[0, 0, 50, 50] = 10.0
model1.data[0, 1, 50, 50] = 15.0
model1.data[0, 2, 50, 50] = 25.0
model1.data[0, 3, 50, 50] = 145.0
model1.groupdq[0, 0, 50, 50] = dqflags.group['JUMP_DET']
model1.groupdq[0, 1, 50, 50] = dqflags.group['JUMP_DET']
model1.groupdq[0, 2, 50, 50] = dqflags.group['JUMP_DET']
model1.groupdq[0, 3, 50, 50] = dqflags.group['JUMP_DET']
slopes = ramp_fit(model1, 1024 * 30000., True, rnModel, gain, 'OLS',
'optimal', 'none')
np.testing.assert_allclose(slopes[0].data[50, 50], 0, 1e-6)
def test_oneCR_10_groups_combination(self, method):
grouptime = 3.0
# deltaDN = 5
ingain = 200 # use large gain to show that Poisson noise doesn't affect the recombination
inreadnoise = np.float64(7)
ngroups = 10
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(
ngroups=ngroups,
gain=ingain,
readnoise=inreadnoise,
deltatime=grouptime)
# two segments perfect fit, second segment has twice the slope
model1.data[0, 0, 50, 50] = 15.0
model1.data[0, 1, 50, 50] = 20.0
model1.data[0, 2, 50, 50] = 25.0
model1.data[0, 3, 50, 50] = 30.0
model1.data[0, 4, 50, 50] = 35.0
model1.data[0, 5, 50, 50] = 140.0
model1.data[0, 6, 50, 50] = 150.0
model1.data[0, 7, 50, 50] = 160.0
model1.data[0, 8, 50, 50] = 170.0
model1.data[0, 9, 50, 50] = 180.0
model1.groupdq[0, 5, 50, 50] = dqflags.group['JUMP_DET']
slopes, int_model, opt_model, gls_opt_model = ramp_fit(
model1, 1024 * 30000., True, rnModel, gain, 'OLS', 'optimal',
'none')
segment_groups = 5
single_sample_readnoise = np.float64(inreadnoise / np.sqrt(2))
#check that the segment variance is as expected
np.testing.assert_allclose(opt_model.var_rnoise[0, 0, 50, 50],
(12.0 * single_sample_readnoise**2 /
(segment_groups *
(segment_groups**2 - 1) * grouptime**2)),
rtol=1e-6)
# check the combined slope is the average of the two segments since they have the same number of groups
np.testing.assert_allclose(slopes.data[50, 50], 2.5, rtol=1e-5)
#check that the slopes of the two segments are correct
np.testing.assert_allclose(opt_model.slope[0, 0, 50, 50],
5 / 3.0,
rtol=1e-5)
np.testing.assert_allclose(opt_model.slope[0, 1, 50, 50],
10 / 3.0,
rtol=1e-5)
def test_pix_2():
"""
CASE B: segment has >2 groups, not at end of ramp.
CASE F: (twice) segment has 2 good groups not at array end.
SCI segs are: seg0[15,25,35](B), seg1[54,55](F), seg2[65,75](F)
"""
ngroups, nints, nrows, ncols, deltatime, gain, readnoise = set_scalars()
ramp_model, rnoise_model, gain_model, pixdq, groupdq, err = create_mod_arrays(
ngroups, nints, nrows, ncols, deltatime, gain, readnoise)
# Populate pixel-specific SCI and GROUPDQ arrays
ramp_model.data[0, :, 0, 0] = np.array(
[15., 25., 35., 54., 55., 65., 75., 94., 95., 105.], dtype=np.float32)
ramp_model.groupdq[0, :, 0, 0] = np.array([0, 0, 0, 4, 0, 4, 0, 4, 2, 2])
new_mod, int_mod, opt_mod, gls_opt_mod = ramp_fit(ramp_model, 1024 * 300.,
True, rnoise_model,
gain_model, 'OLS',
'optimal', 'none')
# Set truth values for PRIMARY results:
p_true = [0.84761256, 6, 0.42986465, 0.00659091, 0.1781927]
# Set truth values for OPTIONAL results for all segments
o_true = [
[1.0000001, 0.1, 1.], # slopes for 3 segments
[28.435, 56.870003, 56.870003], # sigslope
[
0.01318182,
0.02636364,
0.02636364,
], # var_poisson
[0.26728904, 1.0691562, 1.0691562], # var_rnoise
[14.999998, 51., 15.], # yint
[36.709427, 56.870003, 56.870003], # sigyint
[6.5238733], # pedestal
[12.712313, 0.83321977, 0.83321977], # weights
]
assert_pri(p_true, new_mod, 0)
assert_opt(o_true, opt_mod, 0)
def test_two_groups_fit(self, method):
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=2, gain=1, readnoise=10)
model1.data[0, 0, 50, 50] = 10.0
model1.data[0, 1, 50, 50] = 15.0
model1.data[0, 0, 50, 51] = 20.0
model1.data[0, 1, 50, 51] = 60.0
model1.data[0, 0, 50, 52] = 200.0
model1.data[0, 1, 50, 52] = 600.0
model1.meta.exposure.drop_frames1 = 0
# 2nd group is saturated
model1.groupdq[0, 1, 50, 51] = SATURATED
# 1st group is saturated
model1.groupdq[0, 0, 50, 52] = SATURATED
model1.groupdq[0, 1, 50, 52] = SATURATED # should not be set this way
slopes = ramp_fit(model1, 1024 * 30000., True, rnModel, gain, 'OLS', 'optimal', 'none')
cds_slope = (model1.data[0, 1, 50, 50] - model1.data[0, 0, 50, 50])
np.testing.assert_allclose(slopes[0].data[50, 50], cds_slope, 1e-6)
# expect SATURATED
assert slopes[0].dq[50, 51] == SATURATED
# expect SATURATED and DO_NOT_USE, because 1st group is Saturated
assert slopes[0].dq[50, 52] == SATURATED + DO_NOT_USE
def test_two_groups_fit(self, method):
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=2,gain=1,readnoise=10)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 15.0
model1.data[0, 0, 500, 501] = 20.0
model1.data[0, 1, 500, 501] = 60.0
model1.data[0, 0, 500, 502] = 200.0
model1.data[0, 1, 500, 502] = 600.0
model1.meta.exposure.drop_frames1 = 0
#2nd group is saturated
model1.groupdq[0,1,500,501]=dqflags.group['SATURATED']
#1st group is saturated
model1.groupdq[0,0,500,502]=dqflags.group['SATURATED']
model1.groupdq[0,1,500,502]=dqflags.group['SATURATED'] #should not be set this way
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
cds_slope = (model1.data[0,1,500,500] - model1.data[0,0,500,500])
np.testing.assert_allclose(slopes[0].data[500, 500], cds_slope, 1e-6)
#expect SATURATED
assert slopes[0].dq[500, 501] == 2 # is there a better way to do this test?
#expect SATURATED since 1st group is Saturated
assert slopes[0].dq[500, 502] == 2 # is there a better way to do this test?
def test_two_groups_fit():
#
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=2,gain=1,readnoise=10)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 15.0
model1.data[0, 0, 500, 501] = 20.0
model1.data[0, 1, 500, 501] = 60.0
model1.data[0, 0, 500, 502] = 200.0
model1.data[0, 1, 500, 502] = 600.0
model1.meta.exposure.drop_frames1 = 0
#2nd group is saturated
model1.groupdq[0,1,500,501]=dqflags.group['SATURATED']
#1st group is saturated
model1.groupdq[0,0,500,502]=dqflags.group['SATURATED']
model1.groupdq[0,1,500,502]=dqflags.group['SATURATED'] #should not be set this way
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
cds_slope = (model1.data[0,1,500,500] - model1.data[0,0,500,500])
np.testing.assert_allclose(slopes[0].data[500, 500], cds_slope, 1e-6)
#expect SATURATED
assert slopes[0].dq[500, 501] == 2 # is there a better way to do this test?
#expect SATURATED since 1st group is Saturated
assert slopes[0].dq[500, 502] == 2 # is there a better way to do this test?
def test_five_groups_two_ints_Poisson_noise_only(self, method):
grouptime=3.0
ingain = 2000
inreadnoise =7
ngroups=5
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=ngroups,
gain=ingain, readnoise=inreadnoise,deltatime=grouptime, nints=2)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 15.0
model1.data[0, 2, 500, 500] = 25.0
model1.data[0, 3, 500, 500] = 33.0
model1.data[0, 4, 500, 500] = 60.0
model1.data[1, 0, 500, 500] = 10.0
model1.data[1, 1, 500, 500] = 15.0
model1.data[1, 2, 500, 500] = 25.0
model1.data[1, 3, 500, 500] = 33.0
model1.data[1, 4, 500, 500] = 160.0
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
out_slope=slopes[0].data[500, 500]
deltaDN1 = 50
deltaDN2 = 150
np.testing.assert_allclose(out_slope, (deltaDN1 + deltaDN2)/2.0, 75.0, 1e-6)
def test_miri_first_last():
''' This is a test of whether ramp fitting correctly handles having all 0th
group dq flagged as DO_NOT_USE, and all final group dq flagged as
DO_NOT_USE for MIRI data. For 1 pixel ([1,1]) the 1st (again, 0-based)
group is flagged as a CR. For such a ramp, the code removes the CR-flag
from such a CR-affected 1st group; so if it initially was 4 it is reset
to 0 ("good"), in which case it's as if that CR was not even there.
'''
(ngroups, nints, nrows, ncols, deltatime) = (10, 1, 2, 2, 3.)
model1, gdq, rnModel, pixdq, err, gain = setup_small_cube(
ngroups, nints, nrows, ncols, deltatime)
# Make smooth ramps having outlier SCI values in the 0th and final groups
# to reveal if they are included in the fit (they shouldn't be, as those
# groups are flagged as DO_NOT_USE)
model1.data[0, :, 0, 0] = np.array(
[-200., 30., 40., 50., 60., 70., 80., 90., 100., -500.],
dtype=np.float32)
model1.data[0, :, 0, 1] = np.array(
[-300., 80., 90., 100., 110., 120., 130., 140., 150., -600.],
dtype=np.float32)
model1.data[0, :, 1, 0] = np.array(
[-200., 40., 50., 60., 70., 80., 90., 100., 110., 900.],
dtype=np.float32)
model1.data[0, :, 1, 1] = np.array(
[-600., 140., 150., 160., 170., 180., 190., 200., 210., -400.],
dtype=np.float32)
# For all pixels, set gdq for 0th and final groups to DO_NOT_USE
model1.groupdq[:, 0, :, :] = dqflags.group['DO_NOT_USE']
model1.groupdq[:, -1, :, :] = dqflags.group['DO_NOT_USE']
# Put CR in 1st (0-based) group
model1.groupdq[0, 1, 1, 1] = dqflags.group['JUMP_DET']
new_mod, int_model, opt_model, gls_opt_model = ramp_fit(
model1, 1024 * 30000., True, rnModel, gain, 'OLS', 'optimal', 'none')
np.testing.assert_allclose(new_mod.data, 10. / 3., rtol=1E-5)
def test_twenty_groups_two_segments():
'''Test to verify weighting of multiple segments in combination:
a) gdq all 0 ; b) 1 CR (2 segments) c) 1 CR then SAT (2 segments)
'''
(ngroups, nints, nrows, ncols, deltatime) = (20, 1, 1, 3, 6.)
model1, gdq, rnModel, pixdq, err, gain = setup_small_cube(
ngroups, nints, nrows, ncols, deltatime)
# a) ramp having gdq all 0
model1.data[0, :, 0, 0] = np.arange(ngroups) * 10. + 30.
# b) ramp having 1 CR at group 15; 2 segments
model1.data[0, :, 0, 1] = np.arange(ngroups) * 10. + 50.
gdq[0, 15, 0, 1] = dqflags.group['JUMP_DET']
model1.data[0, 15:, 0, 1] += 1000.
# c) ramp having 1 CR at group 2; SAT starting in group 15
model1.data[0, :, 0, 2] = np.arange(ngroups) * 10. + 70.
gdq[0, 2, 0, 2] = dqflags.group['JUMP_DET']
model1.data[0, 2:, 0, 2] += 2000.
gdq[0, 15:, 0, 2] = dqflags.group['SATURATED']
model1.data[0, 15:, 0, 2] = 25000.
new_mod, int_model, opt_model, gls_opt_model = ramp_fit(
model1, 1024 * 30000., True, rnModel, gain, 'OLS', 'optimal', 'none')
# Check some PRI & OPT output arrays
np.testing.assert_allclose(new_mod.data, 10. / deltatime, rtol=1E-4)
wh_data = opt_model.slope != 0. # only test existing segments
np.testing.assert_allclose(opt_model.slope[wh_data],
10. / deltatime,
rtol=1E-4)
np.testing.assert_allclose(opt_model.yint[0, 0, 0, :],
model1.data[0, 0, 0, :],
rtol=1E-5)
np.testing.assert_allclose(opt_model.pedestal[0, 0, :],
model1.data[0, 0, 0, :] - 10.,
rtol=1E-5)
def test_pix_3():
"""
CASE B: segment has >2 groups, not at end of ramp.
CASE E: segment has 2 good groups, at end of ramp.
SCI segs are: seg0[15,25,35,54,55,65,75,94](B), seg1[95,105](E)
"""
ngroups, nints, nrows, ncols, deltatime, gain, readnoise = set_scalars()
ramp_model, rnoise_model, gain_model, pixdq, groupdq, err = create_mod_arrays(
ngroups, nints, nrows, ncols, deltatime, gain, readnoise)
# Populate pixel-specific SCI and GROUPDQ arrays
ramp_model.data[0, :, 0, 0] = np.array(
[15., 25., 35., 54., 55., 65., 75., 94., 95., 105.], dtype=np.float32)
ramp_model.groupdq[0, :, 0, 0] = np.array([0, 0, 0, 0, 0, 0, 0, 0, 4, 0])
new_mod, int_mod, opt_mod, gls_opt_mod = ramp_fit(ramp_model, 1024 * 300.,
True, rnoise_model,
gain_model, 'OLS',
'optimal', 'none')
# Set truth values for PRIMARY results:
p_true = [1.0746869, 4, 0.12186482, 0.00227273, 0.01257831]
# Set truth values for OPTIONAL results:
o_true = [
[1.0757396, 1.],
[6.450687, 56.870003],
[0.0025974, 0.01818182],
[0.01272805, 1.0691562],
[14.504965, 15.],
[27.842508, 56.870003],
[4.253134],
[4.2576841e+03, 8.458062e-01],
]
assert_pri(p_true, new_mod, 0)
assert_opt(o_true, opt_mod, 0)
def test_five_groups_unc():
grouptime=3.0
deltaDN = 5
ingain = 2
inreadnoise =7
ngroups=5
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=ngroups,
gain=ingain, readnoise=inreadnoise,deltatime=grouptime)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 15.0
model1.data[0, 2, 500, 500] = 25.0
model1.data[0, 3, 500, 500] = 33.0
model1.data[0, 4, 500, 500] = 60.0
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
out_slope=slopes[0].data[500, 500]
median_slope=np.median(np.diff(model1.data[0,:,500,500]))/grouptime
deltaDN = 50
delta_time = (ngroups - 1) * grouptime
delta_electrons = median_slope * ingain *delta_time
single_sample_readnoise = np.float64(inreadnoise/np.sqrt(2))
np.testing.assert_allclose(slopes[0].var_poisson[500,500],((median_slope)/(ingain*delta_time)), 1e-6)
np.testing.assert_allclose(slopes[0].var_rnoise[500,500],(12 * single_sample_readnoise**2/(ngroups * (ngroups**2 - 1) * grouptime**2)), 1e-6)
np.testing.assert_allclose(slopes[0].err[500,500],np.sqrt(slopes[0].var_poisson[500,500] + slopes[0].var_rnoise[500,500] ), 1e-6)
def test_five_groups_unc(self, method):
grouptime=3.0
# deltaDN = 5
ingain = 2
inreadnoise =7
ngroups=5
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=ngroups,
gain=ingain, readnoise=inreadnoise,deltatime=grouptime)
model1.data[0, 0, 500, 500] = 10.0
model1.data[0, 1, 500, 500] = 15.0
model1.data[0, 2, 500, 500] = 25.0
model1.data[0, 3, 500, 500] = 33.0
model1.data[0, 4, 500, 500] = 60.0
slopes = ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
# out_slope=slopes[0].data[500, 500]
median_slope=np.median(np.diff(model1.data[0,:,500,500]))/grouptime
# deltaDN = 50
delta_time = (ngroups - 1) * grouptime
# delta_electrons = median_slope * ingain *delta_time
single_sample_readnoise = np.float64(inreadnoise/np.sqrt(2))
np.testing.assert_allclose(slopes[0].var_poisson[500,500],((median_slope)/(ingain*delta_time)), 1e-6)
np.testing.assert_allclose(slopes[0].var_rnoise[500,500],(12 * single_sample_readnoise**2/(ngroups * (ngroups**2 - 1) * grouptime**2)), 1e-6)
np.testing.assert_allclose(slopes[0].err[500,500],np.sqrt(slopes[0].var_poisson[500,500] + slopes[0].var_rnoise[500,500] ), 1e-6)
def test_pix_10():
"""
CASE F: segment has 2 good groups not at array end.
CASE B: segment has >2 groups, not at end of ramp.
CASE A: segment has >2 groups, at end of ramp.
SCI segs are: seg0[15,25](F), seg1[35,54,55](B), seg2[65,75,94,95,105](A)
"""
ngroups, nints, nrows, ncols, deltatime, gain, readnoise = set_scalars()
RampMod, RnMod, GMod, pixdq, groupdq, err = create_mod_arrays( ngroups,
nints, nrows, ncols, deltatime, gain, readnoise )
# Populate pixel-specific SCI and GROUPDQ arrays
RampMod.data[0,:,0,0] = np.array([ 15., 25., 35., 54., 55., 65., 75.,
94., 95., 105.], dtype=np.float32)
RampMod.groupdq[0,:,0,0] = np.array([0, 0, 4, 0, 0, 4, 0, 0, 0, 0])
new_mod, int_mod, opt_mod, gls_opt_mod = ramp_fit( RampMod, 1024*300., True,
RnMod, GMod, 'OLS', 'optimal' )
# Set truth values for PRIMARY results:
p_true = [ 1., 4, 0.21298744, 0.0025974, 0.04276625 ]
# Set truth values for OPTIONAL results:
o_true = [[ 1., 1.0000014, 0.99999964 ],
[ 56.870003, 28.434996, 13.036095 ],
[ 0.01818182, 0.00909091, 0.00454545],
[ 1.0691562, 0.26728904, 0.05345781],
[ 15., 17.999956, 15.000029],
[ 56.870003, 88.40799, 93.73906 ],
[ 5. ],
[ 0.84580624, 13.091425, 297.23172 ]
]
assert_pri( p_true, new_mod )
assert_opt( o_true, opt_mod )
def test_pix_9():
"""
CASE F: segment has 2 good groups not at array end.
CASE B: segment has >2 groups, not at end of ramp.
CASE E: segment has 2 good groups, at end of ramp.
SCI seg are: seg0[15,25](F), seg1[54, 55, 65, 75, 94](B), seg2[95, 105](E)
"""
ngroups, nints, nrows, ncols, deltatime, gain, readnoise = set_scalars()
RampMod, RnMod, GMod, pixdq, groupdq, err = create_mod_arrays( ngroups,
nints, nrows, ncols, deltatime, gain, readnoise )
# Populate pixel-specific SCI and GROUPDQ arrays
RampMod.data[0,:,0,0] = np.array([ 15., 25., 35., 54., 55., 65., 75.,
94., 95., 105.], dtype=np.float32)
RampMod.groupdq[0,:,0,0] = np.array([0, 0, 4, 4, 0, 0, 0, 0, 4, 0])
new_mod, int_mod, opt_mod, gls_opt_mod = ramp_fit( RampMod, 1024*300., True,
RnMod, GMod, 'OLS', 'optimal' )
# Set truth values for PRIMARY results:
p_true = [ 0.9999994, 4, 0.22721863, 0.0030303, 0.048598]
# Set truth values for OPTIONAL results:
o_true = [[ 1., 0.9999994, 1. ],
[ 56.870003, 13.036095, 56.870003 ],
[ 0.01818182, 0.00454545, 0.01818182 ],
[ 1.0691562, 0.05345781, 1.0691562 ],
[ 15., 20.119896, 15. ],
[ 56.870003, 68.618195, 56.870003 ],
[ 5.000005 ],
[ 0.84580624, 297.23172, 0.84580624]
]
assert_pri( p_true, new_mod )
assert_opt( o_true, opt_mod )
def test_oneCR_10_groups_combination_noisy2ndSegment(self, method):
grouptime=3.0
# deltaDN = 5
ingain = 200 # use large gain to show that Poisson noise doesn't affect the recombination
inreadnoise =7
ngroups=10
model1, gdq, rnModel, pixdq, err, gain = setup_inputs(ngroups=ngroups,
gain=ingain, readnoise=inreadnoise,deltatime=grouptime)
# two segments perfect fit, second segment has twice the slope
model1.data[0, 0, 500, 500] = 15.0
model1.data[0, 1, 500, 500] = 20.0
model1.data[0, 2, 500, 500] = 25.0
model1.data[0, 3, 500, 500] = 30.0
model1.data[0, 4, 500, 500] = 35.0
model1.data[0, 5, 500, 500] = 135.0
model1.data[0, 6, 500, 500] = 155.0
model1.data[0, 7, 500, 500] = 160.0
model1.data[0, 8, 500, 500] = 168.0
model1.data[0, 9, 500, 500] = 180.0
model1.groupdq[0,5,500,500]=dqflags.group['JUMP_DET']
slopes, int_model, opt_model, gls_opt_model= ramp_fit(model1, 1024*30000., True, rnModel, gain, 'OLS', 'optimal')
avg_slope = (opt_model.slope[0,0,500, 500] + opt_model.slope[0,1,500, 500])/2.0
#even with noiser second segment, final slope should be just the average since they have the same number of groups
np.testing.assert_allclose(slopes.data[500, 500], avg_slope,rtol=1e-5)