def read_datamodel(self):
h = RampModel(self.file)
#remove any non-pipeline related keywords (e.g. CV3 temps/voltages)
if 'extra_fits' in dir(h):
h.__delattr__('extra_fits')
self.data = h.data
#Currently a bug in level1bmodel when zeroframe is
#not present and a cube of zeros is returned
#If the datamodel returns a default zeroframe of all
#zeros, then set it to None here
#self.zeroframe = h.zeroframe
if np.all(h.zeroframe == 0):
print("Zeroframe in {}".format(self.file))
print("All zeros. Returning None.")
self.zeroframe = None
else:
self.zeroframe = h.zeroframe
self.sbAndRefpix = None
self.header = {}
for key in self.translate:
try:
self.header[key] = h.meta[self.translate[key]]
except:
self.header[key] = None
def make_rawramp(instrument,
nints,
ngroups,
ysize,
xsize,
ystart,
xstart,
exp_type=None):
# create the data and groupdq arrays
csize = (nints, ngroups, ysize, xsize)
data = np.full(csize, 1.0)
# create a JWST datamodel
if instrument == "FGS":
dm_ramp = GuiderRawModel(data=data)
dm_ramp.meta.exposure.type = exp_type
elif instrument == "MIRI":
dm_ramp = RampModel(data=data)
else:
dm_ramp = RampModel(data=data)
dm_ramp.meta.subarray.xstart = xstart
dm_ramp.meta.subarray.xsize = xsize
dm_ramp.meta.subarray.ystart = ystart
dm_ramp.meta.subarray.ysize = ysize
return dm_ramp
def make_rampmodel(ngroups, ysize, xsize):
'''Make MIRI ramp model for testing'''
# create the data and groupdq arrays
csize = (1, ngroups, ysize, xsize)
# create JWST datamodel and set each frame equal to frame number
dm_ramp = RampModel(csize)
for i in range(0, ngroups - 1):
dm_ramp.data[0, i, :, :] = i
# populate header of data model
dm_ramp.meta.instrument.name = 'MIRI'
dm_ramp.meta.instrument.detector = 'MIRIMAGE'
dm_ramp.meta.instrument.filter = 'F560W'
dm_ramp.meta.instrument.band = 'N/A'
dm_ramp.meta.observation.date = '2016-06-01'
dm_ramp.meta.observation.time = '00:00:00'
dm_ramp.meta.exposure.type = 'MIR_IMAGE'
dm_ramp.meta.subarray.name = 'FULL'
dm_ramp.meta.subarray.xstart = 1
dm_ramp.meta.subarray.xsize = xsize
dm_ramp.meta.subarray.ystart = 1
dm_ramp.meta.subarray.ysize = ysize
return dm_ramp
def setup_cube(ngroups, nrows, ncols):
''' Set up fake data to test.'''
nints = 1
data_model = RampModel()
data_model.data = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float32)
data_model.pixeldq = np.zeros(shape=(nrows, ncols), dtype=np.int32)
data_model.groupdq = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float32)
data_model.meta.subarray.xstart = 1
data_model.meta.subarray.ystart = 1
data_model.meta.subarray.xsize = ncols
data_model.meta.subarray.ysize = nrows
data_model.meta.exposure.ngroups = ngroups
data_model.meta.instrument.name = 'NIRCAM'
saturation_model = SaturationModel()
saturation_model.data = np.zeros(shape=(2048, 2048), dtype=np.float32)
saturation_model.dq = np.zeros(shape=(2048, 2048), dtype=np.int32)
saturation_model.meta.subarray.xstart = 1
saturation_model.meta.subarray.ystart = 1
saturation_model.meta.subarray.xsize = 2048
saturation_model.meta.subarray.ysize = 2048
saturation_model.meta.instrument.name = 'NIRCAM'
saturation_model.meta.description = 'Fake data.'
saturation_model.meta.telescope = 'JWST'
saturation_model.meta.reftype = 'SaturationModel'
saturation_model.meta.author = 'Alicia'
saturation_model.meta.pedigree = 'Dummy'
saturation_model.meta.useafter = '2015-10-01T00:00:00'
return data_model, saturation_model
def test_firstframe_add1_groupdq():
"""
Test if the firstframe code set the groupdq flag on the first
group to 'do_not_use' by adding 1 to the flag, not overwriting to 1
"""
# size of integration
ngroups = 5
xsize = 1032
ysize = 1024
# create the data and groupdq arrays
csize = (1, ngroups, ysize, xsize)
data = np.full(csize, 1.0)
groupdq = np.zeros(csize, dtype=int)
# create a JWST datamodel for MIRI data
dm_ramp = RampModel(data=data, groupdq=groupdq)
# set a flag in the groupdq, first frame
dm_ramp.groupdq[0, 0, 500:510, 500:510] = 4
# run the first frame correction step
dm_ramp_firstframe = do_correction(dm_ramp)
# test if pixels in groupdq were incremented in value by 1
assert (dm_ramp_firstframe.groupdq[0, 0, 505, 505] == 5)
def setup_cube(ngroups, nrows, ncols):
''' Set up fake data to test.'''
nints = 1
data_model = RampModel()
data_model.data = np.zeros(shape=(nints, ngroups, nrows, ncols),
dtype=np.float32)
data_model.pixeldq = np.zeros(shape=(nrows, ncols), dtype=np.int32)
data_model.meta.subarray.xstart = 1
data_model.meta.subarray.ystart = 1
data_model.meta.subarray.xsize = ncols
data_model.meta.subarray.ysize = nrows
data_model.meta.instrument.name = 'NIRCAM'
bias_model = SuperBiasModel()
bias_model.data = np.zeros(shape=(2048, 2048), dtype=np.float32)
bias_model.dq = np.zeros(shape=(2048, 2048), dtype=np.int32)
bias_model.meta.subarray.xstart = 1
bias_model.meta.subarray.ystart = 1
bias_model.meta.subarray.xsize = 2048
bias_model.meta.subarray.ysize = 2048
bias_model.meta.instrument.name = 'NIRCAM'
bias_model.meta.description = 'Fake data.'
bias_model.meta.telescope = 'JWST'
bias_model.meta.reftype = 'SuperBiasModel'
bias_model.meta.author = 'Alicia'
bias_model.meta.pedigree = 'Dummy'
bias_model.meta.useafter = '2015-10-01T00:00:00'
return data_model, bias_model
def _cube(ngroups, nrows, ncols):
nints = 1
data_model = RampModel((nints, ngroups, nrows, ncols))
data_model.meta.subarray.xstart = 1
data_model.meta.subarray.ystart = 1
data_model.meta.subarray.xsize = ncols
data_model.meta.subarray.ysize = nrows
data_model.meta.exposure.ngroups = ngroups
data_model.meta.instrument.name = 'NIRCAM'
data_model.meta.instrument.detector = 'NRCA1'
data_model.meta.observation.date = '2017-10-01'
data_model.meta.observation.time = '00:00:00'
saturation_model = SaturationModel((2048, 2048))
saturation_model.meta.subarray.xstart = 1
saturation_model.meta.subarray.ystart = 1
saturation_model.meta.subarray.xsize = 2048
saturation_model.meta.subarray.ysize = 2048
saturation_model.meta.instrument.name = 'NIRCAM'
saturation_model.meta.description = 'Fake data.'
saturation_model.meta.telescope = 'JWST'
saturation_model.meta.reftype = 'SaturationModel'
saturation_model.meta.author = 'Alicia'
saturation_model.meta.pedigree = 'Dummy'
saturation_model.meta.useafter = '2015-10-01T00:00:00'
return data_model, saturation_model
def test_firstframe_single_group():
"""
Test that the firstframe code does nothing when passed a single
group integration
"""
# size of integration
ngroups = 1
xsize = 1032
ysize = 1024
# create the data and groupdq arrays
csize = (1, ngroups, ysize, xsize)
data = np.full(csize, 1.0)
groupdq = np.zeros(csize, dtype=int)
# create a JWST datamodel for MIRI data
dm_ramp = RampModel(data=data, groupdq=groupdq)
# run the first frame correction step
dm_ramp_firstframe = do_correction(dm_ramp)
# check that the difference in the groupdq flags is equal to
# the 'do_not_use' flag
dq_diff = dm_ramp_firstframe.groupdq[0, 0, :, :] - dm_ramp.groupdq[0,
0, :, :]
np.testing.assert_array_equal(np.full((ysize, xsize), 0, dtype=int),
dq_diff,
err_msg='groupdq changed for single group ' +
'when it should not')
def test_firstframe_3groups():
"""
Test if the firstframe code set the groupdq flag on the first
group to 'do_not_use' or left it as is, which it should do for 3 frames
"""
# size of integration
ngroups = 3
xsize = 1032
ysize = 1024
# create the data and groupdq arrays
csize = (1, ngroups, ysize, xsize)
data = np.full(csize, 1.0)
groupdq = np.zeros(csize, dtype=int)
# create a JWST datamodel for MIRI data
dm_ramp = RampModel(data=data, groupdq=groupdq)
# run the first frame correction step
dm_ramp_firstframe = do_correction(dm_ramp)
# check that the difference in the groupdq flags is equal to
# 0 since the step should not run if frames 3 or fewer
dq_diff = dm_ramp_firstframe.groupdq[0, 0, :, :] - dm_ramp.groupdq[0,
0, :, :]
np.testing.assert_array_equal(np.full((ysize, xsize), 0, dtype=int),
dq_diff,
err_msg='Diff in groupdq flags is not ' +
'equal to 0')
def _cube(xstart, ystart, ngroups, nrows, ncols):
nints = 1
# create a JWST datamodel for NIRCam SUB320A335R data
data_model = RampModel((nints, ngroups, nrows, ncols))
data_model.meta.subarray.name = 'SUB320A335R'
data_model.meta.subarray.xstart = xstart
data_model.meta.subarray.ystart = ystart
data_model.meta.subarray.xsize = ncols
data_model.meta.subarray.ysize = nrows
data_model.meta.exposure.ngroups = ngroups
data_model.meta.instrument.name = 'NIRCAM'
data_model.meta.instrument.detector = 'NRCALONG'
data_model.meta.observation.date = '2019-10-14'
data_model.meta.observation.time = '16:44:12.000'
# create a superbias model for the superbias step
bias_model = SuperBiasModel((2048, 2048))
bias_model.meta.subarray.xstart = 1
bias_model.meta.subarray.ystart = 1
bias_model.meta.subarray.xsize = 2048
bias_model.meta.subarray.ysize = 2048
bias_model.meta.instrument.name = 'NIRCAM'
bias_model.meta.description = 'Fake data.'
bias_model.meta.telescope = 'JWST'
bias_model.meta.reftype = 'SuperBiasModel'
bias_model.meta.author = 'Alicia'
bias_model.meta.pedigree = 'Dummy'
bias_model.meta.useafter = '2015-10-01T00:00:00'
return data_model, bias_model
def test_lastframe_ngroup2():
"""
Test if the lastframe code set the groupdq flag on the last
group to 'do_not_use'. Lastframe should not be flagged with ngroups=2.
"""
# size of integration
ngroups = 2
xsize = 1032
ysize = 1024
# create the data and groupdq arrays
csize = (1, ngroups, ysize, xsize)
data = np.full(csize, 1.0)
groupdq = np.zeros(csize, dtype=int)
# create a JWST datamodel for MIRI data
dm_ramp = RampModel(data=data, groupdq=groupdq)
# run the last frame correction step
dm_ramp_lastframe = do_correction(dm_ramp)
# check that the difference in the groupdq flags is equal to
# zero
dq_diff = dm_ramp_lastframe.groupdq[0, ngroups -
1, :, :] - dm_ramp.groupdq[0, ngroups -
1, :, :]
np.testing.assert_array_equal(np.full((ysize, xsize), 0, dtype=int),
dq_diff,
err_msg='groupdq flag changed ' +
'when it should not')
def setup_subarray_inputs(nints=1,
ngroups=10,
nrows=1032,
ncols=1024,
subxstart=1,
subxsize=1024,
subystart=1,
subysize=1032,
nframes=1,
grouptime=1.0,
deltatime=1,
readnoise=10,
gain=1):
data = np.zeros(shape=(nints, ngroups, subysize, subxsize),
dtype=np.float32)
err = np.ones(shape=(nints, ngroups, nrows, ncols), dtype=np.float32)
pixdq = np.zeros(shape=(subysize, subxsize), dtype=np.uint32)
gdq = np.zeros(shape=(nints, ngroups, subysize, subxsize), dtype=np.uint8)
gain = np.ones(shape=(nrows, ncols), dtype=np.float64) * gain
read_noise = np.full((nrows, ncols), readnoise, dtype=np.float32)
times = np.array(list(range(ngroups)), dtype=np.float64) * deltatime
model1 = RampModel(data=data,
err=err,
pixeldq=pixdq,
groupdq=gdq,
times=times)
model1.meta.instrument.name = 'MIRI'
model1.meta.instrument.detector = 'MIRIMAGE'
model1.meta.instrument.filter = 'F480M'
model1.meta.observation.date = '2015-10-13'
model1.meta.exposure.type = 'MIR_IMAGE'
model1.meta.exposure.group_time = deltatime
model1.meta.subarray.name = 'FULL'
model1.meta.subarray.xstart = subxstart
model1.meta.subarray.ystart = subystart
model1.meta.subarray.xsize = subxsize
model1.meta.subarray.ysize = subysize
model1.meta.exposure.frame_time = deltatime
model1.meta.exposure.ngroups = ngroups
model1.meta.exposure.group_time = deltatime
model1.meta.exposure.nframes = 1
model1.meta.exposure.groupgap = 0
gain = GainModel(data=gain)
gain.meta.instrument.name = 'MIRI'
gain.meta.subarray.xstart = 1
gain.meta.subarray.ystart = 1
gain.meta.subarray.xsize = 1024
gain.meta.subarray.ysize = 1032
rnModel = ReadnoiseModel(data=read_noise)
rnModel.meta.instrument.name = 'MIRI'
rnModel.meta.subarray.xstart = 1
rnModel.meta.subarray.ystart = 1
rnModel.meta.subarray.xsize = 1024
rnModel.meta.subarray.ysize = 1032
return model1, gdq, rnModel, pixdq, err, gain
def setup_inputs(ngroups=10, readnoise=10, nints=1,
nrows=103, ncols=102, nframes=1, grouptime=1.0, gain=1, deltatime=1):
data = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float32)
err = np.ones(shape=(nints, ngroups, nrows, ncols), dtype=np.float32)
pixdq = np.zeros(shape=(nrows, ncols), dtype=np.uint32)
gdq = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.uint8)
gain = np.ones(shape=(nrows, ncols), dtype=np.float64) * gain
rnoise = np.full((nrows, ncols), readnoise, dtype=np.float32)
int_times = np.zeros((nints,))
model1 = RampModel(data=data, err=err, pixeldq=pixdq, groupdq=gdq, int_times=int_times)
model1.meta.instrument.name = 'MIRI'
model1.meta.instrument.detector = 'MIRIMAGE'
model1.meta.instrument.filter = 'F480M'
model1.meta.observation.date = '2015-10-13'
model1.meta.exposure.type = 'MIR_IMAGE'
model1.meta.exposure.group_time = deltatime
model1.meta.subarray.name = 'FULL'
model1.meta.subarray.xstart = 1
model1.meta.subarray.ystart = 1
model1.meta.subarray.xsize = ncols
model1.meta.subarray.ysize = nrows
model1.meta.exposure.frame_time = deltatime
model1.meta.exposure.ngroups = ngroups
model1.meta.exposure.group_time = deltatime
model1.meta.exposure.nframes = 1
model1.meta.exposure.groupgap = 0
model1.meta.exposure.drop_frames1 = 0
return model1, gdq, rnoise, pixdq, err, gain
def run(self):
'''Main function.'''
# initialize ramp model instance and get zeroframe file
model = RampModel(self.simfile)
outname = self.get_zeroframe(self.simfile, model)
bad_hdulist = fits.open(outname)
print('\nUnformatted file: ' + str(outname))
dat = bad_hdulist[1].data
shape = dat.shape
# get headers from original simulated exposure
good_hdulist = fits.open(self.simfile)
print('\nFile with correct formatting: ' + str(self.simfile))
headers = good_hdulist[0].header
for k, v in zip(headers.keys(), headers.values()):
#print(k,v)
bad_hdulist[0].header[k] = v
# fix NGROUPS for zeroframe size
bad_hdulist[0].header['NGROUP'] = shape[1]
bad_hdulist[0].header['NGROUPS'] = shape[1]
print('\nTesting header values...')
print('TFRAME: ' + str(bad_hdulist[0].header['TFRAME']))
print('DETECTOR: ' + str(bad_hdulist[0].header['DETECTOR']))
print('GROUPS: ' + str(bad_hdulist[0].header['NGROUPS']))
# save out the formatted file
final_outname = outname[:-5] + "_properHeaders.fits"
print('\nSaving final file: ' + str(final_outname))
bad_hdulist.writeto(final_outname, clobber=True)
def test_miri():
# test that the code runs given MIRI as an instrument
# size of integration
ngroups = 5
xsize = 1032
ysize = 1024
# create the data and groupdq arrays
csize = (1, ngroups, ysize, xsize)
data = np.full(csize, 1.0)
groupdq = np.zeros(csize, dtype=int)
# create a JWST datamodel for data
dm_ramp = RampModel(data=data, groupdq=groupdq)
dm_ramp.meta.instrument.name = 'MIRI'
dm_ramp.meta.instrument.detector = 'MIRIMAGE'
# run the first frame correction step
dm_ramp_firstframe = FirstFrameStep.call(dm_ramp)
# check that the difference in the groupdq flags is equal to
# DO_NOT_USE flag
dq_diff = dm_ramp_firstframe.groupdq[0, 0, :, :] - dm_ramp.groupdq[0,
0, :, :]
np.testing.assert_array_equal(np.full((ysize, xsize),
dqflags.group['DO_NOT_USE'],
dtype=int),
dq_diff,
err_msg='Diff in groupdq flags is not ' +
'equal to DO_NOT_USE')
def setup_small_cube(ngroups=10, nints=1, nrows=2, ncols=2, deltatime=10.,
gain=1., readnoise=10.):
"""
Create input MIRI datacube having the specified dimensions
"""
gain = np.ones(shape=(nrows, ncols), dtype=np.float64) * gain
err = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float64)
data = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float64)
pixdq = np.zeros(shape=(nrows, ncols), dtype=np.int32)
rnoise = np.full((nrows, ncols), readnoise, dtype=np.float64)
gdq = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.uint8)
model1 = RampModel(data=data, err=err, pixeldq=pixdq, groupdq=gdq)
model1.meta.instrument.name = 'MIRI'
model1.meta.instrument.detector = 'MIRIMAGE'
model1.meta.instrument.filter = 'F480M'
model1.meta.observation.date = '2015-10-13'
model1.meta.exposure.type = 'MIR_IMAGE'
model1.meta.exposure.group_time = deltatime
model1.meta.subarray.name = 'FULL'
model1.meta.subarray.xstart = 1
model1.meta.subarray.ystart = 1
model1.meta.subarray.xsize = ncols
model1.meta.subarray.ysize = nrows
model1.meta.exposure.drop_frames1 = 0
model1.meta.exposure.frame_time = deltatime
model1.meta.exposure.ngroups = ngroups
model1.meta.exposure.group_time = deltatime
model1.meta.exposure.nframes = 1
model1.meta.exposure.groupgap = 0
return model1, gdq, rnoise, pixdq, err, gain
def test_lin_subarray():
"""Test that the pipeline properly extracts the subarray from the reference file.
put dq flags in specific pixels and make sure they match in the output subarray file"""
# create input data
# create model of data with 0 value array
ngroups = 50
ysize = 224
xsize = 288
# create a JWST datamodel for MIRI data
im = RampModel((1, ngroups, ysize, xsize))
im.data += 1
im.meta.instrument.name = 'MIRI'
im.meta.instrument.detector = 'MIRIMAGE'
im.meta.instrument.filter = 'F1500W'
im.meta.instrument.band = 'N/A'
im.meta.observation.date = '2016-06-01'
im.meta.observation.time = '00:00:00'
im.meta.exposure.type = 'MIR_IMAGE'
im.meta.subarray.name = 'MASK1550'
im.meta.subarray.xstart = 1
im.meta.subarray.xsize = xsize
im.meta.subarray.ystart = 467
im.meta.subarray.ysize = ysize
# Read in reference file
dq = np.zeros((1024, 1032), dtype=int)
numcoeffs = 3
# place dq flags in dq array that would be in subarray
# MASK1550 file has colstart=1, rowstart=467
dq[542, 100:105] = 1
ref_model = LinearityModel((numcoeffs, 1024, 1032))
ref_model.dq = dq
ref_model.meta.instrument.name = 'MIRI'
ref_model.meta.instrument.detector = 'MIRIMAGE'
ref_model.meta.description = "MIRI LINEARITY Correction"
ref_model.meta.reftype = "LINEARITY"
ref_model.meta.author = "Monty Pytest"
ref_model.meta.pedigree = "GROUND"
ref_model.meta.useafter = '2015-08-01T00:00:00'
ref_model.meta.subarray.xstart = 1
ref_model.meta.subarray.xsize = 1032
ref_model.meta.subarray.ystart = 1
ref_model.meta.subarray.ysize = 1024
# run through pipeline
outfile = lincorr(im, ref_model)
# read dq array
outpixdq = outfile.pixeldq
# check for dq flag in pixeldq of subarray image
assert (outpixdq[76, 100] == 1)
assert (outpixdq[76, 104] == 1)
def test_nircam():
# test that the code skips processing for a NIR instrument
# size of integration
ngroups = 3
xsize = 2048
ysize = 2048
# create the data and groupdq arrays
csize = (1, ngroups, ysize, xsize)
data = np.full(csize, 1.0)
groupdq = np.zeros(csize, dtype=int)
# create a JWST datamodel for data
dm_ramp = RampModel(data=data, groupdq=groupdq)
dm_ramp.meta.instrument.name = 'NIRCAM'
dm_ramp.meta.instrument.detector = 'NRCA1'
# run the first frame correction step
dm_ramp_firstframe = FirstFrameStep.call(dm_ramp)
# check that the difference in the groupdq flags is equal to
# the 0 since the step should not run for NIR data
dq_diff = dm_ramp_firstframe.groupdq[0, 0, :, :] - dm_ramp.groupdq[0,
0, :, :]
np.testing.assert_array_equal(np.full((ysize, xsize), 0, dtype=int),
dq_diff,
err_msg='Diff in groupdq flags is not ' +
'equal to 0')
def test_err_array():
"""Test that the error array is not changed by the linearity step"""
# size of integration
ngroups = 10
xsize = 1032
ysize = 1024
# create a JWST datamodel for MIRI data
im = RampModel((1, ngroups, ysize, xsize))
im.data += 1
im.err += 2
# set file header values
im.meta.instrument.detector = 'MIRIMAGE'
im.meta.instrument.name = 'MIRI'
im.meta.observation.date = '2018-01-01'
im.meta.observation.time = '00:00:00'
im.meta.subarray.xstart = 1
im.meta.subarray.xsize = xsize
im.meta.subarray.ystart = 1
im.meta.subarray.ysize = ysize
# run pipeline
outfile = LinearityStep.call(im)
# check output of error array
# test that the science data are not changed
np.testing.assert_allclose(im.err, outfile.err)
def test_rscd_baseline_too_few_groups():
"""
Test that the baseline algorithm is skipped if too few groups are present
in the integrations.
"""
# size of exposure
nints = 2
ngroups = 3
xsize = 10
ysize = 10
# create the data and groupdq arrays
csize = (nints, ngroups, ysize, xsize)
data = np.full(csize, 1.0, dtype=np.float32)
groupdq = np.zeros(csize, dtype=np.uint8)
# create a JWST datamodel for MIRI data
dm_ramp = RampModel(data=data, groupdq=groupdq)
# get the number of groups to flag
nflag = 3
# run the RSCD baseline correction step
dm_ramp_rscd = correction_skip_groups(dm_ramp, nflag)
# test that the groupdq flags are not changed for any integration
dq_diff = (dm_ramp_rscd.groupdq[:, :, :, :] - dm_ramp.groupdq[:, :, :, :])
np.testing.assert_array_equal(np.full((nints, ngroups, ysize, xsize),
0,
dtype=int),
dq_diff,
err_msg='groupdq flags changed when ' +
'not enough groups are present')
def insert_into_datamodel(self, subfile):
#read in a dummy/substitute file as a datamodel,
#and insert the data and self.header metadata
#into it
h = RampModel(subfile)
h.data = self.data
try:
h.zeroframe = self.zeroframe
except:
pass
h.err = np.zeros_like(self.data)
h.groupdq = np.zeros_like(self.data)
nint, ng, ny, nx = self.data.shape
h.pixeldq = np.zeros((ny, nx))
h.meta.exposure.readpatt = self.header['READPATT']
h.meta.exposure.nints = self.header['NINTS']
h.meta.exposure.ngroups = self.header['NGROUPS']
h.meta.exposure.nframes = self.header['NFRAMES']
h.meta.exposure.nskip = self.header['NSKIP']
h.meta.exposure.groupgap = self.header['GROUPGAP']
h.meta.exposure.type = self.header['EXP_TYPE']
h.meta.instrument.detector = self.header['DETECTOR']
h.meta.instrument.name = self.header['INSTRUME']
h.meta.subarray.fastaxis = self.header['FASTAXIS']
h.meta.subarray.slowaxis = self.header['SLOWAXIS']
return h
def create_mod_arrays(ngroups, nints, nrows, ncols, deltatime, gain,
readnoise):
"""
For an input datacube (arbitrarily chosen to be MIRI), create arrays having
the specified dimensions for the pixel DQ, the group DQ, and the
ERR extensions, and create datamodels for the ramp, readnoise, and gain.
"""
gain = np.ones(shape=(nrows, ncols), dtype=np.float32) * gain
err = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float32)
data = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float32)
pixdq = np.zeros(shape=(nrows, ncols), dtype=np.uint32)
read_noise = np.full((nrows, ncols), readnoise, dtype=np.float32)
gdq = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.uint8)
# Create and populate ramp model
ramp_model = RampModel(data=data, err=err, pixeldq=pixdq, groupdq=gdq)
ramp_model.meta.instrument.name = 'MIRI'
ramp_model.meta.instrument.detector = 'MIRIMAGE'
ramp_model.meta.instrument.filter = 'F480M'
ramp_model.meta.observation.date = '2015-10-13'
ramp_model.meta.exposure.type = 'MIR_IMAGE'
ramp_model.meta.exposure.group_time = deltatime
ramp_model.meta.subarray.name = 'FULL'
ramp_model.meta.subarray.xstart = 1
ramp_model.meta.subarray.ystart = 1
ramp_model.meta.subarray.xsize = ncols
ramp_model.meta.subarray.ysize = nrows
ramp_model.meta.exposure.frame_time = deltatime
ramp_model.meta.exposure.ngroups = ngroups
ramp_model.meta.exposure.group_time = deltatime
ramp_model.meta.exposure.nframes = 1
ramp_model.meta.exposure.groupgap = 0
ramp_model.meta.exposure.drop_frames1 = 0
return ramp_model, read_noise, gain, pixdq, gdq, err
# Create and populate gain model
gain_model = GainModel(data=gain)
gain_model.meta.instrument.name = 'MIRI'
gain_model.meta.subarray.xstart = 1
gain_model.meta.subarray.ystart = 1
gain_model.meta.subarray.xsize = ncols
gain_model.meta.subarray.ysize = nrows
# Create and populate readnoise model
rnoise_model = ReadnoiseModel(data=read_noise)
rnoise_model.meta.instrument.name = 'MIRI'
rnoise_model.meta.subarray.xstart = 1
rnoise_model.meta.subarray.ystart = 1
rnoise_model.meta.subarray.xsize = ncols
rnoise_model.meta.subarray.ysize = nrows
return ramp_model, rnoise_model, gain_model, pixdq, gdq, err
def test_rscd_baseline_set_groupdq():
"""
For a 2 integration exposure, test if the rscd code sets the groupdq flag on
the n groups to 'do_not_use' for the 2nd integration and did not change the
groupdq flags in the 1st integration
"""
# size of integration
ngroups = 10
xsize = 10
ysize = 10
# create the data and groupdq arrays
csize = (2, ngroups, ysize, xsize)
data = np.full(csize, 1.0, dtype=np.float32)
groupdq = np.zeros(csize, dtype=np.uint8)
# create a JWST datamodel for MIRI data
dm_ramp = RampModel(data=data, groupdq=groupdq)
# get the number of groups to flag
nflag = 3
# run the RSCD baseline correction step
dm_ramp_rscd = correction_skip_groups(dm_ramp, nflag)
# check that the difference in the groupdq flags is equal to
# the 'do_not_use' flag for the 2nd integration
dq_diff = (dm_ramp_rscd.groupdq[1, 0:nflag, :, :] -
dm_ramp.groupdq[1, 0:nflag, :, :])
np.testing.assert_array_equal(np.full((nflag, ysize, xsize),
dqflags.group['DO_NOT_USE'],
dtype=int),
dq_diff,
err_msg='Diff in groupdq flags is not ' +
'equal to the DO_NOT_USE flag')
# test that the groupdq flags are not changed for the rest of the groups
# in the 2nd integration
dq_diff = (dm_ramp_rscd.groupdq[1, nflag:ngroups, :, :] -
dm_ramp.groupdq[1, nflag:ngroups, :, :])
np.testing.assert_array_equal(np.full((ngroups - nflag, ysize, xsize),
0,
dtype=int),
dq_diff,
err_msg='groupdq flags changed after ' +
'maximum requested')
# test that the groupdq flags are not changed for the 1st integration
dq_diff = (dm_ramp_rscd.groupdq[0, :, :, :] - dm_ramp.groupdq[0, :, :, :])
np.testing.assert_array_equal(np.full((ngroups, ysize, xsize),
0,
dtype=int),
dq_diff,
err_msg='groupdq flags changed in 1st ' +
'integration this should not happen')
def _setup(ngroups=10,
readnoise=10,
nints=1,
nrows=1024,
ncols=1032,
nframes=1,
grouptime=1.0,
gain=1,
deltatime=1):
times = np.array(list(range(ngroups)), dtype=np.float64) * deltatime
gain = np.ones(shape=(nrows, ncols), dtype=np.float64) * gain
err = np.ones(shape=(nints, ngroups, nrows, ncols), dtype=np.float64)
data = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float64)
pixdq = np.zeros(shape=(nrows, ncols), dtype=np.uint32)
read_noise = np.full((nrows, ncols), readnoise, dtype=np.float64)
gdq = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.uint32)
rampmodel = RampModel(data=data,
err=err,
pixeldq=pixdq,
groupdq=gdq,
times=times)
rampmodel.meta.instrument.name = 'MIRI'
rampmodel.meta.instrument.detector = 'MIRIMAGE'
rampmodel.meta.instrument.filter = 'F480M'
rampmodel.meta.observation.date = '2015-10-13'
rampmodel.meta.exposure.type = 'MIR_IMAGE'
rampmodel.meta.exposure.group_time = deltatime
rampmodel.meta.subarray.name = 'FULL'
rampmodel.meta.subarray.xstart = 1
rampmodel.meta.subarray.ystart = 1
rampmodel.meta.subarray.xsize = ncols
rampmodel.meta.subarray.ysize = nrows
rampmodel.meta.exposure.frame_time = deltatime
rampmodel.meta.exposure.ngroups = ngroups
rampmodel.meta.exposure.group_time = deltatime
rampmodel.meta.exposure.nframes = 1
rampmodel.meta.exposure.groupgap = 0
gain = GainModel(data=gain)
gain.meta.instrument.name = 'MIRI'
gain.meta.subarray.xstart = 1
gain.meta.subarray.ystart = 1
gain.meta.subarray.xsize = ncols
gain.meta.subarray.ysize = nrows
rnmodel = ReadnoiseModel(data=read_noise)
rnmodel.meta.instrument.name = 'MIRI'
rnmodel.meta.subarray.xstart = 1
rnmodel.meta.subarray.ystart = 1
rnmodel.meta.subarray.xsize = ncols
rnmodel.meta.subarray.ysize = nrows
return rampmodel, gdq, rnmodel, pixdq, err, gain
def _cube():
# create a JWST datamodel for NIRSPEC IRS2 data
data_model = RampModel((1, 5, 3200, 2048))
data_model.data = np.ones(((1, 5, 3200, 2048)))
data_model.groupdq = np.zeros(((1, 5, 3200, 2048)))
data_model.pixeldq = np.zeros(((3200, 2048)))
data_model.meta.instrument.name = 'NIRSPEC'
data_model.meta.instrument.detector = 'NRS1'
data_model.meta.instrument.filter = 'F100LP'
data_model.meta.observation.date = '2019-07-19'
data_model.meta.observation.time = '23:23:30.912'
data_model.meta.exposure.type = 'NRS_LAMP'
data_model.meta.subarray.name = 'FULL'
data_model.meta.subarray.xstart = 1
data_model.meta.subarray.xsize = 2048
data_model.meta.subarray.ystart = 1
data_model.meta.subarray.ysize = 2048
data_model.meta.exposure.nrs_normal = 16
data_model.meta.exposure.nrs_reference = 4
data_model.meta.exposure.readpatt = 'NRSIRS2RAPID'
# create a saturation model for the saturation step
saturation_model = SaturationModel((2048, 2048))
saturation_model.data = np.ones(
(2048, 2048)) * 60000 # saturation limit for every pixel is 60000
saturation_model.meta.description = 'Fake data.'
saturation_model.meta.telescope = 'JWST'
saturation_model.meta.reftype = 'SaturationModel'
saturation_model.meta.useafter = '2015-10-01T00:00:00'
saturation_model.meta.instrument.name = 'NIRSPEC'
saturation_model.meta.instrument.detector = 'NRS1'
saturation_model.meta.author = 'Clare'
saturation_model.meta.pedigree = 'Dummy'
saturation_model.meta.subarray.xstart = 1
saturation_model.meta.subarray.xsize = 2048
saturation_model.meta.subarray.ystart = 1
saturation_model.meta.subarray.ysize = 2048
return data_model, saturation_model
def make_rawramp(nints, ngroups, ysize, xsize):
# create the data and groupdq arrays
csize = (nints, ngroups, ysize, xsize)
data = np.full(csize, 1.0)
# create a JWST datamodel for MIRI data
dm_ramp = RampModel(data=data)
dm_ramp.meta.subarray.xstart = 1
dm_ramp.meta.subarray.xsize = xsize
dm_ramp.meta.subarray.ystart = 1
dm_ramp.meta.subarray.ysize = ysize
return dm_ramp
def _setup(ngroups=10,
readnoise=10,
nints=1,
nrows=102,
ncols=103,
nframes=1,
grouptime=1.0,
gain=1,
deltatime=1,
subarray=False):
if subarray:
shape = (nints, ngroups, 20, 20)
else:
shape = (nints, ngroups, nrows, ncols)
model = RampModel(shape)
model.meta.instrument.name = 'MIRI'
model.meta.instrument.detector = 'MIRIMAGE'
model.meta.instrument.filter = 'F480M'
model.meta.observation.date = '2015-10-13'
model.meta.exposure.type = 'MIR_IMAGE'
model.meta.exposure.group_time = deltatime
model.meta.subarray.name = 'FULL'
model.meta.subarray.xstart = 1
model.meta.subarray.ystart = 1
model.meta.subarray.xsize = shape[3]
model.meta.subarray.ysize = shape[2]
model.meta.exposure.frame_time = deltatime
model.meta.exposure.ngroups = ngroups
model.meta.exposure.group_time = deltatime
model.meta.exposure.nframes = nframes
model.meta.exposure.groupgap = 0
gain_model = GainModel((nrows, ncols))
gain_model.data += gain
gain_model.meta.instrument.name = 'MIRI'
gain_model.meta.subarray.xstart = 1
gain_model.meta.subarray.ystart = 1
gain_model.meta.subarray.xsize = ncols
gain_model.meta.subarray.ysize = nrows
read_noise_model = ReadnoiseModel((nrows, ncols))
read_noise_model.data += readnoise
read_noise_model.meta.instrument.name = 'MIRI'
read_noise_model.meta.subarray.xstart = 1
read_noise_model.meta.subarray.ystart = 1
read_noise_model.meta.subarray.xsize = ncols
read_noise_model.meta.subarray.ysize = nrows
return model, read_noise_model, gain_model
def read_datamodel(self):
logger = logging.getLogger('mirage.utils.read_fits.read_datamodel')
h = RampModel(self.file)
#remove any non-pipeline related keywords (e.g. CV3 temps/voltages)
if 'extra_fits' in dir(h):
h.__delattr__('extra_fits')
self.data = h.data
#Currently a bug in level1bmodel when zeroframe is
#not present and a cube of zeros is returned
#If the datamodel returns a default zeroframe of all
#zeros, then set it to None here
#self.zeroframe = h.zeroframe
if np.all(h.zeroframe == 0):
if 'RAPID' in h.meta.exposure.readpatt:
logger.info((
f"Zeroframe in {os.path.basename(self.file)} is all zeros. Since the readpattern is RAPID, "
"we grab a copy of the first group to be the zeroframe."))
self.zeroframe = h.data[:, 0, :, :]
else:
logger.info(
"Zeroframe in {} is all zeros. Returning None.".format(
self.file))
self.zeroframe = None
else:
self.zeroframe = h.zeroframe
self.sbAndRefpix = None
self.header = {}
for key in self.translate:
try:
self.header[key] = eval(f'h.meta.{self.translate[key]}')
except:
self.header[key] = None
def make_rampmodel(nints, ngroups, ysize, xsize):
"""Function to provide ramp model to tests"""
dm_ramp = RampModel((nints, ngroups, ysize, xsize))
dm_ramp.data += 1
dm_ramp.meta.instrument.name = 'MIRI'
dm_ramp.meta.observation.date = '2018-01-01'
dm_ramp.meta.observation.time = '00:00:00'
dm_ramp.meta.subarray.xstart = 1
dm_ramp.meta.subarray.xsize = xsize
dm_ramp.meta.subarray.ystart = 1
dm_ramp.meta.subarray.ysize = ysize
return dm_ramp
def _dm(ngroups, ysize, xsize):
# create the data and groupdq arrays
nints = 2
csize = (nints, ngroups, ysize, xsize)
data = np.random.randint(low=1, high=50, size=csize)
err = np.random.randint(low=0.1, high=5.0, size=csize)
# create a JWST datamodel for NIRSPEC data
dm = RampModel(data=data, err=err)
dm.meta.instrument.name = 'NIRSPEC'
dm.meta.date = '2018-01-01'
dm.meta.instrument.detector= 'NRS1'
dm.meta.observation.date = '2018-01-01'
dm.meta.exposure.gain_factor = 2
return dm
