def test_linearity_step(fits_input):
"""Make sure the LinearityStep runs without error."""
fname = fits_input[0].header['filename'].replace('.fits',
'_linearitystep.fits')
LinearityStep.call(datamodels.open(fits_input),
output_file=fname,
save_results=True)
def test_err_array():
"""Test that the error array is not changed by the linearity step"""
# size of integration
ngroups = 10
xsize = 1032
ysize = 1024
# create a JWST datamodel for MIRI data
im = RampModel((1, ngroups, ysize, xsize))
im.data += 1
im.err += 2
# set file header values
im.meta.instrument.detector = 'MIRIMAGE'
im.meta.instrument.name = 'MIRI'
im.meta.observation.date = '2018-01-01'
im.meta.observation.time = '00:00:00'
im.meta.subarray.xstart = 1
im.meta.subarray.xsize = xsize
im.meta.subarray.ystart = 1
im.meta.subarray.ysize = ysize
# run pipeline
outfile = LinearityStep.call(im)
# check output of error array
# test that the science data are not changed
np.testing.assert_allclose(im.err, outfile.err)
def test_err_array():
"""Test that the error array is not changed by the linearity step"""
# size of integration
ngroups = 10
xsize = 1032
ysize = 1024
# create a JWST datamodel for MIRI data
im = MIRIRampModel((1, ngroups, ysize, xsize))
im.data += 1
im.err += 2
# set file header values
im.meta.instrument.detector = 'MIRIMAGE'
im.meta.instrument.name = 'MIRI'
im.meta.observation.date = '2018-01-01'
im.meta.observation.time = '00:00:00'
im.meta.subarray.xstart = 1
im.meta.subarray.xsize = xsize
im.meta.subarray.ystart = 1
im.meta.subarray.ysize = ysize
# run pipeline
outfile = LinearityStep.call(im)
# check output of error array
# test that the science data are not changed
np.testing.assert_allclose(im.err, outfile.err)
def run_jump_step(self, infile, threshold, run_steps):
'''Function to run the jump detection step.'''
# output file name
out = infile[:-5] + "_jump_CRthresh" + str(threshold) + ".fits"
# if run_steps, run all steps prior to jump
if run_steps:
m = DQInitStep.call(infile)
m = SaturationStep.call(m)
m = SuperBiasStep.call(m)
m_ref = RefPixStep.call(m, config_file='refpix.cfg')
m_lin = LinearityStep.call(m)
m_dark = DarkCurrentStep.call(m)
# if threshold is given, use that rejection threshold
if threshold is not None:
m = JumpStep.call(m,
output_file=out,
rejection_threshold=threshold)
else:
m = JumpStep.call(m, output_file=out)
# else, run only jump_step
else:
if threshold is not None:
m = JumpStep.call(infile,
output_file=out,
rejection_threshold=threshold)
else:
m = JumpStep.call(infile, output_file=out)
return m
def test_err_array():
"""test that the error array is not changed by the linearity step"""
# size of integration
ngroups = 20
xsize = 1032
ysize = 1024
# create the data and groupdq arrays
csize = (1, ngroups, ysize, xsize)
data = np.full(csize, 1.0)
pixeldq = np.zeros((ysize, xsize), dtype=int)
groupdq = np.zeros(csize, dtype=int)
err = np.full(csize, 2.0)
# create a JWST datamodel for MIRI data
im = MIRIRampModel(data=data, pixeldq=pixeldq, groupdq=groupdq, err=err)
# set file header values
im.meta.instrument.detector = 'MIRIMAGE'
im.meta.instrument.name = 'MIRI'
im.meta.observation.date = '2018-01-01'
im.meta.observation.time = '00:00:00'
im.meta.subarray.xstart = 1
im.meta.subarray.xsize = xsize
im.meta.subarray.ystart = 1
im.meta.subarray.ysize = ysize
# run pipeline
outfile = LinearityStep.call(im)
# check output of error array
# test that the science data are not changed
np.testing.assert_array_equal(np.full(csize, 2.0, dtype=float),
outfile.err, err_msg='no changes should be seen in array ')
def test_linearity_miri3(self):
"""
Regression test of linearity step performed on MIRI data.
"""
input_file = self.get_data(self.test_dir,
'jw00001001001_01109_00001_MIRIMAGE_dark_current.fits')
# get supplemental input
override_file = self.get_data(self.test_dir,
"lin_nan_flag_miri.fits")
# run calibration step
result = LinearityStep.call(input_file,
override_linearity=override_file)
output_file = result.meta.filename
result.save(output_file)
result.close()
outputs = [(output_file,
'jw00001001001_01109_00001_MIRIMAGE_linearity.fits') ]
self.compare_outputs(outputs)
def test_linearity_miri3(self):
"""
Regression test of linearity step performed on MIRI data.
"""
input_file = self.get_data(
self.test_dir,
'jw00001001001_01109_00001_MIRIMAGE_dark_current.fits')
# get supplemental input
override_file = self.get_data(self.test_dir, "lin_nan_flag_miri.fits")
# run calibration step
result = LinearityStep.call(input_file,
override_linearity=override_file)
output_file = result.meta.filename
result.save(output_file)
result.close()
outputs = [(output_file,
'jw00001001001_01109_00001_MIRIMAGE_linearity.fits')]
self.compare_outputs(outputs)
def test_saturation():
"""Check that correction is not applied for groups flagged as SATURATED in GROUPDQ"""
# size of integration
nints = 1
ngroups = 16
xsize = 1032
ysize = 1024
# create a JWST datamodel for MIRI data
im = make_rampmodel(nints, ngroups, ysize, xsize)
im.meta.instrument.detector = 'MIRIMAGE'
# set groupdq pixels to saturated
im.groupdq[0, 10:, 200, 150] = dqflags.pixel['SATURATED'] # saturated dq flag
im.data[0, 15, 200, 150] = 1000.0 # value shouldn't change
# run through Linearity pipeline
outfile = LinearityStep.call(im)
assert(outfile.data[0, 15, 200, 150] == 1000.0) # pixel flagged as saturated, shouldn't change
def test_saturation():
"""Check that correction is not applied for groups flagged as SATURATED in GROUPDQ"""
# size of integration
nints = 1
ngroups = 160
xsize = 1032
ysize = 1024
# create a JWST datamodel for MIRI data
im = make_rampmodel(nints, ngroups, ysize, xsize)
im.meta.instrument.detector = 'MIRIMAGE'
# set groupdq pixels to saturated
im.groupdq[0, 100:, 200, 150] = dqflags.pixel['SATURATED'] # saturated dq flag
im.data[0, 150, 200, 150] = 1000.0 # value shouldn't change
# run through Linearity pipeline
outfile = LinearityStep.call(im)
assert(outfile.data[0, 150, 200, 150] == 1000.0) # pixel flagged as saturated, shouldn't change
if len(line) > 3:
files.append(line.strip())
#assume they've been run thru bpm and saturation
#so now do superbias subtraction, refpix corr,
#linearity corr, jump step
#hardwire for A1 at the moment:
reffile_dir = '/ifs/jwst/wit/witserv/data7/nrc/reference_files/SSB/CV3/cv3_reffile_conversion/'
sbfile = '/ifs/jwst/wit/witserv/data4/nrc/hilbert/superbias/cv3/A1/A1_superbias_from_list_of_biasfiles.list.fits'
linfile = reffile_dir + 'linearity/NRCA1_17004_LinearityCoeff_ADU0_2016-05-14_ssblinearity_DMSorient.fits'
gainfile = reffile_dir + 'gain/NRCA1_17004_Gain_ISIMCV3_2016-01-23_ssbgain_DMSorient.fits'
ronfile = '/grp/jwst/wit/nircam/reference_files/SSB/CV2/delivery_Dec_2015/Read_Noise/NRCA1_16989_CDSNoise_2014-10-24_ssbreadnoise_DMSorient.fits'
for file in files:
sbout = file[0:-5] + '_superbias.fits'
data = SuperBiasStep.call(file,
override_superbias=sbfile,
output_file=sbout)
refout = sbout[0:-5] + '_refpix.fits'
data = RefPixStep.call(data, output_file=refout)
linout = refout[0:-5] + '_linearity.fits'
data = LinearityStep.call(data,
override_linearity=linfile,
output_file=linout)
jumpout = linout[0:-5] + '_jump.fits'
data = JumpStep.call(data,
override_gain=gainfile,
override_readnoise=ronfile,
output_file=jumpout)
def linearize_dark(self, darkobj):
"""Beginning with the input dark current ramp, run the dq_init, saturation, superbias
subtraction, refpix and nonlin pipeline steps in order to produce a linearized
version of the ramp. This will be used when combining the dark ramp with the
simulated signal ramp.
Parameters
-----------
darkobj : obj
Instance of read_fits class containing dark current data and info
Returns
-------
linDarkObj : obj
Modified read_fits instance with linearized dark current data
"""
from jwst.dq_init import DQInitStep
from jwst.saturation import SaturationStep
from jwst.superbias import SuperBiasStep
from jwst.refpix import RefPixStep
from jwst.linearity import LinearityStep
# First we need to place the read_fits object into a RampModel instance
if self.runStep['linearized_darkfile']:
subfile = self.params['Reffiles']['linearized_darkfile']
else:
subfile = self.params['Reffiles']['dark']
dark = darkobj.insert_into_datamodel(subfile)
print('Creating a linearized version of the dark current input ramp')
print('using JWST calibration pipeline.')
# Run the DQ_Init step
if self.runStep['badpixmask']:
linDark = DQInitStep.call(dark,
config_file=self.params['newRamp']['dq_configfile'],
override_mask=self.params['Reffiles']['badpixmask'])
else:
linDark = DQInitStep.call(dark, config_file=self.params['newRamp']['dq_configfile'])
# If the saturation map is provided, use it. If not, default to whatever is in CRDS
if self.runStep['saturation_lin_limit']:
linDark = SaturationStep.call(linDark,
config_file=self.params['newRamp']['sat_configfile'],
override_saturation=self.params['Reffiles']['saturation'])
else:
linDark = SaturationStep.call(linDark,
config_file=self.params['newRamp']['sat_configfile'])
# If the superbias file is provided, use it. If not, default to whatever is in CRDS
if self.runStep['superbias']:
linDark = SuperBiasStep.call(linDark,
config_file=self.params['newRamp']['superbias_configfile'],
override_superbias=self.params['Reffiles']['superbias'])
else:
linDark = SuperBiasStep.call(linDark,
config_file=self.params['newRamp']['superbias_configfile'])
# Reference pixel correction
linDark = RefPixStep.call(linDark,
config_file=self.params['newRamp']['refpix_configfile'])
# Save a copy of the superbias- and reference pixel-subtracted
# dark. This will be used later to add these effects back in
# after the synthetic signals have been added and the non-linearity
# effects are added back in when using the PROPER combine method.
sbAndRefpixEffects = dark.data - linDark.data
# Linearity correction - save the output so that you won't need to
# re-run the pipeline when using the same dark current file in the
# future. Use the linearity coefficient file if provided
base_name = self.params['Output']['file'].split('/')[-1]
linearoutfile = base_name[0:-5] + '_linearized_dark_current_ramp.fits'
linearoutfile = os.path.join(self.params['Output']['directory'], linearoutfile)
if self.runStep['linearity']:
linDark = LinearityStep.call(linDark,
config_file=self.params['newRamp']['linear_configfile'],
override_linearity=self.params['Reffiles']['linearity'],
output_file=linearoutfile)
else:
linDark = LinearityStep.call(linDark,
config_file=self.params['newRamp']['linear_configfile'],
output_file=linearoutfile)
print(("Linearized dark (output directly from pipeline saved as {}"
.format(linearoutfile)))
# Now we need to put the data back into a read_fits object
linDarkobj = read_fits.Read_fits()
linDarkobj.model = linDark
linDarkobj.rampmodel_to_obj()
linDarkobj.sbAndRefpix = sbAndRefpixEffects
return linDarkobj
def test_linearity_step(fits_input):
"""Make sure the LinearityStep runs without error."""
LinearityStep.call(datamodels.open(fits_input), save_results=True)
det = det.lower()[3:]
if 'long' in det:
det = det[0] + '5'
refdict = reffiles[det]
#m = calwebb_detector1.Detector1Pipeline(config_file='calwebb_detector1.cfg')
#m.saturation.override_saturation = satdir+refdict['saturation']
#m.superbias.override_superbias = sbdir+refdict['superbias']
#m.refpix.odd_even_rows = False
#m.group_scale.skip = True
#m.ipc.skip = True
#m.rscd.skip = True
#m.lastframe.skip = True
#m.dark_current.skip = True
#m.persistence.skip = True
#m.jump.skip = True
#m.ramp_fit.skip = False #bug in pipeline means this must
#be run.
#m.linearity.override_linearity = lindir+refdict['linearity']
#m.output_file = outfile
#m.run(file)
m = DQInitStep.call(file,config_file = 'dq_init.cfg')
m = SaturationStep.call(m,config_file = 'saturation.cfg')
m = SuperBiasStep.call(m,config_file = 'superbias.cfg')
m = RefPixStep.call(m,config_file='refpix.cfg')
m = LinearityStep.call(m,config_file='linearity.cfg',output_file = outfile)
