def test_ramp_model_zero_frame_open_file(tmpdir):
"""
Ensures opening a FITS with ZEROFRAME results in a good ZEROFRAME.
"""
nints, ngroups, nrows, ncols = 2, 10, 5, 5
dims = nints, ngroups, nrows, ncols
zdims = (nints, nrows, ncols)
dbase = 1000.
zbase = dbase * .75
data = np.ones(dims, dtype=float) * dbase
err = np.zeros(dims, dtype=float)
pdq = np.zeros(dims[-2:], dtype=np.uint32)
gdq = np.zeros(dims, dtype=np.uint8)
# Test default exposure zero_frame
ramp = datamodels.RampModel(data=data, err=err, pixeldq=pdq, groupdq=gdq)
ramp.meta.exposure.zero_frame = True
ramp.zeroframe = np.ones(zdims, dtype=ramp.data.dtype) * zbase
ofile = "my_temp_ramp.fits"
fname = os.path.join(tmpdir, ofile)
ramp.save(fname)
# Check opening a file doesn't change the dimensions
with datamodels.RampModel(fname) as ramp1:
assert ramp1.zeroframe.shape == ramp.zeroframe.shape
zframe0 = ramp.zeroframe
zframe1 = ramp1.zeroframe
np.testing.assert_allclose(zframe0, zframe1, 1.e-5)
def test_ramp_model_zero_frame_by_dimensions():
"""
Ensures creating a RampModel by dimensions results in the correct
dimensions for ZEROFRAME.
"""
nints, ngroups, nrows, ncols = 2, 10, 5, 5
dims = (nints, ngroups, nrows, ncols)
zdims = (nints, nrows, ncols)
with datamodels.RampModel(dims) as ramp:
assert ramp.zeroframe.shape == zdims
def test_mirirampmodel_deprecation(_jail):
"""Test that a deprecated MIRIRampModel can be opened"""
# Create a MIRIRampModel, working around the deprecation.
model = datamodels.RampModel((1, 1, 10, 10))
model.save('ramp.fits')
hduls = fits.open('ramp.fits', mode='update')
hduls[0].header['datamodl'] = 'MIRIRampModel'
hduls.close()
# Test it.
miri_ramp = datamodels.open('ramp.fits')
assert isinstance(miri_ramp, datamodels.RampModel)
def test_mirirampmodel_deprecation(tmp_path):
"""Test that a deprecated MIRIRampModel can be opened"""
path = str(tmp_path / "ramp.fits")
# Create a MIRIRampModel, working around the deprecation.
model = datamodels.RampModel((1, 1, 10, 10))
model.save(path)
hduls = fits.open(path, mode='update')
hduls[0].header['datamodl'] = 'MIRIRampModel'
hduls.close()
# Test it.
with pytest.warns(DeprecationWarning):
miri_ramp = datamodels.open(path)
assert isinstance(miri_ramp, datamodels.RampModel)
def process(self, input):
with datamodels.RampModel(input) as input_model:
tstart = time.time()
# Check for consistency between keyword values and data shape
ngroups = input_model.data.shape[1]
ngroups_kwd = input_model.meta.exposure.ngroups
if ngroups != ngroups_kwd:
self.log.error("Keyword 'NGROUPS' value of '{0}' does not match data array size of '{1}'".format(ngroups_kwd,ngroups))
raise ValueError("Bad data dimensions")
# Check for an input model with NGROUPS<=6.
if ngroups <= 6:
self.log.warn('Will not apply column jump detection when NGROUPS<=6;')
self.log.warn('Column jump step will be skipped')
result = input_model.copy()
result.meta.cal_step.jump = 'SKIPPED'
return result
# Retrieve the parameter values
nsigma1jump = self.nsigma1jump
nsigma2jumps = self.nsigma2jumps
outputdiagnostics = self.outputdiagnostics
self.log.info('Sigma rejection threshold for one break = %g ', nsigma1jump)
self.log.info('Sigma rejection threshold for two breaks = %g ', nsigma2jumps)
self.log.info('Output Diagnostic files = %s ', outputdiagnostics)
# Call the column jump detection routine
result = detect_columnjumps(input_model, nsigma1jump, nsigma2jumps, outputdiagnostics)
tstop = time.time()
self.log.info('The execution time in seconds: %f', tstop - tstart)
result.meta.cal_step.columnjump = 'COMPLETE'
return result
def image2ssb(self,inputfilename, outfilebasename='auto',outdir=None,outsuffix=None,outsubdir=None):
outfilebasename = self.mkoutfilebasename(inputfilename, outfilebasename=outfilebasename,outdir=outdir,outsuffix=outsuffix,outsubdir=outsubdir)
(self.data,self.hdr)=pyfits.getdata(inputfilename, 0, header=True)
print('input shape:',self.data.shape)
self.runID = self.getRunID(filename=inputfilename)
self.hdr['SUBARRAY'] = 'FULL'
# How many groups and integrations?
if self.runID=='TUCSONNEW':
Nint=1
else:
Nint = int(self.hdr['NINT'])
Ngroup =int(self.hdr['NGROUP'])
Nframe = int(self.hdr['NFRAME'])
print('NGROUP:',Ngroup)
print('NINT:',Nint)
if (Ngroup*Nint)!=self.data.shape[0]:
raise RuntimeError('something is wrong! NGROUP=%d, NINT=%d, sum is not shape[0]=%d' % (Ngroup,Nint,self.data.shape[0]))
# rearrange the data: 4th coordinate is integration
scinew=scipy.zeros((Nint,Ngroup,self.data.shape[1],self.data.shape[2]), dtype=float)
for i in range(Nint):
scinew[i,:,:,:]=self.data[i*Ngroup:(i+1)*Ngroup,:,:]
print('output shape:',scinew.shape)
self.outputmodel = models.RampModel(data=scinew)
#Mask dead pixels
#mask = scipy.where(np.isnan(scinew))
# mask = np.where(np.isnan(a))
#mask = np.isnan(scinew)
# a=scinew[mask]
#print mask
#sys.exit(0)
#updates the date string
self.updatemetadata(inputfilename,reffileflag=False)
print('meta data updated')
#update detector
self.cryo_update_meta_detector(reffileflag=False)
print('cryo meta data updated')
#flip the data around to place it in the science orientation expected by SSB
if self.runID in ['CV2','CV3', 'OTIS']:
self.native_to_science_image_flip()
if self.runID in ['CV3']:
self.outputmodel.meta.exposure.readpatt = self.hdr['READOUT']
if self.runID in ['OTIS']:
self.outputmodel.meta.exposure.readpatt = self.hdr['READOUT']
self.outputmodel.meta.exposure.nints = Nint
self.outputmodel.meta.exposure.nframes = Nframe
#self.outputmodel.meta.exposure.ngroups = self.hdr['NAXIS3']
self.outputmodel.meta.exposure.ngroups = Ngroup
# put the version of nircam2ssb into header
# self.outputmodel.meta.channel = self.version
outfilename = outfilebasename
if not re.search('fits$',outfilename): outfilename += '_uncal.fits'
print('Saving %s' % outfilename)
self.outputmodel.save(outfilename)
return(outfilename)
def process(self, input):
'''Process a Stage 0 *_uncal.fits file to Stage 1 *_rate.fits and *_rateints.fits files.
Steps taken to perform this processing can follow the default JWST pipeline, or alternative methods.
Parameters
----------
input: str, tuple, `~astropy.io.fits.HDUList`, ndarray, dict, None
- None: Create a default data model with no shape.
- tuple: Shape of the data array.
Initialize with empty data array with shape specified by the.
- file path: Initialize from the given file (FITS or ASDF)
- readable file object: Initialize from the given file
object
- `~astropy.io.fits.HDUList`: Initialize from the given
`~astropy.io.fits.HDUList`.
- A numpy array: Used to initialize the data array
Returns
-------
out_model: jwst.datamodels.ImageModel
The output ImageModel to be returned from the ramp fit step.
int_model: jwst.datamodels.CubeModel
The output CubeModel to be returned from the ramp fit step.
Notes
-----
History:
- October 2021 Aarynn Carter and Eva-Maria Ahrer
Initial version
- February 2022 Aarynn Carter and Eva-Maria Ahrer
Updated for JWST version 1.3.3, code restructure
'''
with datamodels.RampModel(input) as input_model:
readnoise_filename = self.get_reference_file(input_model, 'readnoise')
gain_filename = self.get_reference_file(input_model, 'gain')
log.info('Using READNOISE reference file: %s', readnoise_filename)
log.info('Using GAIN reference file: %s', gain_filename)
with datamodels.ReadnoiseModel(readnoise_filename) as readnoise_model, \
datamodels.GainModel(gain_filename) as gain_model:
# Try to retrieve the gain factor from the gain reference file.
# If found, store it in the science model meta data, so that it's
# available later in the gain_scale step, which avoids having to
# load the gain ref file again in that step.
if gain_model.meta.exposure.gain_factor is not None:
input_model.meta.exposure.gain_factor = gain_model.meta.exposure.gain_factor
# Get gain arrays, subarrays if desired.
frames_per_group = input_model.meta.exposure.nframes
readnoise_2d, gain_2d = get_reference_file_subarrays(
input_model, readnoise_model, gain_model, frames_per_group)
log.info('Using algorithm = %s' % self.algorithm)
log.info('Using weighting = %s' % self.weighting)
buffsize = ramp_fit.BUFSIZE
if pipe_utils.is_tso(input_model) and hasattr(input_model, 'int_times'):
input_model.int_times = input_model.int_times
else:
input_model.int_times = None
# DEFAULT RAMP FITTING ALGORITHM
if self.algorithm == 'default':
# In our case, default just means Optimal Least Squares
self.algorithm = 'OLS'
if self.weighting == 'default':
# Want to use the default optimal weighting
pass
elif self.weighting == 'fixed':
# Want to use default weighting, but don't want to
# change exponent between pixels.
if not isinstance(self.fixed_exponent, (int, float)):
raise ValueError('Weighting exponent must be of type "int" or "float" for "default_fixed" weighting')
# Overwrite the exponent calculation function from ols_fit
stcal.ramp_fitting.ols_fit.calc_power = partial(fixed_power, weighting_exponent=self.fixed_exponent) #Pipeline version 1.3.3
elif self.weighting == 'interpolated':
# Want to use an interpolated version of the default weighting.
# Overwrite the exponent calculation function from ols_fit
stcal.ramp_fitting.ols_fit.calc_power = interpolate_power #Pipeline version 1.3.3
elif self.weighting == 'uniform':
# Want each frame and pixel weighted equally
# Overwrite the entire optimal calculation function
stcal.ramp_fitting.ols_fit.calc_opt_sums = calc_opt_sums_uniform_weight #Pipeline version 1.3.3
elif self.weighting == 'custom':
# Want to manually assign snr bounds for exponent changes
# Overwrite the exponent calculation function from ols_fit
stcal.ramp_fitting.ols_fit.calc_power = partial(custom_power, snr_bounds=self.custom_snr_bounds, exponents=self.custom_exponents) #Pipeline version 1.3.3
else:
raise ValueError('Could not interpret weighting "{}".'.format(self.weighting))
# Important! Must set the weighting to 'optimal' for the actual
# ramp_fit() function, previous if statements will have changed
# it's underlying functionality.
self.weighting = 'optimal'
image_info, integ_info, opt_info, gls_opt_model = ramp_fit.ramp_fit(input_model, buffsize, self.save_opt, readnoise_2d,\
gain_2d, self.algorithm, self.weighting,\
self.maximum_cores, dqflags.pixel)
# FUTURE IMPROVEMENT, WFC3-like differenced frames.
elif self.algorithm == 'differenced':
raise ValueError('I do not know how to do differenced frames yet.')
# PRIMARILY FOR TESTING, MEAN OF RAMP
elif self.algorithm == 'mean':
image_info, integ_info, opt_info = mean_ramp_fit_single(input_model, buffsize, self.save_opt, readnoise_2d,\
gain_2d, self.algorithm, self.weighting,\
self.maximum_cores, dqflags.pixel)
else:
raise ValueError('Ramp fitting algorithm "{}" not implemented.'.format(self.algorithm))
if image_info is not None:
out_model = create_image_model(input_model, image_info)
out_model.meta.bunit_data = 'DN/s'
out_model.meta.bunit_err = 'DN/s'
out_model.meta.cal_step.ramp_fit = 'COMPLETE'
if integ_info is not None:
int_model = create_integration_model(input_model, integ_info)
int_model.meta.bunit_data = 'DN/s'
int_model.meta.bunit_err = 'DN/s'
int_model.meta.cal_step.ramp_fit = 'COMPLETE'
return out_model, int_model
def calwebb_detector1_save_jump(input_file,
output_dir,
ramp_fit=True,
save_fitopt=True):
"""Call ``calwebb_detector1`` on the provided file, running all
steps up to the ``ramp_fit`` step, and save the result. Optionally
run the ``ramp_fit`` step and save the resulting slope file as well.
Parameters
----------
input_file : str
Name of fits file to run on the pipeline
output_dir : str
Directory into which the pipeline outputs are saved
ramp_fit : bool
If ``False``, the ``ramp_fit`` step is not run. The output file
will be a ``*_jump.fits`` file.
If ``True``, the ``*jump.fits`` file will be produced and saved.
In addition, the ``ramp_fit`` step will be run and a
``*rate.fits`` or ``*_rateints.fits`` file will be saved.
(``rateints`` if the input file has >1 integration)
save_fitopt : bool
If ``True``, the file of optional outputs from the ramp fitting
step of the pipeline is saved.
Returns
-------
jump_output : str
Name of the saved file containing the output prior to the
``ramp_fit`` step.
pipe_output : str
Name of the saved file containing the output after ramp-fitting
is performed (if requested). Otherwise ``None``.
"""
input_file_only = os.path.basename(input_file)
# Find the instrument used to collect the data
datamodel = datamodels.RampModel(input_file)
instrument = datamodel.meta.instrument.name.lower()
# If the data pre-date jwst version 1.2.1, then they will have
# the NUMDTHPT keyword (with string value of the number of dithers)
# rather than the newer NRIMDTPT keyword (with an integer value of
# the number of dithers). If so, we need to update the file here so
# that it doesn't cause the pipeline to crash later. Both old and
# new keywords are mapped to the model.meta.dither.dither_points
# metadata entry. So we should be able to focus on that.
if isinstance(datamodel.meta.dither.dither_points, str):
# If we have a string, change it to an integer
datamodel.meta.dither.dither_points = int(
datamodel.meta.dither.dither_points)
elif datamodel.meta.dither.dither_points is None:
# If the information is missing completely, put in a dummy value
datamodel.meta.dither.dither_points = 1
# Switch to calling the pipeline rather than individual steps,
# and use the run() method so that we can set parameters
# progammatically.
model = Detector1Pipeline()
# Always true
if instrument == 'nircam':
model.refpix.odd_even_rows = False
# Default CR rejection threshold is too low
model.jump.rejection_threshold = 15
# Turn off IPC step until it is put in the right place
model.ipc.skip = True
model.jump.save_results = True
model.jump.output_dir = output_dir
jump_output = os.path.join(output_dir,
input_file_only.replace('uncal', 'jump'))
# Check to see if the jump version of the requested file is already
# present
run_jump = not os.path.isfile(jump_output)
if ramp_fit:
model.ramp_fit.save_results = True
# model.save_results = True
model.output_dir = output_dir
# pipe_output = os.path.join(output_dir, input_file_only.replace('uncal', 'rate'))
pipe_output = os.path.join(
output_dir, input_file_only.replace('uncal', '0_ramp_fit'))
run_slope = not os.path.isfile(pipe_output)
if save_fitopt:
model.ramp_fit.save_opt = True
fitopt_output = os.path.join(
output_dir, input_file_only.replace('uncal', 'fitopt'))
run_fitopt = not os.path.isfile(fitopt_output)
else:
model.ramp_fit.save_opt = False
fitopt_output = None
run_fitopt = False
else:
model.ramp_fit.skip = True
pipe_output = None
fitopt_output = None
run_slope = False
run_fitopt = False
# Call the pipeline if any of the files at the requested calibration
# states are not present in the output directory
if run_jump or (ramp_fit and run_slope) or (save_fitopt and run_fitopt):
model.run(datamodel)
else:
print((
"Files with all requested calibration states for {} already present in "
"output directory. Skipping pipeline call.".format(input_file)))
return jump_output, pipe_output, fitopt_output