def test_is_tso_nrcgrism_nints1():
"""Test is_tso with NRC_TSGRISM and NINTS=1"""
model = datamodels.JwstDataModel()
model.meta.exposure.type = "NRC_TSGRISM"
model.meta.visit.tsovisit = False
model.meta.exposure.nints = 10
# with NINTS>1, should be True
assert pipe_utils.is_tso(model)
# with NINTS=1, should be False
model.meta.exposure.nints = 1
assert not pipe_utils.is_tso(model)
# with hardwired TSO EXP_TYPE's, should always be True
assert (is_tso(model) or model.meta.exposure.type.lower()
in ['nrc_tsimage', 'nrc_tsgrism'])
def process(self, input):
'''Process a Stage 0 *_uncal.fits file to Stage 1 *_rate.fits and *_rateints.fits files.
Steps taken to perform this processing can follow the default JWST pipeline, or alternative methods.
Parameters
----------
input: str, tuple, `~astropy.io.fits.HDUList`, ndarray, dict, None
- None: Create a default data model with no shape.
- tuple: Shape of the data array.
Initialize with empty data array with shape specified by the.
- file path: Initialize from the given file (FITS or ASDF)
- readable file object: Initialize from the given file
object
- `~astropy.io.fits.HDUList`: Initialize from the given
`~astropy.io.fits.HDUList`.
- A numpy array: Used to initialize the data array
Returns
-------
out_model: jwst.datamodels.ImageModel
The output ImageModel to be returned from the ramp fit step.
int_model: jwst.datamodels.CubeModel
The output CubeModel to be returned from the ramp fit step.
Notes
-----
History:
- October 2021 Aarynn Carter and Eva-Maria Ahrer
Initial version
- February 2022 Aarynn Carter and Eva-Maria Ahrer
Updated for JWST version 1.3.3, code restructure
'''
with datamodels.RampModel(input) as input_model:
readnoise_filename = self.get_reference_file(input_model, 'readnoise')
gain_filename = self.get_reference_file(input_model, 'gain')
log.info('Using READNOISE reference file: %s', readnoise_filename)
log.info('Using GAIN reference file: %s', gain_filename)
with datamodels.ReadnoiseModel(readnoise_filename) as readnoise_model, \
datamodels.GainModel(gain_filename) as gain_model:
# Try to retrieve the gain factor from the gain reference file.
# If found, store it in the science model meta data, so that it's
# available later in the gain_scale step, which avoids having to
# load the gain ref file again in that step.
if gain_model.meta.exposure.gain_factor is not None:
input_model.meta.exposure.gain_factor = gain_model.meta.exposure.gain_factor
# Get gain arrays, subarrays if desired.
frames_per_group = input_model.meta.exposure.nframes
readnoise_2d, gain_2d = get_reference_file_subarrays(
input_model, readnoise_model, gain_model, frames_per_group)
log.info('Using algorithm = %s' % self.algorithm)
log.info('Using weighting = %s' % self.weighting)
buffsize = ramp_fit.BUFSIZE
if pipe_utils.is_tso(input_model) and hasattr(input_model, 'int_times'):
input_model.int_times = input_model.int_times
else:
input_model.int_times = None
# DEFAULT RAMP FITTING ALGORITHM
if self.algorithm == 'default':
# In our case, default just means Optimal Least Squares
self.algorithm = 'OLS'
if self.weighting == 'default':
# Want to use the default optimal weighting
pass
elif self.weighting == 'fixed':
# Want to use default weighting, but don't want to
# change exponent between pixels.
if not isinstance(self.fixed_exponent, (int, float)):
raise ValueError('Weighting exponent must be of type "int" or "float" for "default_fixed" weighting')
# Overwrite the exponent calculation function from ols_fit
stcal.ramp_fitting.ols_fit.calc_power = partial(fixed_power, weighting_exponent=self.fixed_exponent) #Pipeline version 1.3.3
elif self.weighting == 'interpolated':
# Want to use an interpolated version of the default weighting.
# Overwrite the exponent calculation function from ols_fit
stcal.ramp_fitting.ols_fit.calc_power = interpolate_power #Pipeline version 1.3.3
elif self.weighting == 'uniform':
# Want each frame and pixel weighted equally
# Overwrite the entire optimal calculation function
stcal.ramp_fitting.ols_fit.calc_opt_sums = calc_opt_sums_uniform_weight #Pipeline version 1.3.3
elif self.weighting == 'custom':
# Want to manually assign snr bounds for exponent changes
# Overwrite the exponent calculation function from ols_fit
stcal.ramp_fitting.ols_fit.calc_power = partial(custom_power, snr_bounds=self.custom_snr_bounds, exponents=self.custom_exponents) #Pipeline version 1.3.3
else:
raise ValueError('Could not interpret weighting "{}".'.format(self.weighting))
# Important! Must set the weighting to 'optimal' for the actual
# ramp_fit() function, previous if statements will have changed
# it's underlying functionality.
self.weighting = 'optimal'
image_info, integ_info, opt_info, gls_opt_model = ramp_fit.ramp_fit(input_model, buffsize, self.save_opt, readnoise_2d,\
gain_2d, self.algorithm, self.weighting,\
self.maximum_cores, dqflags.pixel)
# FUTURE IMPROVEMENT, WFC3-like differenced frames.
elif self.algorithm == 'differenced':
raise ValueError('I do not know how to do differenced frames yet.')
# PRIMARILY FOR TESTING, MEAN OF RAMP
elif self.algorithm == 'mean':
image_info, integ_info, opt_info = mean_ramp_fit_single(input_model, buffsize, self.save_opt, readnoise_2d,\
gain_2d, self.algorithm, self.weighting,\
self.maximum_cores, dqflags.pixel)
else:
raise ValueError('Ramp fitting algorithm "{}" not implemented.'.format(self.algorithm))
if image_info is not None:
out_model = create_image_model(input_model, image_info)
out_model.meta.bunit_data = 'DN/s'
out_model.meta.bunit_err = 'DN/s'
out_model.meta.cal_step.ramp_fit = 'COMPLETE'
if integ_info is not None:
int_model = create_integration_model(input_model, integ_info)
int_model.meta.bunit_data = 'DN/s'
int_model.meta.bunit_err = 'DN/s'
int_model.meta.cal_step.ramp_fit = 'COMPLETE'
return out_model, int_model
def process(self, input_data):
"""Perform outlier detection processing on input data."""
with datamodels.open(input_data) as input_models:
self.input_models = input_models
if not isinstance(self.input_models, datamodels.ModelContainer):
self.input_container = False
else:
self.input_container = True
# Setup output path naming if associations are involved.
asn_id = None
try:
asn_id = self.input_models.meta.asn_table.asn_id
except (AttributeError, KeyError):
pass
if asn_id is None:
asn_id = self.search_attr('asn_id')
if asn_id is not None:
_make_output_path = self.search_attr('_make_output_path',
parent_first=True)
self._make_output_path = partial(_make_output_path,
asn_id=asn_id)
# Setup outlier detection parameters
pars = {
'weight_type': self.weight_type,
'pixfrac': self.pixfrac,
'kernel': self.kernel,
'fillval': self.fillval,
'nlow': self.nlow,
'nhigh': self.nhigh,
'maskpt': self.maskpt,
'grow': self.grow,
'snr': self.snr,
'scale': self.scale,
'backg': self.backg,
'allowed_memory': self.allowed_memory,
'save_intermediate_results': self.save_intermediate_results,
'resample_data': self.resample_data,
'good_bits': self.good_bits,
'make_output_path': self.make_output_path,
}
# Add logic here to select which version of OutlierDetection
# needs to be used depending on the input data
if self.input_container:
single_model = self.input_models[0]
else:
single_model = self.input_models
exptype = single_model.meta.exposure.type
self.check_input()
# check for TSO models first
if is_tso(single_model) or exptype in CORON_IMAGE_MODES:
# algorithm selected for TSO data (no resampling)
pars['resample_data'] = False # force resampling off...
detection_step = outlier_registry['imaging']
elif exptype in IMAGE_MODES:
# imaging with resampling
detection_step = outlier_registry['imaging']
pars['resample_suffix'] = 'i2d'
elif exptype in SLIT_SPEC_MODES:
detection_step = outlier_registry['slitspec']
pars['resample_suffix'] = 's2d'
elif exptype in IFU_SPEC_MODES:
# select algorithm for IFU data
detection_step = outlier_registry['ifu']
pars['resample_suffix'] = 's3d'
else:
self.log.error(
"Outlier detection failed for unknown/unsupported ",
f"exposure type: {exptype}")
self.valid_input = False
if not self.valid_input:
if self.input_container:
for model in self.input_models:
model.meta.cal_step.outlier_detection = "SKIPPED"
else:
self.input_models.meta.cal_step.outlier_detection = "SKIPPED"
self.skip = True
return self.input_models
self.log.debug(
f"Using {detection_step.__name__} class for outlier_detection")
reffiles = {}
# Set up outlier detection, then do detection
step = detection_step(self.input_models, reffiles=reffiles, **pars)
step.do_detection()
state = 'COMPLETE'
if self.input_container:
for model in self.input_models:
model.meta.cal_step.outlier_detection = state
model.meta.filetype = 'cosmic-ray flagged'
else:
self.input_models.meta.cal_step.outlier_detection = state
self.input_models.meta.filetype = 'cosmic-ray flagged'
return self.input_models
def test_is_tso_from_tsoflag(tsovisit, expected):
"""Test is_tso integrity based on the TSO flag"""
model = datamodels.JwstDataModel()
model.meta.visit.tsovisit = tsovisit
assert pipe_utils.is_tso(model) is expected
def test_is_tso_from_exptype(exp_type, expected):
"""Test is_tso integrity based on exp_type"""
model = datamodels.JwstDataModel()
model.meta.exposure.type = exp_type.upper()
assert pipe_utils.is_tso(model) is expected
