def test_saturation_grism_step(rtdata, ignore_asdf_paths):
# Testing retrieval of best ref file for grism data,
# and creation of a ramp file with CRDS selected saturation file applied.
rtdata.get_data("WFI/grism/ramp_grism.asdf")
rtdata.input = "ramp_grism.asdf"
# Test CRDS
step = SaturationStep()
model = rdm.open(rtdata.input)
ref_file_path = step.get_reference_file(model, "saturation")
ref_file_name = os.path.split(ref_file_path)[-1]
# Confirm that a WFI saturation reference file name was returned
assert "roman_wfi_saturation" in ref_file_name
# Test SaturationStep
output = "ramp_grism_saturationstep.asdf"
rtdata.output = output
args = ["romancal.step.SaturationStep", rtdata.input]
RomanStep.from_cmdline(args)
ramp_out = rdm.open(rtdata.output)
assert ("roman.pixeldq" in ramp_out.to_flat_dict())
rtdata.get_truth(f"truth/WFI/grism/{output}")
assert (compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) is
None)
def process(self, input):
"""Perform the photometric calibration step
Parameters
----------
input : Roman level 2 image datamodel (wfi_image-1.x.x)
input roman datamodel
Returns
-------
output_model : Roman level 2 image datamodel (wfi_image-1.x.x)
output roman datamodel
"""
# Open the input data model
with rdm.open(input) as input_model:
# Get reference file
reffile = self.get_reference_file(input_model, "photom")
# Open the relevant photom reference file as a datamodel
if reffile is not None:
# If there is a reference file, perform photom application
photom_model = rdm.open(reffile)
self.log.debug(f'Using PHOTOM ref file: {reffile}')
# Do the correction
if input_model.meta.exposure.type == "WFI_IMAGE":
output_model = photom.apply_photom(input_model,
photom_model)
output_model.meta.cal_step.photom = 'COMPLETE'
else:
self.log.warning('No photometric corrections for '
'spectral data')
self.log.warning('Photom step will be skipped')
input_model.meta.cal_step.photom = 'SKIPPED'
try:
photom_model.close()
except AttributeError:
pass
return input_model
else:
# Skip Photom step if no photom file
self.log.warning('No PHOTOM reference file found')
self.log.warning('Photom step will be skipped')
input_model.meta.cal_step.photom = 'SKIPPED'
return input_model
# Close the input and reference files
input_model.close()
try:
photom_model.close()
except AttributeError:
pass
return output_model
def _datamodels_open(cls, init, **kwargs):
"""
Provide access to this package's datamodels.open function
so that the stpipe infrastructure knows how to instantiate
models.
"""
return rdm.open(init, **kwargs)
def test_flat_field_grism_step(rtdata, ignore_asdf_paths):
"""Test for the flat field step using grism data. The reference file for
the grism and prism data should be None, only testing the grism
case here."""
input_file = "r0000201001001001002_01101_0001_WFI01_uncal.asdf"
rtdata.get_data(f"WFI/grism/{input_file}")
rtdata.input = input_file
# Test CRDS
step = FlatFieldStep()
model = rdm.open(rtdata.input)
try:
ref_file_path = step.get_reference_file(model, "flat")
ref_file_name = os.path.split(ref_file_path)[-1]
except CrdsLookupError:
ref_file_name = None
assert ref_file_name is None
# Test FlatFieldStep
output = "r0000201001001001002_01101_0001_WFI01_flat.asdf"
rtdata.output = output
args = ["romancal.step.FlatFieldStep", rtdata.input]
RomanStep.from_cmdline(args)
rtdata.get_truth(f"truth/WFI/grism/{output}")
assert (compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) is
None)
def process(self, input):
# Open the input data model
with rdm.open(input) as input_model:
# Get the name of the saturation reference file
self.ref_name = self.get_reference_file(input_model, 'saturation')
self.log.info('Using SATURATION reference file %s', self.ref_name)
# Check for a valid reference file
if self.ref_name == 'N/A':
self.log.warning('No SATURATION reference file found')
self.log.warning('Saturation step will be skipped')
result = input_model.copy()
result.meta.cal_step.saturation = 'SKIPPED'
return result
# Open the reference file data model
ref_model = SaturationRefModel(self.ref_name)
# Perform saturation check
sat = saturation.flag_saturation(input_model, ref_model)
# Close the reference file and update the step status
ref_model.close()
sat.meta.cal_step.saturation = 'COMPLETE'
return sat
def test_dark_step_output_dark_file(tmpdir, instrument, exptype):
"""Test that the the step can output a proper (optional) dark file"""
path = str(tmpdir / "dark_out.asdf")
# Set test size
shape = (2, 20, 20)
# Create test ramp and dark models
ramp_model, darkref_model = create_ramp_and_dark(shape, instrument,
exptype)
# Perform Dark Current subtraction step
DarkCurrentStep.call(ramp_model,
override_dark=darkref_model,
dark_output=path)
# Open dark file
dark_out_file_model = rdm.open(path)
# Test dark file results
assert type(dark_out_file_model) == DarkRefModel
assert dark_out_file_model.validate() is None
assert dark_out_file_model.data.shape == shape
assert dark_out_file_model.dq.shape == shape[1:]
assert dark_out_file_model.err.shape == shape
def test_dq_init_grism_step(rtdata, ignore_asdf_paths):
"""DMS26 Test: Testing retrieval of best ref file for grism data,
and creation of a ramp file with CRDS selected mask file applied."""
input_file = "r0000101001001001001_01102_0001_WFI01_uncal.asdf"
rtdata.get_data(f"WFI/grism/{input_file}")
rtdata.input = input_file
# Test CRDS
step = DQInitStep()
model = rdm.open(rtdata.input)
step.log.info('DMS26 MSG: Testing retrieval of best '
'ref file for grism data, '
'Success is creation of a ramp file with CRDS selected '
'mask file applied.')
step.log.info(f'DMS26 MSG: First data file: '
f'{rtdata.input.rsplit("/", 1)[1]}')
ref_file_path = step.get_reference_file(model, "mask")
step.log.info(f'DMS26 MSG: CRDS matched mask file: '
f'{ref_file_path.rsplit("/", 1)[1]}')
ref_file_name = os.path.split(ref_file_path)[-1]
assert "roman_wfi_mask" in ref_file_name
# Test DQInitStep
output = "r0000101001001001001_01102_0001_WFI01_dqinitstep.asdf"
rtdata.output = output
args = ["romancal.step.DQInitStep", rtdata.input]
step.log.info('DMS26 MSG: Running data quality initialization step.'
'The first ERROR is expected, due to extra CRDS parameters '
'not having been implemented yet.')
RomanStep.from_cmdline(args)
ramp_out = rdm.open(rtdata.output)
step.log.info('DMS26 MSG: Does ramp data contain pixeldq '
'from mask file? : '
f'{("roman.pixeldq" in ramp_out.to_flat_dict())}')
assert "roman.pixeldq" in ramp_out.to_flat_dict()
rtdata.get_truth(f"truth/WFI/grism/{output}")
step.log.info(
'DMS26 MSG: Was proper data quality initialized '
'ramp data produced? : '
f'{(compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) is None)}'
)
assert compare_asdf(rtdata.output, rtdata.truth, **
ignore_asdf_paths) is None
def test_dq_init_image_step(rtdata, ignore_asdf_paths):
# DMS25 Test: Testing retrieval of best ref file for image data,
# and creation of a ramp file with CRDS selected mask file applied.
rtdata.get_data("WFI/image/l1_0005_science_raw.asdf")
rtdata.input = "l1_0005_science_raw.asdf"
# Test CRDS
step = DQInitStep()
model = rdm.open(rtdata.input)
step.log.info(
'DMS25 MSG: Testing retrieval of best ref file for image data, '
'Success is creation of a ramp file with CRDS selected mask file applied.'
)
step.log.info(
f'DMS25 MSG: First data file: {rtdata.input.rsplit("/", 1)[1]}')
ref_file_path = step.get_reference_file(model, "mask")
step.log.info(
f'DMS25 MSG: CRDS matched mask file: {ref_file_path.rsplit("/", 1)[1]}'
)
ref_file_name = os.path.split(ref_file_path)[-1]
assert "roman_wfi_mask" in ref_file_name
# Test DQInitStep
output = "l1_0005_science_raw_dqinitstep.asdf"
rtdata.output = output
args = ["romancal.step.DQInitStep", rtdata.input]
step.log.info(
'DMS25 MSG: Running data quality initialization step. The first ERROR is '
'expected, due to extra CRDS parameters not having been implemented yet.'
)
RomanStep.from_cmdline(args)
ramp_out = rdm.open(rtdata.output)
step.log.info(
f'DMS25 MSG: Does ramp data contain pixeldq from mask file? : '
f'{("roman.pixeldq" in ramp_out.to_flat_dict())}')
assert ("roman.pixeldq" in ramp_out.to_flat_dict())
rtdata.get_truth(f"truth/WFI/image/{output}")
step.log.info(
f'DMS25 MSG: Was proper data quality initialized ramp data produced? : '
f'{(compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) is None)}'
)
assert (compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) is
None)
def process(self, step_input):
input_model = rdm.open(step_input)
# Get reference file paths
reference_file_names = {}
try:
reffile = self.get_reference_file(input_model, "flat")
except CrdsLookupError:
reffile = None
reference_file_names['flat'] = reffile if reffile != 'N/A' else None
# Open the relevant reference files as datamodels
reference_file_models = {}
if reffile is not None:
reference_file_models['flat'] = rdm.open(reffile)
self.log.debug(f'Using FLAT ref file: {reffile}')
else:
reference_file_models['flat'] = None
self.log.debug('Using FLAT ref file')
# Do the flat-field correction
output_model = flat_field.do_correction(
input_model,
**reference_file_models,
)
# Close the input and reference files
input_model.close()
try:
for model in reference_file_models.values():
model.close()
except AttributeError:
pass
if self.save_results:
try:
self.suffix = 'flat'
except AttributeError:
self['suffix'] = 'flat'
return output_model
def process(self, input):
# Open the input data model
with rdm.open(input) as input_model:
# Get the name of the linearity reference file to use
self.lin_name = self.get_reference_file(input_model, 'linearity')
self.log.info('Using Linearity reference file %s', self.lin_name)
# Check for a valid reference file
if self.lin_name == 'N/A':
self.log.warning('No Linearity reference file found')
self.log.warning('Linearity step will be skipped')
result = input_model.copy()
result.meta.cal_step['linearity'] = 'SKIPPED'
return result
lin_model = rdd.LinearityRefModel(self.lin_name)
# copy poly coeffs from linearity model so Nan's can be updated
lin_coeffs = lin_model.coeffs.copy()
lin_dq = lin_model.dq # 2D pixeldq from linearity model
gdq = input_model.groupdq # groupdq array of input model
pdq = input_model.pixeldq # pixeldq array of input model
gdq = gdq[np.newaxis, :]
output_model = input_model.copy()
output_model.data = output_model.data[np.newaxis, :]
# Call linearity correction function in stcal
# The third return value is the procesed zero frame which
# Roman does not use.
new_data, new_pdq, _ = linearity_correction(
output_model.data, gdq, pdq, lin_coeffs, lin_dq, dqflags.pixel)
output_model.data = new_data[0, :, :, :]
output_model.pixeldq = new_pdq
# Close the reference file and update the step status
lin_model.close()
output_model.meta.cal_step['linearity'] = 'COMPLETE'
return output_model
def test_flat_field_image_step(rtdata, ignore_asdf_paths):
rtdata.get_data("WFI/image/l2_0004_rate.asdf")
rtdata.input = "l2_0004_rate.asdf"
# Test CRDS
step = FlatFieldStep()
model = rdm.open(rtdata.input)
ref_file_path = step.get_reference_file(model, "flat")
ref_file_name = os.path.split(ref_file_path)[-1]
assert "roman_wfi_flat" in ref_file_name
# Test FlatFieldStep
output = "l2_0004_rate_flat.asdf"
rtdata.output = output
args = ["romancal.step.FlatFieldStep", rtdata.input]
RomanStep.from_cmdline(args)
rtdata.get_truth(f"truth/WFI/image/{output}")
assert (compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) is
None)
def test_flat_field_image_step(rtdata, ignore_asdf_paths):
"""Test for the flat field step using imaging data."""
input_data = "r0000101001001001001_01101_0001_WFI01_rampfitstep.asdf"
rtdata.get_data(f"WFI/image/{input_data}")
rtdata.input = input_data
# Test CRDS
step = FlatFieldStep()
model = rdm.open(rtdata.input)
ref_file_path = step.get_reference_file(model, "flat")
ref_file_name = os.path.split(ref_file_path)[-1]
assert "roman_wfi_flat" in ref_file_name
# Test FlatFieldStep
output = "r0000101001001001001_01101_0001_WFI01_flat.asdf"
rtdata.output = output
args = ["romancal.step.FlatFieldStep", rtdata.input]
RomanStep.from_cmdline(args)
rtdata.get_truth(f"truth/WFI/image/{output}")
assert compare_asdf(rtdata.output, rtdata.truth, **
ignore_asdf_paths) is None
def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
"""DMS140 Test: Testing application of photometric correction using
CRDS selected photom file."""
input_data = "r0000201001001001001_01101_0001_WFI01_cal.asdf"
rtdata.get_data(f"WFI/image/{input_data}")
rtdata.input = input_data
# Define step (for running and log access)
step = PhotomStep()
# In Wide Field Imaging mode, the DMS shall generate Level 2 science
# data products with absolute photometry calibrated in the WFI filter
# used for the exposure.
step.log.info('DMS140 MSG: Testing absolute photometric '
'calibrated image data. '
'Success is creation of a Level 2 image file with '
'CRDS selected photom file applied.')
step.log.info('DMS140 MSG: Image data file: '
f'{rtdata.input.rsplit("/", 1)[1]}')
# Note: if any of the following tests fail, check for a different
# photom match from CRDS. Values come from roman_wfi_photom_0034.asdf
# Test PhotomStep
output = "r0000201001001001001_01101_0001_WFI01_cal_photomstep.asdf"
rtdata.output = output
args = ["romancal.step.PhotomStep", rtdata.input]
step.log.info('DMS140 MSG: Running photometric conversion step.'
' The first ERROR is expected, due to extra CRDS parameters'
' not having been implemented yet.')
RomanStep.from_cmdline(args)
photom_out = rdm.open(rtdata.output)
step.log.info(f'DMS140 MSG: Photom step recorded as complete? : '
f'{photom_out.meta.cal_step.photom == "COMPLETE"}')
assert photom_out.meta.cal_step.photom == "COMPLETE"
step.log.info(
'DMS140 MSG: Photom megajansky conversion calculated? : ' +
str((photom_out.meta.photometry.conversion_megajanskys.unit == u.MJy /
u.sr) and math.isclose(
photom_out.meta.photometry.conversion_megajanskys.value,
0.3214,
abs_tol=0.0001)))
assert photom_out.meta.photometry.conversion_megajanskys.unit == u.MJy / u.sr
assert math.isclose(
photom_out.meta.photometry.conversion_megajanskys.value,
0.3214,
abs_tol=0.0001)
step.log.info(
'DMS140 MSG: Photom microjanskys conversion calculated? : ' +
str((photom_out.meta.photometry.conversion_microjanskys.unit == u.uJy /
u.arcsec**2) and (math.isclose(
photom_out.meta.photometry.conversion_microjanskys.value,
7.5549,
abs_tol=0.0001))))
assert photom_out.meta.photometry.conversion_microjanskys.unit == u.uJy / u.arcsec**2
assert math.isclose(
photom_out.meta.photometry.conversion_microjanskys.value,
7.5549,
abs_tol=0.0001)
step.log.info('DMS140 MSG: Pixel area in steradians calculated? : ' + str(
(photom_out.meta.photometry.pixelarea_steradians.unit == u.sr) and
(math.isclose(photom_out.meta.photometry.pixelarea_steradians.value,
2.8083e-13,
abs_tol=1.0e-17))))
assert photom_out.meta.photometry.pixelarea_steradians.unit == u.sr
assert math.isclose(photom_out.meta.photometry.pixelarea_steradians.value,
2.8083e-13,
abs_tol=1.0e-17)
step.log.info(
'DMS140 MSG: Pixel area in square arcseconds calculated? : ' +
str((photom_out.meta.photometry.pixelarea_arcsecsq.unit == u.arcsec**2)
and
(math.isclose(photom_out.meta.photometry.pixelarea_arcsecsq.value,
0.011948,
abs_tol=1.0e-6))))
assert photom_out.meta.photometry.pixelarea_arcsecsq.unit == u.arcsec**2
assert math.isclose(photom_out.meta.photometry.pixelarea_arcsecsq.value,
0.011948,
abs_tol=1.0e-6)
step.log.info('DMS140 MSG: Photom megajansky conversion uncertainty calculated? : ' +
str((photom_out.meta.photometry.conversion_megajanskys_uncertainty.unit
== u.MJy / u.sr) and
(math.isclose(photom_out.meta.photometry.\
conversion_megajanskys_uncertainty.value, 0.0,
abs_tol=1.0e-6))))
assert photom_out.meta.photometry.conversion_megajanskys_uncertainty.unit == u.MJy / u.sr
assert math.isclose(
photom_out.meta.photometry.conversion_megajanskys_uncertainty.value,
0.0,
abs_tol=1.0e-6)
step.log.info('DMS140 MSG: Photom megajansky conversion uncertainty calculated? : ' +
str((photom_out.meta.photometry.conversion_microjanskys_uncertainty.unit ==
u.uJy / u.arcsec ** 2) and
(math.isclose(photom_out.meta.photometry.\
conversion_microjanskys_uncertainty.value, 0.0,
abs_tol=1.0e-6))))
assert photom_out.meta.photometry.conversion_microjanskys_uncertainty.unit == \
u.uJy / u.arcsec ** 2
assert math.isclose(
photom_out.meta.photometry.conversion_microjanskys_uncertainty.value,
0.0,
abs_tol=1.0e-6)
rtdata.get_truth(f"truth/WFI/image/{output}")
step.log.info(
f'DMS140 MSG: Was the proper absolute photometry calibrated image data produced?'
f' : {(compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) is None)}'
)
assert compare_asdf(rtdata.output, rtdata.truth, **
ignore_asdf_paths) is None
def process(self, input):
"""Perform the dq_init calibration step
Parameters
----------
input : Roman datamodel
input roman datamodel
Returns
-------
output_model : Roman datamodel
result roman datamodel
"""
# Open datamodel
input_model = self.open_model(input)
# Convert to RampModel if needed
if not (isinstance(input_model, RampModel)):
# Create base ramp node with dummy values (for validation)
input_ramp = testutil.mk_ramp(input_model.shape)
# Copy input_model contents into RampModel
for key in input_model.keys():
# If a dictionary (like meta), overwrite entires (but keep
# required dummy entries that may not be in input_model)
if isinstance(input_ramp[key], dict):
input_ramp[key].update(input_model.__getattr__(key))
elif isinstance(input_ramp[key], np.ndarray):
# Cast input ndarray as RampModel dtype
input_ramp[key] = input_model.__getattr__(key).astype(
input_ramp[key].dtype)
else:
input_ramp[key] = input_model.__getattr__(key)
# Create model from node
init_model = RampModel(input_ramp)
else:
init_model = input_model
# Get reference file paths
reference_file_names = {}
reffile = self.get_reference_file(init_model, "mask")
reference_file_names['mask'] = reffile if reffile != 'N/A' else None
# Open the relevant reference files as datamodels
reference_file_models = {}
# Test for reference files
if reffile is not None:
# If there are mask files, perform dq step
reference_file_models['mask'] = rdm.open(reffile)
self.log.debug(f'Using MASK ref file: {reffile}')
# Apply the DQ step
output_model = dq_initialization.do_dqinit(
init_model,
**reference_file_models,
)
else:
# Skip DQ step if no mask files
reference_file_models['mask'] = None
self.log.warning('No MASK reference file found.')
self.log.warning('DQ initialization step will be skipped.')
output_model = init_model
output_model.meta.cal_step.dq_init = 'SKIPPED'
# Close the input and reference files
input_model.close()
init_model.close()
try:
for model in reference_file_models.values():
model.close()
except AttributeError:
pass
return output_model
def test_level2_image_processing_pipeline(rtdata, ignore_asdf_paths):
"""Tests for flat field imaging processing requirements DMS86 & DMS 87 """
input_data = "r0000101001001001001_01101_0001_WFI01_uncal.asdf"
rtdata.get_data(f"WFI/image/{input_data}")
rtdata.input = input_data
# Test Pipeline
output = "r0000101001001001001_01101_0001_WFI01_cal.asdf"
rtdata.output = output
args = [
"--disable-crds-steppars",
"--steps.jump.rejection_threshold=180.0",
"--steps.jump.three_group_rejection_threshold=185.0",
"--steps.jump.four_group_rejection_threshold=190",
"roman_elp",
rtdata.input,
]
ExposurePipeline.from_cmdline(args)
rtdata.get_truth(f"truth/WFI/image/{output}")
assert compare_asdf(rtdata.output, rtdata.truth, **
ignore_asdf_paths) is None
# Perform DMS tests
# Initial prep
model = rdm.open(rtdata.output)
pipeline = ExposurePipeline()
# DMS86 instrument artifact correction tests
pipeline.log.info('Status of the step: assign_wcs ' +
str(model.meta.cal_step.assign_wcs))
pipeline.log.info('DMS86 MSG: Testing completion of wcs assignment in'
'Level 2 image output.......' +
passfail(model.meta.cal_step.assign_wcs == 'COMPLETE'))
assert model.meta.cal_step.assign_wcs == 'COMPLETE'
pipeline.log.info('Status of the step: flat_field ' +
str(model.meta.cal_step.flat_field))
pipeline.log.info('DMS86 MSG: Testing completion of flat fielding in'
'Level 2 image output.......' +
passfail(model.meta.cal_step.flat_field == 'PASS'))
assert model.meta.cal_step.flat_field == 'COMPLETE'
pipeline.log.info('Status of the step: dark ' +
str(model.meta.cal_step.dark))
pipeline.log.info('DMS86 MSG: Testing completion of dark correction in'
'Level 2 image output.......' +
passfail(model.meta.cal_step.dark == 'COMPLETE'))
assert model.meta.cal_step.dark == 'COMPLETE'
pipeline.log.info('Status of the step: dq_init ' +
str(model.meta.cal_step.dq_init))
pipeline.log.info('DMS86 MSG: Testing completion of data quality'
' correction in Level 2 image output.......' +
passfail(model.meta.cal_step.dq_init == 'COMPLETE'))
assert model.meta.cal_step.dq_init == 'COMPLETE'
pipeline.log.info('Status of the step: jump ' +
str(model.meta.cal_step.jump))
pipeline.log.info('DMS86 MSG: Testing completion of jump detection in'
'Level 2 image output.......' +
passfail(model.meta.cal_step.jump == 'COMPLETE'))
assert model.meta.cal_step.jump == 'COMPLETE'
pipeline.log.info('Status of the step: linearity ' +
str(model.meta.cal_step.assign_wcs))
pipeline.log.info('DMS86 MSG: Testing completion of linearity correction'
' in Level 2 image output.......' +
passfail(model.meta.cal_step.linearity == 'COMPLETE'))
assert model.meta.cal_step.linearity == 'COMPLETE'
pipeline.log.info('Status of the step: ramp_fit ' +
str(model.meta.cal_step.ramp_fit))
pipeline.log.info('DMS86 MSG: Testing completion of ramp fitting in'
'Level 2 image output.......' +
passfail(model.meta.cal_step.ramp_fit == 'COMPLETE'))
assert model.meta.cal_step.ramp_fit == 'COMPLETE'
pipeline.log.info('Status of the step: saturation ' +
str(model.meta.cal_step.saturation))
pipeline.log.info('DMS86 MSG: Testing completion of saturation detection '
'in Level 2 image output.......' +
passfail(model.meta.cal_step.saturation == 'COMPLETE'))
assert model.meta.cal_step.saturation == 'COMPLETE'
# DMS87 data quality tests
pipeline.log.info('DMS87 MSG: Testing existence of data quality array (dq)'
' in Level 2 image output.......' +
passfail("dq" in model.keys()))
assert "dq" in model.keys()
pipeline.log.info('DMS87 MSG: Testing existence of general error array '
'(err) in Level 2 image output.......' +
passfail("err" in model.keys()))
assert "err" in model.keys()
pipeline.log.info('DMS87 MSG: Testing existence of Poisson noise variance'
'array (var_poisson) in Level 2 image output.......' +
passfail("var_poisson" in model.keys()))
assert "var_poisson" in model.keys()
pipeline.log.info('DMS87 MSG: Testing existence of read noise variance '
'array (var_rnoise) in level 2 image output.......' +
passfail("var_rnoise" in model.keys()))
assert "var_rnoise" in model.keys()
pipeline.log.info('DMS87 MSG: Testing existence of flatfield uncertainty '
'variance array (var_flat) in Level 2 image output....' +
passfail("var_flat" in model.keys()))
assert "var_flat" in model.keys()
# DMS88 total exposure time test
pipeline.log.info('DMS88 MSG: Testing existence of total exposure time '
'(exposure_time) in Level 2 image output.......' +
passfail("exposure_time" in model.meta.exposure))
assert "exposure_time" in model.meta.exposure
# DMS89 WCS tests
pipeline.log.info('DMS89 MSG: Testing that the wcs bounding'
'box was generated.......' +
passfail((len(model.meta.wcs.bounding_box) == 2)))
assert len(model.meta.wcs.bounding_box) == 2
# Save original wcs information
orig_wcs = copy.deepcopy(model.meta.wcs)
del model.meta['wcs']
# Create new pointing for the model
# RA & Dec are each shifted + 10 degrees, unless they are near
# the upper limit, in which case they are shifted -10 degrees.
delta = [10.0, 10.0]
if model.meta.wcsinfo.ra_ref >= 350.0:
delta[0] *= -1.0
if model.meta.wcsinfo.dec_ref >= 80.0:
delta[1] *= -1.0
model.meta.wcsinfo.ra_ref += delta[0]
model.meta.wcsinfo.dec_ref += delta[1]
# Create new wcs object for the new pointing
model = AssignWcsStep.call(model)
rtdata.output = output.rsplit(".", 1)[0] + "_repoint.asdf"
model.to_asdf(rtdata.output)
# Test that repointed file matches truth
rtdata.get_truth("truth/WFI/image/" + output.rsplit(".", 1)[0] +
"_repoint.asdf")
assert compare_asdf(rtdata.output, rtdata.truth, **
ignore_asdf_paths) is None
pipeline.log.info(
'DMS89 MSG: Testing that the different pointings '
'create differing wcs.......' + passfail((
(np.abs(orig_wcs(2048, 2048)[0] - model.meta.wcs(2048, 2048)[0])) -
10.0) < 1.0))
assert_allclose([
orig_wcs(2048, 2048)[0] + delta[0],
orig_wcs(2048, 2048)[1] + delta[1]
],
model.meta.wcs(2048, 2048),
atol=1.0)
def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths):
"""DMS79 Test: Testing that different datetimes pull different
flat files and successfully make level 2 output"""
# First file
input_l2_file = "r0000101001001001001_01101_0001_WFI01_assign_wcs.asdf"
rtdata.get_data(f"WFI/image/{input_l2_file}")
rtdata.input = input_l2_file
# Test CRDS
step = FlatFieldStep()
model = rdm.open(rtdata.input)
step.log.info('DMS79 MSG: Testing retrieval of best ref file, '
'Success is flat file with correct use after date')
step.log.info('DMS79 MSG: First data file: '
f'{rtdata.input.rsplit("/", 1)[1]}')
step.log.info('DMS79 MSG: Observation date: '
f'{model.meta.exposure.start_time}')
ref_file_path = step.get_reference_file(model, "flat")
step.log.info('DMS79 MSG: CRDS matched flat file: '
f'{ref_file_path.rsplit("/", 1)[1]}')
flat = rdm.open(ref_file_path)
step.log.info(f'DMS79 MSG: flat file UseAfter date: {flat.meta.useafter}')
step.log.info(f'DMS79 MSG: UseAfter date before observation date? : '
f'{(flat.meta.useafter < model.meta.exposure.start_time)}')
# Test FlatFieldStep
output = "r0000101001001001001_01101_0001_WFI01_flat.asdf"
rtdata.output = output
args = ["romancal.step.FlatFieldStep", rtdata.input]
step.log.info('DMS79 MSG: Running flat fielding step. The first ERROR is'
'expected, due to extra CRDS parameters not having been '
'implemented yet.')
RomanStep.from_cmdline(args)
rtdata.get_truth(f"truth/WFI/image/{output}")
step.log.info(
'DMS79 MSG: Was proper flat fielded '
'Level 2 data produced? : '
f'{(compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) is None)}'
)
assert compare_asdf(rtdata.output, rtdata.truth, **
ignore_asdf_paths) is None
# This test requires a second file, in order to meet the DMS79 requirement.
# The test will show that two files with different observation dates match
# to separate flat files in CRDS.
# Second file
input_file = "r0000101001001001001_01101_0002_WFI01_assign_wcs.asdf"
rtdata.get_data(f"WFI/image/{input_file}")
rtdata.input = input_file
# Test CRDS
step = FlatFieldStep()
model = rdm.open(rtdata.input)
step.log.info('DMS79 MSG: Second data file: '
f'{rtdata.input.rsplit("/", 1)[1]}')
step.log.info('DMS79 MSG: Observation date: '
f'{model.meta.exposure.start_time}')
ref_file_path_b = step.get_reference_file(model, "flat")
step.log.info('DMS79 MSG: CRDS matched flat file: '
f'{ref_file_path_b.rsplit("/", 1)[1]}')
flat = rdm.open(ref_file_path_b)
step.log.info(f'DMS79 MSG: flat file UseAfter date: {flat.meta.useafter}')
step.log.info(f'DMS79 MSG: UseAfter date before observation date? : '
f'{(flat.meta.useafter < model.meta.exposure.start_time)}')
# Test FlatFieldStep
output = "r0000101001001001001_01101_0002_WFI01_flat.asdf"
rtdata.output = output
args = ["romancal.step.FlatFieldStep", rtdata.input]
step.log.info('DMS79 MSG: Running flat fielding step. The first ERROR is'
'expected, due to extra CRDS parameters not having been '
'implemented yet.')
RomanStep.from_cmdline(args)
rtdata.get_truth(f"truth/WFI/image/{output}")
step.log.info(
'DMS79 MSG: Was proper flat fielded '
'Level 2 data produced? : '
f'{(compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) is None)}'
)
assert compare_asdf(rtdata.output, rtdata.truth, **
ignore_asdf_paths) is None
# Test differing flat matches
step.log.info('DMS79 MSG REQUIRED TEST: Are the two data files '
'matched to different flat files? : '
f'{("/".join(ref_file_path.rsplit("/", 3)[1:]))} != '
f'{("/".join(ref_file_path_b.rsplit("/", 3)[1:]))}')
assert ("/".join(ref_file_path.rsplit("/", 1)[1:])) != \
("/".join(ref_file_path_b.rsplit("/", 1)[1:]))