def test_core_schema(tmp_path):
# Set temporary asdf file
file_path = tmp_path / "test.asdf"
wfi_image = utils.mk_level2_image(arrays=(10, 10))
with asdf.AsdfFile() as af:
af.tree = {'roman': wfi_image}
with pytest.raises(ValidationError):
af.tree['roman'].meta.telescope = 'NOTROMAN'
af.tree['roman'].meta['telescope'] = 'NOTROMAN'
with pytest.raises(ValidationError):
af.write_to(file_path)
af.tree['roman'].meta.telescope = 'ROMAN'
af.write_to(file_path)
# Now mangle the file
with open(file_path, 'rb') as fp:
fcontents = fp.read()
romanloc = fcontents.find(bytes('ROMAN', 'utf-8'))
newcontents = fcontents[:romanloc] + \
bytes('X', 'utf-8') + fcontents[romanloc + 1:]
with open(file_path, 'wb') as fp:
fp.write(newcontents)
with pytest.raises(ValidationError):
with datamodels.open(file_path) as model:
pass
asdf.get_config().validate_on_read = False
with datamodels.open(file_path) as model:
assert model.meta.telescope == 'XOMAN'
asdf.get_config().validate_on_read = True
def process(self, input):
# Open the input data model
with rdd.open(input) as input_model:
# Get the name of the dark reference file to use
self.dark_name = self.get_reference_file(input_model, 'dark')
self.log.info('Using DARK reference file %s', self.dark_name)
# Check for a valid reference file
if self.dark_name == 'N/A':
self.log.warning('No DARK reference file found')
self.log.warning('Dark current step will be skipped')
result = input_model.copy()
result.meta.cal_step.dark = 'SKIPPED'
return result
# Create name for the intermediate dark, if desired.
dark_output = self.dark_output
if dark_output is not None:
dark_output = self.make_output_path(None,
basepath=dark_output,
ignore_use_model=True)
dark_model = rdd.open(self.dark_name)
# Do the dark correction
result = dark_sub.do_correction(input_model, dark_model,
dark_output)
dark_model.close()
return result
def test_path_input(tmp_path):
file_path = tmp_path / "test.asdf"
with asdf.AsdfFile() as af:
tree = utils.mk_level2_image()
af.tree = {'roman': tree}
af.write_to(file_path)
# Test with PurePath input:
with datamodels.open(file_path) as model:
assert model.meta.telescope == "ROMAN"
af = model._asdf
# When open creates the file pointer, it should be
# closed when the model is closed:
assert af._closed
# Test with string input:
with datamodels.open(str(file_path)) as model:
assert model.meta.telescope == "ROMAN"
af = model._asdf
assert af._closed
# Appropriate error when file is missing:
with pytest.raises(FileNotFoundError):
with datamodels.open(tmp_path / "missing.asdf"):
pass
def test_opening_linearity_ref(tmp_path):
# First make test reference file
file_path = tmp_path / 'testlinearity.asdf'
utils.mk_linearity(filepath=file_path)
linearity = datamodels.open(file_path)
assert linearity.meta.instrument.optical_element == 'F158'
assert isinstance(linearity, datamodels.LinearityRefModel)
def test_opening_flat_ref(tmp_path):
# First make test reference file
file_path = tmp_path / 'testflat.asdf'
utils.mk_flat(file_path)
flat = datamodels.open(file_path)
assert flat.meta.instrument.optical_element == 'F158'
assert isinstance(flat, datamodels.FlatRefModel)
def test_opening_superbias_ref(tmp_path):
# First make test reference file
file_path = tmp_path / 'testsuperbias.asdf'
utils.mk_superbias(filepath=file_path)
superbias = datamodels.open(file_path)
assert superbias.meta.instrument.optical_element == 'F158'
assert isinstance(superbias, datamodels.SuperbiasRefModel)
def test_opening_pixelarea_ref(tmp_path):
# First make test reference file
file_path = tmp_path / 'testpixelarea.asdf'
utils.mk_pixelarea(filepath=file_path)
pixelarea = datamodels.open(file_path)
assert pixelarea.meta.instrument.optical_element == 'F158'
assert isinstance(pixelarea, datamodels.PixelareaRefModel)
def test_opening_saturation_ref(tmp_path):
# First make test reference file
file_path = tmp_path / 'testsaturation.asdf'
utils.mk_saturation(filepath=file_path)
saturation = datamodels.open(file_path)
assert saturation.meta.instrument.optical_element == 'F158'
assert isinstance(saturation, datamodels.SaturationRefModel)
def test_asdf_file_input():
tree = utils.mk_level2_image()
with asdf.AsdfFile() as af:
af.tree = {'roman': tree}
model = datamodels.open(af)
assert model.meta.telescope == 'ROMAN'
model.close()
def test_opening_mask_ref(tmp_path):
# First make test reference file
file_path = tmp_path / 'testmask.asdf'
utils.mk_mask(filepath=file_path)
mask = datamodels.open(file_path)
assert mask.meta.instrument.optical_element == 'F158'
assert isinstance(mask, datamodels.MaskRefModel)
def test_opening_dark_ref(tmp_path):
# First make test reference file
file_path = tmp_path / 'testdark.asdf'
utils.mk_dark(filepath=file_path)
dark = datamodels.open(file_path)
assert dark.meta.instrument.optical_element == 'F158'
assert isinstance(dark, datamodels.DarkRefModel)
def test_opening_gain_ref(tmp_path):
# First make test reference file
file_path = tmp_path / 'testgain.asdf'
utils.mk_gain(filepath=file_path)
gain = datamodels.open(file_path)
assert gain.meta.instrument.optical_element == 'F158'
assert isinstance(gain, datamodels.GainRefModel)
def test_opening_ramp_ref(tmp_path):
# First make test reference file
file_path = tmp_path / 'testramp.asdf'
utils.mk_ramp(filepath=file_path)
ramp = datamodels.open(file_path)
assert ramp.meta.instrument.optical_element == 'F062'
assert isinstance(ramp, datamodels.RampModel)
def test_opening_rampfitoutput_ref(tmp_path):
# First make test reference file
file_path = tmp_path / 'testrampfitoutput.asdf'
utils.mk_rampfitoutput(filepath=file_path)
rampfitoutput = datamodels.open(file_path)
assert rampfitoutput.meta.instrument.optical_element == 'F062'
assert isinstance(rampfitoutput, datamodels.RampFitOutputModel)
def test_flat_model(tmp_path):
# Set temporary asdf file
file_path = tmp_path / "test.asdf"
meta = {}
utils.add_ref_common(meta)
meta['reftype'] = "FLAT"
flatref = stnode.FlatRef()
flatref['meta'] = meta
flatref.meta.instrument['optical_element'] = 'F062'
shape = (4096, 4096)
flatref['data'] = np.zeros(shape, dtype=np.float32)
flatref['dq'] = np.zeros(shape, dtype=np.uint16)
flatref['dq_def'] = np.zeros(10, dtype=table_definitions.DQ_DEF_DTYPE)
flatref['err'] = np.zeros(shape, dtype=np.float32)
# Testing flat file asdf file
with asdf.AsdfFile(meta) as af:
af.tree = {'roman': flatref}
af.write_to(file_path)
# Test that asdf file opens properly
with datamodels.open(file_path) as model:
with pytest.warns(None):
model.validate()
# Confirm that asdf file is opened as flat file model
assert isinstance(model, datamodels.FlatRefModel)
def test_opening_readnoise_ref(tmp_path):
# First make test reference file
file_path = tmp_path / 'testreadnoise.asdf'
utils.mk_readnoise(filepath=file_path)
readnoise = datamodels.open(file_path)
assert readnoise.meta.instrument.optical_element == 'F158'
assert isinstance(readnoise, datamodels.ReadnoiseRefModel)
def test_add_model_attribute(tmp_path):
# First make test reference file
file_path = tmp_path / 'testreadnoise.asdf'
utils.mk_readnoise(filepath=file_path)
readnoise = datamodels.open(file_path)
readnoise['new_attribute'] = 77
assert readnoise.new_attribute == 77
with pytest.raises(ValueError):
readnoise['_underscore'] = 'bad'
file_path2 = tmp_path / 'testreadnoise2.asdf'
readnoise.save(file_path2)
readnoise2 = datamodels.open(file_path2)
assert readnoise2.new_attribute == 77
readnoise2.new_attribute = 88
assert readnoise2.new_attribute == 88
with pytest.raises(ValidationError):
readnoise['data'] = 'bad_data_value'
def test_opening_level1_science_raw(tmp_path):
# First make test reference file
file_path = tmp_path / 'testwfi_science_raw.asdf'
utils.mk_level1_science_raw(filepath=file_path)
wfi_science_raw = datamodels.open(file_path)
assert wfi_science_raw.meta.instrument.optical_element == 'F062'
assert isinstance(wfi_science_raw, datamodels.ScienceRawModel)
def test_opening_level2_image(tmp_path):
# First make test reference file
file_path = tmp_path / 'testwfi_image.asdf'
utils.mk_level2_image(filepath=file_path)
wfi_image = datamodels.open(file_path)
assert wfi_image.meta.instrument.optical_element == 'F062'
assert isinstance(wfi_image, datamodels.ImageModel)
def process(self, input):
# Open the input data model
with rdd.open(input) as input_model:
# Get the name of the dark reference file to use
self.dark_name = self.get_reference_file(input_model, 'dark')
self.log.info('Using DARK reference file %s', self.dark_name)
# Open dark model
dark_model = rdd.open(self.dark_name)
# Temporary patch to utilize stcal dark step until MA table support is fully implemented
if 'ngroups' not in dark_model.meta.exposure:
dark_model.meta.exposure['ngroups'] = dark_model.data.shape[0]
if 'nframes' not in dark_model.meta.exposure:
dark_model.meta.exposure['nframes'] = input_model.meta.exposure.nframes
if 'groupgap' not in dark_model.meta.exposure:
dark_model.meta.exposure['groupgap'] = input_model.meta.exposure.groupgap
# Reshaping data variables for stcal compatibility
input_model.data = input_model.data.astype(np.float32)[np.newaxis, :]
input_model.groupdq = input_model.groupdq[np.newaxis, :]
input_model.err = input_model.err[np.newaxis, :]
# Do the dark correction
out_data, dark_data = dark_sub.do_correction(
input_model, dark_model, self.dark_output
)
# Save dark data to file
if dark_data is not None and dark_data.save:
save_dark_data_as_dark_model(dark_data, dark_model)
dark_model.close()
# Reshaping data variables back from stcal
input_model.data = input_model.data[0]
input_model.groupdq = input_model.groupdq[0]
input_model.err = input_model.err[0]
# Convert data to RampModel
out_ramp = dark_output_data_as_ramp_model(out_data, input_model)
dark_model.close()
return out_ramp
def test_opening_wfi_img_photom_ref(tmp_path):
# First make test reference file
file_path = tmp_path / 'testwfi_img_photom.asdf'
utils.mk_wfi_img_photom(filepath=file_path)
wfi_img_photom = datamodels.open(file_path)
assert wfi_img_photom.meta.instrument.optical_element == 'F158'
assert isinstance(wfi_img_photom, datamodels.WfiImgPhotomRefModel)
def test_memmap(tmp_path):
file_path = tmp_path / "test.asdf"
with asdf.AsdfFile() as af:
af.tree = {'roman': utils.mk_level2_image()}
af.tree['roman'].data = np.zeros((
400,
400,
), dtype=np.float32)
af.write_to(file_path)
with datamodels.open(file_path, memmap=True) as model:
assert isinstance(model.data.base, np.memmap)
with datamodels.open(file_path, memmap=False) as model:
assert not isinstance(model.data.base, np.memmap)
# Default should be false:
with datamodels.open(file_path) as model:
assert not isinstance(model.data.base, np.memmap)
def test_model_input(tmp_path):
file_path = tmp_path / "test.asdf"
data = np.random.uniform(size=(1024, 1024)).astype(np.float32)
with asdf.AsdfFile() as af:
af.tree = {'roman': utils.mk_level2_image()}
af.tree['roman'].meta['bozo'] = 'clown'
af.tree['roman'].data = data
af.write_to(file_path)
original_model = datamodels.open(file_path)
reopened_model = datamodels.open(original_model)
# It's essential that we get a new instance so that the original
# model can be closed without impacting the new model.
assert reopened_model is not original_model
assert_array_equal(original_model.data, data)
original_model.close()
assert_array_equal(reopened_model.data, data)
reopened_model.close()
def process(self, input):
reference_file_names = {}
with rdm.open(input) as input_model:
for reftype in self.reference_file_types:
log.info(f'reftype, {reftype}')
reffile = self.get_reference_file(input_model, reftype)
reference_file_names[reftype] = reffile if reffile else ""
log.debug(
f'reference files used in assign_wcs: {reference_file_names}')
result = load_wcs(input_model, reference_file_names)
return result
def test_datamodel_info_search(capsys):
wfi_science_raw = utils.mk_level1_science_raw()
af = asdf.AsdfFile()
af.tree = {'roman': wfi_science_raw}
dm = datamodels.open(af)
dm.info(max_rows=200)
captured = capsys.readouterr()
assert "optical_element" in captured.out
result = dm.search('optical_element')
assert 'F062' in repr(result)
assert result.node == 'F062'
def process(self, input):
with rdd.open(input, mode='rw') as input_model:
max_cores = self.maximum_cores
readnoise_filename = self.get_reference_file(
input_model, 'readnoise')
gain_filename = self.get_reference_file(input_model, 'gain')
input_model.data = input_model.data[np.newaxis, :]
input_model.data.dtype = np.float32
input_model.groupdq = input_model.groupdq[np.newaxis, :]
input_model.err = input_model.err[np.newaxis, :]
log.info('Using READNOISE reference file: %s', readnoise_filename)
readnoise_model = rdd.open(readnoise_filename, mode='rw')
log.info('Using GAIN reference file: %s', gain_filename)
gain_model = rdd.open(gain_filename, mode='rw')
log.info('Using algorithm = %s' % self.algorithm)
log.info('Using weighting = %s' % self.weighting)
buffsize = ramp_fit.BUFSIZE
image_info, integ_info, opt_info, gls_opt_model = ramp_fit.ramp_fit(
input_model, buffsize, self.save_opt, readnoise_model.data,
gain_model.data, self.algorithm, self.weighting, max_cores,
dqflags.pixel)
readnoise_model.close()
gain_model.close()
# Save the OLS optional fit product, if it exists
if opt_info is not None:
opt_model = create_optional_results_model(input_model, opt_info)
self.save_model(opt_model, 'fitopt', output_file=self.opt_name)
if image_info is not None:
out_model = create_image_model(input_model, image_info)
out_model.meta.cal_step.ramp_fit = 'COMPLETE'
return out_model
def process(self, input):
""" Process the Roman WFI data """
log.info('Starting Roman exposure calibration pipeline ...')
if isinstance(input, str):
input_filename = basename(input)
# open the input file
input = rdd.open(input)
# propagate output_dir to steps that might need it
# self.dark_current.output_dir = self.output_dir
# self.ramp_fit.output_dir = self.output_dir
log.debug('Exposure Processing a WFI exposure')
self.dq_init.suffix = 'dq_init'
result = self.dq_init(input)
if input_filename:
result.meta.filename = input_filename
result = self.saturation(result)
result = self.linearity(result)
result = self.dark_current(result)
result = self.jump(result)
result = self.rampfit(result)
result = self.assign_wcs(result)
if result.meta.exposure.type == 'WFI_IMAGE':
result = self.flatfield(result)
else:
log.info('Flat Field step is being SKIPPED')
result.meta.cal_step.flat_field = 'SKIPPED'
result = self.photom(result)
# setup output_file for saving
self.setup_output(result)
log.info('Roman exposure calibration pipeline ending...')
return result
def test_invalid_input():
with pytest.raises(TypeError):
datamodels.open(fits.HDUList())
def process(self, input):
# Open input as a Roman DataModel (single integration; 3D arrays)
with rdd.open(input) as input_model:
# Extract the needed info from the Roman Data Model
meta = input_model.meta
r_data = input_model.data
r_gdq = input_model.groupdq
r_pdq = input_model.pixeldq
r_err = input_model.err
result = input_model
frames_per_group = meta.exposure.nframes
# Modify the arrays for input into the 'common' jump (4D)
data = np.copy(r_data[np.newaxis, :])
gdq = r_gdq[np.newaxis, :]
pdq = r_pdq[np.newaxis, :]
err = np.copy(r_err[np.newaxis, :])
tstart = time.time()
# Check for an input model with NGROUPS<=2
ngroups = data.shape[1]
if ngroups <= 2:
self.log.warning('Cannot apply jump detection as NGROUPS<=2;')
self.log.warning('Jump step will be skipped')
result = input_model
result.meta.cal_step.jump = 'SKIPPED'
return result
# Retrieve the parameter values
rej_thresh = self.rejection_threshold
three_grp_rej_thresh = self.three_group_rejection_threshold
four_grp_rej_thresh = self.four_group_rejection_threshold
max_cores = self.maximum_cores
max_jump_to_flag_neighbors = self.max_jump_to_flag_neighbors
min_jump_to_flag_neighbors = self.min_jump_to_flag_neighbors
flag_4_neighbors = self.flag_4_neighbors
self.log.info('CR rejection threshold = %g sigma', rej_thresh)
if self.maximum_cores != 'none':
self.log.info('Maximum cores to use = %s', max_cores)
# Get the gain and readnoise reference files
gain_filename = self.get_reference_file(input_model, 'gain')
self.log.info('Using GAIN reference file: %s', gain_filename)
gain_model = rdd.GainRefModel(gain_filename)
gain_2d = gain_model.data
readnoise_filename = self.get_reference_file(input_model,
'readnoise')
self.log.info('Using READNOISE reference file: %s',
readnoise_filename)
readnoise_model = rdd.ReadnoiseRefModel(readnoise_filename)
# This is to clear the WRITEABLE=False flag?
readnoise_2d = np.copy(readnoise_model.data)
# DG 0810/21: leave for now; make dqflags changes in a later,
# separate PR
dqflags_d = {} # Dict of DQ flags
dqflags_d = {
"GOOD": dqflags.group["GOOD"],
"DO_NOT_USE": dqflags.group["DO_NOT_USE"],
"SATURATED": dqflags.group["SATURATED"],
"JUMP_DET": dqflags.group["JUMP_DET"],
"NO_GAIN_VALUE": dqflags.pixel["NO_GAIN_VALUE"]
}
gdq, pdq = detect_jumps(frames_per_group, data, gdq, pdq, err,
gain_2d, readnoise_2d, rej_thresh,
three_grp_rej_thresh, four_grp_rej_thresh,
max_cores,
max_jump_to_flag_neighbors,
min_jump_to_flag_neighbors,
flag_4_neighbors,
dqflags_d)
gdq = gdq[0, :, :, :]
pdq = pdq[0, :, :]
result.groupdq = gdq
result.pixeldq = pdq
gain_model.close()
readnoise_model.close()
tstop = time.time()
self.log.info('The execution time in seconds: %f', tstop - tstart)
result.meta.cal_step.jump = 'COMPLETE'
return result
