def test_opening_level2_image(tmp_path):
# First make test reference file
file_path = tmp_path / 'testwfi_image.asdf'
utils.mk_level2_image(filepath=file_path)
wfi_image = datamodels.open(file_path)
assert wfi_image.meta.instrument.optical_element == 'F062'
assert isinstance(wfi_image, datamodels.ImageModel)
def test_core_schema(tmp_path):
# Set temporary asdf file
file_path = tmp_path / "test.asdf"
wfi_image = utils.mk_level2_image(arrays=(10, 10))
with asdf.AsdfFile() as af:
af.tree = {'roman': wfi_image}
with pytest.raises(ValidationError):
af.tree['roman'].meta.telescope = 'NOTROMAN'
af.tree['roman'].meta['telescope'] = 'NOTROMAN'
with pytest.raises(ValidationError):
af.write_to(file_path)
af.tree['roman'].meta.telescope = 'ROMAN'
af.write_to(file_path)
# Now mangle the file
with open(file_path, 'rb') as fp:
fcontents = fp.read()
romanloc = fcontents.find(bytes('ROMAN', 'utf-8'))
newcontents = fcontents[:romanloc] + \
bytes('X', 'utf-8') + fcontents[romanloc + 1:]
with open(file_path, 'wb') as fp:
fp.write(newcontents)
with pytest.raises(ValidationError):
with datamodels.open(file_path) as model:
pass
asdf.get_config().validate_on_read = False
with datamodels.open(file_path) as model:
assert model.meta.telescope == 'XOMAN'
asdf.get_config().validate_on_read = True
def test_apply_photom1():
"""Test apply_photom applies correct metadata"""
# Create sample WFI Level 2 science datamodel
input_model = testutil.mk_level2_image()
# Create photom reference datamodel
photom_model = create_photom_wfi_image(min_r=3.1, delta=0.1)
# Select optical element
input_model.meta.instrument.optical_element = "W146"
# Apply photom correction for optical element W146
output_model = photom.apply_photom(input_model, photom_model)
# Set reference photometry
area_ster = 2.31307642258977E-14 * u.steradian
area_a2 = 0.000984102303070964 * u.arcsecond * u.arcsecond
# Tests for pixel areas
assert (np.isclose(output_model.meta.photometry.pixelarea_steradians.value,
area_ster.value,
atol=1.e-7))
assert output_model.meta.photometry.pixelarea_steradians.unit == area_ster.unit
assert (np.isclose(output_model.meta.photometry.pixelarea_arcsecsq.value,
area_a2.value,
atol=1.e-7))
assert output_model.meta.photometry.pixelarea_arcsecsq.unit == area_a2.unit
# Set reference photometry
phot_ster = 3.5 * u.megajansky / u.steradian
phot_a2 = phot_ster.to(u.microjansky / u.arcsecond**2)
# Tests for photometry
assert (np.isclose(
output_model.meta.photometry.conversion_megajanskys.value,
phot_ster.value,
atol=1.e-7))
assert output_model.meta.photometry.conversion_megajanskys.unit == phot_ster.unit
assert (np.isclose(
output_model.meta.photometry.conversion_microjanskys.value,
phot_a2.value,
atol=1.e-7))
assert output_model.meta.photometry.conversion_microjanskys.unit == phot_a2.unit
# Set reference photometric uncertainty
muphot_ster = 0.175 * u.megajansky / u.steradian
muphot_a2 = muphot_ster.to(u.microjansky / u.arcsecond**2)
# Tests for photometric uncertainty
assert (np.isclose(
output_model.meta.photometry.conversion_megajanskys_uncertainty.value,
muphot_ster.value,
atol=1.e-7))
assert output_model.meta.photometry.conversion_megajanskys_uncertainty.unit == muphot_ster.unit
assert (np.isclose(
output_model.meta.photometry.conversion_microjanskys_uncertainty.value,
muphot_a2.value,
atol=1.e-7))
assert output_model.meta.photometry.conversion_microjanskys_uncertainty.unit == muphot_a2.unit
def test_path_input(tmp_path):
file_path = tmp_path / "test.asdf"
with asdf.AsdfFile() as af:
tree = utils.mk_level2_image()
af.tree = {'roman': tree}
af.write_to(file_path)
# Test with PurePath input:
with datamodels.open(file_path) as model:
assert model.meta.telescope == "ROMAN"
af = model._asdf
# When open creates the file pointer, it should be
# closed when the model is closed:
assert af._closed
# Test with string input:
with datamodels.open(str(file_path)) as model:
assert model.meta.telescope == "ROMAN"
af = model._asdf
assert af._closed
# Appropriate error when file is missing:
with pytest.raises(FileNotFoundError):
with datamodels.open(tmp_path / "missing.asdf"):
pass
def test_flatfield_step_interface(instrument, exptype):
"""Test that the basic inferface works for data requiring a FLAT reffile"""
shape = (20, 20)
wfi_image = testutil.mk_level2_image(shape=shape)
wfi_image.meta.instrument.name = instrument
wfi_image.meta.instrument.detector = 'WFI01'
wfi_image.meta.instrument.optical_element = 'F158'
wfi_image.meta.exposure.type = exptype
wfi_image.data = np.ones(shape, dtype=np.float32)
wfi_image.dq = np.zeros(shape, dtype=np.uint32)
wfi_image.err = np.zeros(shape, dtype=np.float32)
wfi_image.var_poisson = np.zeros(shape, dtype=np.float32)
wfi_image.var_rnoise = np.zeros(shape, dtype=np.float32)
wfi_image.var_flat = np.zeros(shape, dtype=np.float32)
wfi_image_model = ImageModel(wfi_image)
flatref = stnode.FlatRef()
meta = {}
testutil.add_ref_common(meta)
meta['instrument']['optical_element'] = 'F158'
meta['instrument']['detector'] = 'WFI01'
meta['reftype'] = 'FLAT'
flatref['meta'] = meta
flatref['data'] = np.ones(shape, dtype=np.float32)
flatref['dq'] = np.zeros(shape, dtype=np.uint16)
flatref['err'] = (np.random.random(shape) * 0.05).astype(np.float32)
flatref_model = FlatRefModel(flatref)
result = FlatFieldStep.call(wfi_image_model, override_flat=flatref_model)
assert (result.data == wfi_image.data).all()
assert result.var_flat.shape == shape
assert result.meta.cal_step.flat_field == 'COMPLETE'
def test_asdf_file_input():
tree = utils.mk_level2_image()
with asdf.AsdfFile() as af:
af.tree = {'roman': tree}
model = datamodels.open(af)
assert model.meta.telescope == 'ROMAN'
model.close()
def test_no_photom_match():
"""Test apply_photom warning for no match"""
# Create sample WFI Level 2 science datamodel
input_model = testutil.mk_level2_image()
# Create photom reference datamodel
photom_model = create_photom_wfi_image(min_r=3.1, delta=0.1)
# Remove key for failed test (that won't fail validation)
photom_model.phot_table.pop("W146")
# Select optical element
input_model.meta.instrument.optical_element = "W146"
# Set bad values which would be overwritten by apply_photom
input_model.meta.photometry.pixelarea_steradians = -1.0 * u.sr
input_model.meta.photometry.conversion_megajanskys = -1.0 * u.megajansky / u.steradian
input_model.meta.photometry.conversion_microjanskys_uncertainty = \
-1.0 * u.microjansky / u.arcsecond ** 2
with warnings.catch_warnings(record=True) as caught:
# Look for now non existent W146 optical element
output_model = photom.apply_photom(input_model, photom_model)
# Assert warning key matches that of the input file
assert str(caught[0].message).split(
)[-1] == input_model.meta.instrument.optical_element
# Assert that photom elements are not updated
assert output_model.meta.photometry.pixelarea_steradians == -1.0 * u.sr
assert output_model.meta.photometry.conversion_megajanskys == \
-1.0 * u.megajansky / u.steradian
assert output_model.meta.photometry.conversion_microjanskys_uncertainty == \
-1.0 * u.microjansky / u.arcsecond ** 2
def create_image():
l2 = testutil.mk_level2_image()
l2.meta.wcsinfo.v2_ref = -503
l2.meta.wcsinfo.v3_ref = -318
l2.meta.wcsinfo.ra_ref = 156
l2.meta.wcsinfo.dec_ref = 54.2
l2.meta.wcsinfo.vparity = -1
l2.meta.wcsinfo.roll_ref = 0.15
l2im = rdm.ImageModel(l2)
return l2im
def test_photom_step_interface_spectroscopic(instrument, exptype):
"""Test apply_photom properly populates photometric keywords for spectroscopic data"""
# Create a small area for the file
shape = (20, 20)
# Create input node
wfi_image = testutil.mk_level2_image(shape=shape)
# Select exposure type and optical element
wfi_image.meta.exposure.type = "WFI_PRISM"
wfi_image.meta.instrument.optical_element = "PRISM"
# Set photometric values for spectroscopic data
wfi_image.meta.photometry.pixelarea_steradians = 2.31307642258977E-14 * u.steradian
wfi_image.meta.photometry.pixelarea_arcsecsq = 0.000984102303070964 * u.arcsecond * u.arcsecond
wfi_image.meta.photometry.conversion_megajanskys = None
wfi_image.meta.photometry.conversion_megajanskys_uncertainty = None
wfi_image.meta.photometry.conversion_microjanskys = None
wfi_image.meta.photometry.conversion_microjanskys_uncertainty = None
# Create input model
wfi_image_model = ImageModel(wfi_image)
# Create photom model
photom = testutil.mk_wfi_img_photom()
photom_model = WfiImgPhotomRefModel(photom)
# Run photom correction step
result = PhotomStep.call(wfi_image_model, override_photom=photom_model)
# Test that the data has not changed
assert (np.allclose(result.data, wfi_image_model.data, rtol=1.e-7))
# Test that keywords are properly preserved
assert result.meta.photometry.conversion_megajanskys is None
assert result.meta.photometry.conversion_microjanskys is None
assert result.meta.photometry.conversion_megajanskys_uncertainty is None
assert result.meta.photometry.conversion_microjanskys_uncertainty is None
# Set reference pixel areas
area_ster = 2.31307642258977E-14 * u.steradian
area_a2 = 0.000984102303070964 * u.arcsecond * u.arcsecond
# Tests for pixel areas
assert (np.isclose(result.meta.photometry.pixelarea_steradians.value,
area_ster.value,
atol=1.e-7))
assert result.meta.photometry.pixelarea_steradians.unit == area_ster.unit
assert (np.isclose(result.meta.photometry.pixelarea_arcsecsq.value,
area_a2.value,
atol=1.e-7))
assert result.meta.photometry.pixelarea_arcsecsq.unit == area_a2.unit
def test_level2_image():
wfi_image = utils.mk_level2_image()
assert wfi_image.data.dtype == np.float32
assert wfi_image.dq.dtype == np.uint32
assert wfi_image.err.dtype == np.float32
assert wfi_image.var_poisson.dtype == np.float32
assert wfi_image.var_rnoise.dtype == np.float32
assert wfi_image.var_flat.dtype == np.float32
assert type(wfi_image.cal_logs[0]) == str
# Test validation
wfi_image_model = datamodels.ImageModel(wfi_image)
assert wfi_image_model.validate() is None
def test_spectroscopic_skip(instrument, exptype):
wfi_image = testutil.mk_level2_image()
wfi_image.meta.instrument.name = instrument
wfi_image.meta.instrument.detector = 'WFI01'
wfi_image.meta.instrument.optical_element = 'F158'
wfi_image.meta.exposure.start_time = Time('2020-01-01T11:11:11.110')
wfi_image.meta.exposure.end_time = Time('2020-01-01T11:33:11.110')
wfi_image.meta.exposure.type = exptype
wfi_image_model = ImageModel(wfi_image)
result = FlatFieldStep.call(wfi_image_model)
assert result.meta.cal_step.flat_field == 'SKIPPED'
def test_open_model(step_class, tmp_path):
"""
Test that the class is properly hooked up to datamodels.open.
More comprehensive tests can be found in romancal.datamodels.tests,
this is just a smoke test of the integration.
"""
file_path = tmp_path / "test.asdf"
with asdf.AsdfFile() as af:
imod = mk_level2_image(arrays=(20, 20))
af.tree = {'roman': imod}
af.write_to(file_path)
step = step_class()
with step.open_model(file_path) as model:
assert model.meta.telescope == "ROMAN"
def test_get_reference_file(step_class):
"""
Test that CRDS is properly integrated.
"""
im = mk_level2_image(arrays=(20, 20))
# This will be brittle while we're using the dev server.
# If this test starts failing mysteriously, check the
# metadata values against the flat rmap.
im.meta.instrument.optical_element = "F158"
im.meta.observation.start_time = Time('2021-01-01T12:00:00')
model = ImageModel(im)
step = step_class()
reference_path = step.get_reference_file(model, "flat")
with step.open_model(reference_path) as reference_model:
assert isinstance(reference_model, FlatRefModel)
def test_memmap(tmp_path):
file_path = tmp_path / "test.asdf"
with asdf.AsdfFile() as af:
af.tree = {'roman': utils.mk_level2_image()}
af.tree['roman'].data = np.zeros((
400,
400,
), dtype=np.float32)
af.write_to(file_path)
with datamodels.open(file_path, memmap=True) as model:
assert isinstance(model.data.base, np.memmap)
with datamodels.open(file_path, memmap=False) as model:
assert not isinstance(model.data.base, np.memmap)
# Default should be false:
with datamodels.open(file_path) as model:
assert not isinstance(model.data.base, np.memmap)
def test_model_input(tmp_path):
file_path = tmp_path / "test.asdf"
data = np.random.uniform(size=(1024, 1024)).astype(np.float32)
with asdf.AsdfFile() as af:
af.tree = {'roman': utils.mk_level2_image()}
af.tree['roman'].meta['bozo'] = 'clown'
af.tree['roman'].data = data
af.write_to(file_path)
original_model = datamodels.open(file_path)
reopened_model = datamodels.open(original_model)
# It's essential that we get a new instance so that the original
# model can be closed without impacting the new model.
assert reopened_model is not original_model
assert_array_equal(original_model.data, data)
original_model.close()
assert_array_equal(reopened_model.data, data)
reopened_model.close()
def test_apply_photom2():
"""Test apply_photom does not change data values"""
# Create sample WFI Level 2 science datamodel
input_model = testutil.mk_level2_image()
# Create photom reference datamodel
photom_model = create_photom_wfi_image(min_r=3.1, delta=0.1)
# Apply photom correction
output_model = photom.apply_photom(input_model, photom_model)
# Select pixel for comparison
shape = input_model.data.shape
ix = shape[1] // 2
iy = shape[0] // 2
# Test that the data has not changed
assert (np.allclose(output_model.data[iy, ix],
input_model.data[iy, ix],
rtol=1.e-7))
def test_crds_temporal_match(instrument, exptype):
"""Test that the basic inferface works for data requiring a FLAT reffile"""
wfi_image = testutil.mk_level2_image()
wfi_image.meta.instrument.name = instrument
wfi_image.meta.instrument.detector = 'WFI01'
wfi_image.meta.instrument.optical_element = 'F158'
wfi_image.meta.exposure.start_time = Time('2020-01-01T11:11:11.110')
wfi_image.meta.exposure.end_time = Time('2020-01-01T11:33:11.110')
wfi_image.meta.exposure.type = exptype
wfi_image_model = ImageModel(wfi_image)
step = FlatFieldStep()
ref_file_path = step.get_reference_file(wfi_image_model, "flat")
wfi_image_model.meta.exposure.start_time = Time('2021-08-11T11:11:11.110')
wfi_image_model.meta.exposure.end_time = Time('2021-08-11T11:33:11.110')
ref_file_path_b = step.get_reference_file(wfi_image_model, "flat")
assert ("/".join(ref_file_path.rsplit("/", 1)[1:])) != \
("/".join(ref_file_path_b.rsplit("/", 1)[1:]))
def test_photom_step_interface(instrument, exptype):
"""Test that the basic inferface works for data requiring a photom reffile"""
# Create a small area for the file
shape = (20, 20)
# Create input model
wfi_image = testutil.mk_level2_image(shape=shape)
wfi_image_model = ImageModel(wfi_image)
# Create photom model
photom = testutil.mk_wfi_img_photom()
photom_model = WfiImgPhotomRefModel(photom)
# Run photom correction step
result = PhotomStep.call(wfi_image_model, override_photom=photom_model)
assert (result.data == wfi_image.data).all()
assert result.data.shape == shape
if exptype == "WFI_IMAGE":
assert result.meta.cal_step.photom == 'COMPLETE'
else:
assert result.meta.cal_step.photom == 'SKIPPED'
def process(self):
self.log.warning("Splines failed to reticulate")
return ImageModel(mk_level2_image(shape=(20, 20)))
