def test_open_fits_model_s3(s3_root_dir):
path = str(s3_root_dir.join("test.fits"))
with DataModel() as dm:
dm.save(path)
model = DataModel("s3://test-s3-data/test.fits")
assert isinstance(model, DataModel)
def test_history_from_fits(tmpdir):
tmpfits = str(tmpdir.join('tmp.fits'))
header = fits.Header()
header['HISTORY'] = "First entry"
header['HISTORY'] = "Second entry"
fits.writeto(tmpfits, np.array([]), header, overwrite=True)
with DataModel(tmpfits) as m:
assert m.history == [{
'description': 'First entry'
}, {
'description': 'Second entry'
}]
del m.history[0]
m.history.append(HistoryEntry({'description': "Third entry"}))
assert m.history == [{
'description': "Second entry"
}, {
'description': "Third entry"
}]
m.save(tmpfits)
with DataModel(tmpfits) as m:
assert m.history == [{
'description': "Second entry"
}, {
'description': "Third entry"
}]
def test_table_with_unsigned_int():
schema = {
'title': 'Test data model',
'$schema': 'http://stsci.edu/schemas/fits-schema/fits-schema',
'type': 'object',
'properties': {
'meta': {
'type': 'object',
'properties': {}
},
'test_table': {
'title': 'Test table',
'fits_hdu': 'TESTTABL',
'datatype': [
{'name': 'FLOAT64_COL', 'datatype': 'float64'},
{'name': 'UINT32_COL', 'datatype': 'uint32'}
]
}
}
}
with DataModel(schema=schema) as dm:
float64_info = np.finfo(np.float64)
float64_arr = np.random.uniform(size=(10,))
float64_arr[0] = float64_info.min
float64_arr[-1] = float64_info.max
uint32_info = np.iinfo(np.uint32)
uint32_arr = np.random.randint(uint32_info.min, uint32_info.max + 1, size=(10,), dtype=np.uint32)
uint32_arr[0] = uint32_info.min
uint32_arr[-1] = uint32_info.max
test_table = np.array(list(zip(float64_arr, uint32_arr)), dtype=dm.test_table.dtype)
def assert_table_correct(model):
for idx, (col_name, col_data) in enumerate([('float64_col', float64_arr), ('uint32_col', uint32_arr)]):
# The table dtype and field dtype are stored separately, and may not
# necessarily agree.
assert np.can_cast(model.test_table.dtype[idx], col_data.dtype, 'equiv')
assert np.can_cast(model.test_table.field(col_name).dtype, col_data.dtype, 'equiv')
assert np.array_equal(model.test_table.field(col_name), col_data)
# The datamodel casts our array to FITS_rec on assignment, so here we're
# checking that the data survived the casting.
dm.test_table = test_table
assert_table_correct(dm)
# Confirm that saving the table (and converting the uint32 values to signed int w/TZEROn)
# doesn't mangle the data.
dm.save(TMP_FITS)
assert_table_correct(dm)
# Confirm that the data loads from the file intact (converting the signed ints back to
# the appropriate uint32 values).
with DataModel(TMP_FITS, schema=schema) as dm2:
assert_table_correct(dm2)
def test_open_asdf_no_datamodel_class(tmp_path, suffix):
path = str(tmp_path / f"no_model.{suffix}")
model = DataModel()
model.save(path)
# Note: only the fits open emits a "model_type not found" warning. Both
# fits and asdf should behave the same
with datamodels.open(path) as m:
assert isinstance(m, DataModel)
def test_ad_hoc_attributes(filename, tmp_path):
"""
Test that attributes unrecognized by the schema
can still be assigned and written.
"""
file_path = tmp_path / filename
with DataModel() as dm:
dm.meta.foo = {'a': 42, 'b': ['a', 'b', 'c']}
dm.save(file_path)
with DataModel(file_path) as dm2:
assert dm2.meta.foo == {'a': 42, 'b': ['a', 'b', 'c']}
def test_extra_fits():
path = os.path.join(ROOT_DIR, "headers.fits")
assert os.path.exists(path)
with DataModel(path) as dm:
assert 'BITPIX' not in _header_to_dict(dm.extra_fits.PRIMARY.header)
assert _header_to_dict(dm.extra_fits.PRIMARY.header)['SCIYSTRT'] == 705
dm2 = dm.copy()
dm2.to_fits(TMP_FITS, overwrite=True)
with DataModel(TMP_FITS) as dm:
assert 'BITPIX' not in _header_to_dict(dm.extra_fits.PRIMARY.header)
assert _header_to_dict(dm.extra_fits.PRIMARY.header)['SCIYSTRT'] == 705
def test_fits_without_sci():
from astropy.io import fits
schema = {
"allOf": [
mschema.load_schema(os.path.join(os.path.dirname(__file__),
"../schemas/core.schema.yaml"),
resolve_references=True), {
"type": "object",
"properties": {
"coeffs": {
'max_ndim': 1,
'fits_hdu': 'COEFFS',
'datatype': 'float32'
}
}
}
]
}
fits = fits.HDUList([
fits.PrimaryHDU(),
fits.ImageHDU(name='COEFFS', data=np.array([0.0], np.float32))
])
with DataModel(fits, schema=schema) as dm:
assert_array_equal(dm.coeffs, [0.0])
def test_historylist_methods():
m = DataModel()
h1 = m.history
info = "First entry"
h1.append(info)
assert h1 == info, "Append new history entry"
h2 = m.history
assert h2 == info, "Two history lists point to the same object"
assert len(h1) == 1, "Length of a history list"
entry = h1[0]
assert entry["description"] == info, "Get history list item"
info += " for real"
h1[0] = info
assert h1 == info, "Set history list item"
del h1[0]
assert len(h1) == 0, "Delete history list item"
info = ("First entry", "Second_entry", "Third entry")
h1.extend(info)
assert len(h1) == 3, "Length of extended history list"
assert h1 == info, "Contents of extended history list"
for entry, item in zip(h1, info):
assert entry["description"] == item, "Iterate over history list"
h1.clear()
assert len(h1) == 0, "Clear history list"
def test_open_asdf_model(init):
# Open an empty asdf file, pass extra arguments
with DataModel(init=init,
ignore_version_mismatch=False,
ignore_unrecognized_tag=True) as model:
assert not model._asdf._ignore_version_mismatch
assert model._asdf._ignore_unrecognized_tag
def _create_source():
dm = DataModel(FITS_FILE)
assert dm.meta.instrument.name == 'MIRI'
dm.meta.instrument.name = 'NIRCAM'
dm.meta.subarray.xstart = 42
return dm
def test_to_flat_dict():
with DataModel() as x:
x.meta.origin = 'FOO'
assert x['meta.origin'] == 'FOO'
d = x.to_flat_dict()
assert d['meta.origin'] == 'FOO'
def test_open_asdf_model(tmp_path):
# Open an empty asdf file, pass extra arguments
with DataModel(ignore_version_mismatch=False,
ignore_unrecognized_tag=True) as model:
assert not model._asdf._ignore_version_mismatch
assert model._asdf._ignore_unrecognized_tag
file_path = tmp_path / "test.asdf"
with asdf.AsdfFile() as af:
af.write_to(file_path)
with DataModel(file_path,
ignore_version_mismatch=False,
ignore_unrecognized_tag=True) as model:
assert not model._asdf._ignore_version_mismatch
assert model._asdf._ignore_unrecognized_tag
def test_open_asdf_s3(s3_root_dir):
"""Test opening a model from an ASDF file on S3"""
path = str(s3_root_dir.join("test.asdf"))
with DataModel() as dm:
dm.save(path)
with stdatamodels.open("s3://test-s3-data/test.asdf") as m:
assert isinstance(m, DataModel)
def make_models(tmpdir_factory):
"""Create basic models
Returns
-------
path_just_fits, path_model : (str, str)
`path_just_fits` is a FITS file of `DataModel` without the ASDF extension.
`path_model` is a FITS file of `DataModel` with the ASDF extension.
"""
path = tmpdir_factory.mktemp('skip_fits_update')
path_just_fits = str(path / 'just_fits.fits')
path_model = str(path / 'model.fits')
primary_hdu = fits.PrimaryHDU()
primary_hdu.header['exp_type'] = 'NRC_IMAGE'
hduls = fits.HDUList([primary_hdu])
hduls.writeto(path_just_fits)
model = DataModel(hduls)
model.save(path_model)
return {'just_fits': path_just_fits, 'model': path_model}
def test_metadata_from_fits():
from astropy.io import fits
mask = np.array([[0, 1], [2, 3]])
fits.ImageHDU(data=mask, name='DQ').writeto(TMP_FITS, overwrite=True)
with DataModel(init=TMP_FITS) as dm:
dm.save(TMP_FITS2)
with fits.open(TMP_FITS2, memmap=False) as hdulist:
assert hdulist[2].name == 'ASDF'
def test_add_schema_entry():
with DataModel(strict_validation=True) as dm:
dm.add_schema_entry('meta.foo.bar', {'enum': ['foo', 'bar', 'baz']})
dm.meta.foo.bar
dm.meta.foo.bar = 'bar'
try:
dm.meta.foo.bar = 'what?'
except jsonschema.ValidationError:
pass
else:
assert False
def test_extra_fits(tmp_path):
file_path = tmp_path / "test.fits"
with FitsModel() as dm:
dm.save(file_path)
with fits.open(file_path) as hdul:
hdul[0].header["FOO"] = "BAR"
hdul.writeto(file_path, overwrite=True)
with DataModel(file_path) as dm:
assert any(h for h in dm.extra_fits.PRIMARY.header
if h == ["FOO", "BAR", ""])
def test_stringify(tmpdir):
im = DataModel()
assert str(im) == '<DataModel>'
im = ImageModel((10, 100))
assert str(im) == '<ImageModel(10, 100)>'
path = str(tmpdir.join("nircam_mask.fits"))
m = MaskModel((2048, 2048))
m.save(path)
m.close()
with MaskModel(path) as im:
assert str(im) == '<MaskModel(2048, 2048) from nircam_mask.fits>'
def test_stringify(tmp_path):
dm = DataModel()
assert str(dm) == '<DataModel>'
dm = BasicModel((10, 100))
assert str(dm) == '<BasicModel(10, 100)>'
file_path = tmp_path / "test.asdf"
dm.save(file_path)
dm.close()
with BasicModel(file_path) as dm:
assert str(dm) == '<BasicModel(10, 100) from test.asdf>'
def test_replace_table():
from astropy.io import fits
schema_narrow = {
"allOf": [
mschema.load_schema(os.path.join(os.path.dirname(__file__),
"../schemas/core.schema.yaml"),
resolve_references=True), {
"type": "object",
"properties": {
"data": {
"title":
"relative sensitivity table",
"fits_hdu":
"RELSENS",
"datatype": [{
"name":
"TYPE",
"datatype": ["ascii", 16]
}, {
"name": "T_OFFSET",
"datatype": "float32"
}, {
"name": "DECAY_PEAK",
"datatype": "float32"
}, {
"name": "DECAY_FREQ",
"datatype": "float32"
}, {
"name": "TAU",
"datatype": "float32"
}]
}
}
}
]
}
schema_wide = {
"allOf": [
mschema.load_schema(os.path.join(os.path.dirname(__file__),
"../schemas/core.schema.yaml"),
resolve_references=True), {
"type": "object",
"properties": {
"data": {
"title":
"relative sensitivity table",
"fits_hdu":
"RELSENS",
"datatype": [{
"name":
"TYPE",
"datatype": ["ascii", 16]
}, {
"name": "T_OFFSET",
"datatype": "float64"
}, {
"name": "DECAY_PEAK",
"datatype": "float64"
}, {
"name": "DECAY_FREQ",
"datatype": "float64"
}, {
"name": "TAU",
"datatype": "float64"
}]
}
}
}
]
}
x = np.array([("string", 1., 2., 3., 4.)],
dtype=[('TYPE', 'S16'), ('T_OFFSET', np.float32),
('DECAY_PEAK', np.float32), ('DECAY_FREQ', np.float32),
('TAU', np.float32)])
m = DataModel(schema=schema_narrow)
m.data = x
m.to_fits(TMP_FITS, overwrite=True)
with fits.open(TMP_FITS, memmap=False) as hdulist:
assert records_equal(x, np.asarray(hdulist[1].data))
assert hdulist[1].data.dtype[1].str == '>f4'
assert hdulist[1].header['TFORM2'] == 'E'
with DataModel(TMP_FITS, schema=schema_wide) as m:
m.to_fits(TMP_FITS2, overwrite=True)
with fits.open(TMP_FITS2, memmap=False) as hdulist:
assert records_equal(x, np.asarray(hdulist[1].data))
assert hdulist[1].data.dtype[1].str == '>f8'
assert hdulist[1].header['TFORM2'] == 'D'
def test_base_model_has_no_arrays():
with pytest.raises(AttributeError):
with DataModel() as dm:
dm.data
def test_hasattr():
model = DataModel()
assert model.meta.hasattr('date')
assert not model.meta.hasattr('filename')
def test_datamodel_raises_filenotfound():
with pytest.raises(FileNotFoundError):
DataModel(init='file_does_not_exist.fits')
def test_subarray():
with DataModel(FITS_FILE) as dm:
dm.meta.subarray.xstart
def test_replace_table(tmp_path):
file_path = tmp_path / "test.fits"
file_path2 = tmp_path / "test2.fits"
schema_narrow = {
"allOf": [
asdf.schema.load_schema("http://example.com/schemas/core_metadata",
resolve_references=True), {
"type": "object",
"properties": {
"data": {
"title":
"relative sensitivity table",
"fits_hdu":
"RELSENS",
"datatype": [{
"name":
"TYPE",
"datatype": ["ascii", 16]
}, {
"name":
"T_OFFSET",
"datatype":
"float32"
}, {
"name":
"DECAY_PEAK",
"datatype":
"float32"
}, {
"name":
"DECAY_FREQ",
"datatype":
"float32"
}, {
"name":
"TAU",
"datatype":
"float32"
}]
}
}
}
]
}
schema_wide = {
"allOf": [
asdf.schema.load_schema("http://example.com/schemas/core_metadata",
resolve_references=True), {
"type": "object",
"properties": {
"data": {
"title":
"relative sensitivity table",
"fits_hdu":
"RELSENS",
"datatype": [{
"name":
"TYPE",
"datatype": ["ascii", 16]
}, {
"name":
"T_OFFSET",
"datatype":
"float64"
}, {
"name":
"DECAY_PEAK",
"datatype":
"float64"
}, {
"name":
"DECAY_FREQ",
"datatype":
"float64"
}, {
"name":
"TAU",
"datatype":
"float64"
}]
}
}
}
]
}
x = np.array([("string", 1., 2., 3., 4.)],
dtype=[('TYPE', 'S16'), ('T_OFFSET', np.float32),
('DECAY_PEAK', np.float32), ('DECAY_FREQ', np.float32),
('TAU', np.float32)])
m = DataModel(schema=schema_narrow)
m.data = x
m.to_fits(file_path, overwrite=True)
with fits.open(file_path, memmap=False) as hdulist:
assert records_equal(x, np.asarray(hdulist[1].data))
assert hdulist[1].data.dtype[1].str == '>f4'
assert hdulist[1].header['TFORM2'] == 'E'
with DataModel(file_path, schema=schema_wide) as m:
m.to_fits(file_path2, overwrite=True)
with fits.open(file_path2, memmap=False) as hdulist:
assert records_equal(x, np.asarray(hdulist[1].data))
assert hdulist[1].data.dtype[1].str == '>f8'
assert hdulist[1].header['TFORM2'] == 'D'
def test_data_array(tmp_path):
file_path = tmp_path / "test.fits"
file_path2 = tmp_path / "test2.fits"
data_array_schema = {
"allOf": [
asdf.schema.load_schema("http://example.com/schemas/core_metadata",
resolve_references=True), {
"type": "object",
"properties": {
"arr": {
'title': 'An array of data',
'type': 'array',
"fits_hdu": ["FOO", "DQ"],
"items": {
"title": "entry",
"type": "object",
"properties": {
"data": {
"fits_hdu": "FOO",
"default": 0.0,
"max_ndim": 2,
"datatype":
"float64"
},
"dq": {
"fits_hdu": "DQ",
"default": 1,
"datatype": "uint8"
},
}
}
}
}
}
]
}
array1 = np.random.rand(5, 5)
array2 = np.random.rand(5, 5)
array3 = np.random.rand(5, 5)
with DataModel(schema=data_array_schema) as x:
x.arr.append(x.arr.item())
x.arr[0].data = array1
assert len(x.arr) == 1
x.arr.append(x.arr.item(data=array2))
assert len(x.arr) == 2
x.arr.append({})
assert len(x.arr) == 3
x.arr[2].data = array3
del x.arr[1]
assert len(x.arr) == 2
x.to_fits(file_path)
with DataModel(file_path, schema=data_array_schema) as x:
assert len(x.arr) == 2
assert_array_almost_equal(x.arr[0].data, array1)
assert_array_almost_equal(x.arr[1].data, array3)
del x.arr[0]
assert len(x.arr) == 1
x.arr = []
assert len(x.arr) == 0
x.arr.append({'data': np.empty((5, 5))})
assert len(x.arr) == 1
x.arr.extend([
x.arr.item(data=np.empty((5, 5))),
x.arr.item(data=np.empty((5, 5)),
dq=np.empty((5, 5), dtype=np.uint8))
])
assert len(x.arr) == 3
del x.arr[1]
assert len(x.arr) == 2
x.to_fits(file_path2, overwrite=True)
with fits.open(file_path2) as hdulist:
x = set()
for hdu in hdulist:
x.add((hdu.header.get('EXTNAME'), hdu.header.get('EXTVER')))
assert x == set([('FOO', 2), ('FOO', 1), ('ASDF', None), ('DQ', 2),
(None, None)])
def test_from_fits_write(dm):
dm.to_fits(TMP_FITS, overwrite=True)
return DataModel.from_fits(TMP_FITS)
def test_delete():
with DataModel() as dm:
dm.meta.instrument.name = 'NIRCAM'
assert dm.meta.instrument.name == 'NIRCAM'
del dm.meta.instrument.name
assert dm.meta.instrument.name is None
def test_copy_model():
with DataModel() as dm:
with DataModel(dm) as dm2:
assert hasattr(dm2, 'meta')
def test_history_from_model_to_fits(tmpdir):
tmpfits = str(tmpdir.join('tmp.fits'))
m = DataModel()
m.history = [
HistoryEntry({
'description': 'First entry',
'time': Time(datetime.datetime.now())
})
]
m.history.append(
HistoryEntry({
'description': 'Second entry',
'time': Time(datetime.datetime.now())
}))
m.save(tmpfits)
with fits.open(tmpfits, memmap=False) as hdulist:
assert list(
hdulist[0].header['HISTORY']) == ["First entry", "Second entry"]
with DataModel(tmpfits) as m2:
m2 = DataModel()
m2.update(m)
m2.history = m.history
assert m2.history == [{
'description': "First entry"
}, {
'description': "Second entry"
}]
m2.save(tmpfits)
with fits.open(tmpfits, memmap=False) as hdulist:
assert list(
hdulist[0].header['HISTORY']) == ["First entry", "Second entry"]