def test_open_fits_model_s3(s3_root_dir): path = str(s3_root_dir.join("test.fits")) with DataModel() as dm: dm.save(path) model = DataModel("s3://test-s3-data/test.fits") assert isinstance(model, DataModel)
def test_history_from_fits(tmpdir): tmpfits = str(tmpdir.join('tmp.fits')) header = fits.Header() header['HISTORY'] = "First entry" header['HISTORY'] = "Second entry" fits.writeto(tmpfits, np.array([]), header, overwrite=True) with DataModel(tmpfits) as m: assert m.history == [{ 'description': 'First entry' }, { 'description': 'Second entry' }] del m.history[0] m.history.append(HistoryEntry({'description': "Third entry"})) assert m.history == [{ 'description': "Second entry" }, { 'description': "Third entry" }] m.save(tmpfits) with DataModel(tmpfits) as m: assert m.history == [{ 'description': "Second entry" }, { 'description': "Third entry" }]
def test_table_with_unsigned_int(): schema = { 'title': 'Test data model', '$schema': 'http://stsci.edu/schemas/fits-schema/fits-schema', 'type': 'object', 'properties': { 'meta': { 'type': 'object', 'properties': {} }, 'test_table': { 'title': 'Test table', 'fits_hdu': 'TESTTABL', 'datatype': [ {'name': 'FLOAT64_COL', 'datatype': 'float64'}, {'name': 'UINT32_COL', 'datatype': 'uint32'} ] } } } with DataModel(schema=schema) as dm: float64_info = np.finfo(np.float64) float64_arr = np.random.uniform(size=(10,)) float64_arr[0] = float64_info.min float64_arr[-1] = float64_info.max uint32_info = np.iinfo(np.uint32) uint32_arr = np.random.randint(uint32_info.min, uint32_info.max + 1, size=(10,), dtype=np.uint32) uint32_arr[0] = uint32_info.min uint32_arr[-1] = uint32_info.max test_table = np.array(list(zip(float64_arr, uint32_arr)), dtype=dm.test_table.dtype) def assert_table_correct(model): for idx, (col_name, col_data) in enumerate([('float64_col', float64_arr), ('uint32_col', uint32_arr)]): # The table dtype and field dtype are stored separately, and may not # necessarily agree. assert np.can_cast(model.test_table.dtype[idx], col_data.dtype, 'equiv') assert np.can_cast(model.test_table.field(col_name).dtype, col_data.dtype, 'equiv') assert np.array_equal(model.test_table.field(col_name), col_data) # The datamodel casts our array to FITS_rec on assignment, so here we're # checking that the data survived the casting. dm.test_table = test_table assert_table_correct(dm) # Confirm that saving the table (and converting the uint32 values to signed int w/TZEROn) # doesn't mangle the data. dm.save(TMP_FITS) assert_table_correct(dm) # Confirm that the data loads from the file intact (converting the signed ints back to # the appropriate uint32 values). with DataModel(TMP_FITS, schema=schema) as dm2: assert_table_correct(dm2)
def test_open_asdf_no_datamodel_class(tmp_path, suffix): path = str(tmp_path / f"no_model.{suffix}") model = DataModel() model.save(path) # Note: only the fits open emits a "model_type not found" warning. Both # fits and asdf should behave the same with datamodels.open(path) as m: assert isinstance(m, DataModel)
def test_ad_hoc_attributes(filename, tmp_path): """ Test that attributes unrecognized by the schema can still be assigned and written. """ file_path = tmp_path / filename with DataModel() as dm: dm.meta.foo = {'a': 42, 'b': ['a', 'b', 'c']} dm.save(file_path) with DataModel(file_path) as dm2: assert dm2.meta.foo == {'a': 42, 'b': ['a', 'b', 'c']}
def test_extra_fits(): path = os.path.join(ROOT_DIR, "headers.fits") assert os.path.exists(path) with DataModel(path) as dm: assert 'BITPIX' not in _header_to_dict(dm.extra_fits.PRIMARY.header) assert _header_to_dict(dm.extra_fits.PRIMARY.header)['SCIYSTRT'] == 705 dm2 = dm.copy() dm2.to_fits(TMP_FITS, overwrite=True) with DataModel(TMP_FITS) as dm: assert 'BITPIX' not in _header_to_dict(dm.extra_fits.PRIMARY.header) assert _header_to_dict(dm.extra_fits.PRIMARY.header)['SCIYSTRT'] == 705
def test_fits_without_sci(): from astropy.io import fits schema = { "allOf": [ mschema.load_schema(os.path.join(os.path.dirname(__file__), "../schemas/core.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "coeffs": { 'max_ndim': 1, 'fits_hdu': 'COEFFS', 'datatype': 'float32' } } } ] } fits = fits.HDUList([ fits.PrimaryHDU(), fits.ImageHDU(name='COEFFS', data=np.array([0.0], np.float32)) ]) with DataModel(fits, schema=schema) as dm: assert_array_equal(dm.coeffs, [0.0])
def test_historylist_methods(): m = DataModel() h1 = m.history info = "First entry" h1.append(info) assert h1 == info, "Append new history entry" h2 = m.history assert h2 == info, "Two history lists point to the same object" assert len(h1) == 1, "Length of a history list" entry = h1[0] assert entry["description"] == info, "Get history list item" info += " for real" h1[0] = info assert h1 == info, "Set history list item" del h1[0] assert len(h1) == 0, "Delete history list item" info = ("First entry", "Second_entry", "Third entry") h1.extend(info) assert len(h1) == 3, "Length of extended history list" assert h1 == info, "Contents of extended history list" for entry, item in zip(h1, info): assert entry["description"] == item, "Iterate over history list" h1.clear() assert len(h1) == 0, "Clear history list"
def test_open_asdf_model(init): # Open an empty asdf file, pass extra arguments with DataModel(init=init, ignore_version_mismatch=False, ignore_unrecognized_tag=True) as model: assert not model._asdf._ignore_version_mismatch assert model._asdf._ignore_unrecognized_tag
def _create_source(): dm = DataModel(FITS_FILE) assert dm.meta.instrument.name == 'MIRI' dm.meta.instrument.name = 'NIRCAM' dm.meta.subarray.xstart = 42 return dm
def test_to_flat_dict(): with DataModel() as x: x.meta.origin = 'FOO' assert x['meta.origin'] == 'FOO' d = x.to_flat_dict() assert d['meta.origin'] == 'FOO'
def test_open_asdf_model(tmp_path): # Open an empty asdf file, pass extra arguments with DataModel(ignore_version_mismatch=False, ignore_unrecognized_tag=True) as model: assert not model._asdf._ignore_version_mismatch assert model._asdf._ignore_unrecognized_tag file_path = tmp_path / "test.asdf" with asdf.AsdfFile() as af: af.write_to(file_path) with DataModel(file_path, ignore_version_mismatch=False, ignore_unrecognized_tag=True) as model: assert not model._asdf._ignore_version_mismatch assert model._asdf._ignore_unrecognized_tag
def test_open_asdf_s3(s3_root_dir): """Test opening a model from an ASDF file on S3""" path = str(s3_root_dir.join("test.asdf")) with DataModel() as dm: dm.save(path) with stdatamodels.open("s3://test-s3-data/test.asdf") as m: assert isinstance(m, DataModel)
def make_models(tmpdir_factory): """Create basic models Returns ------- path_just_fits, path_model : (str, str) `path_just_fits` is a FITS file of `DataModel` without the ASDF extension. `path_model` is a FITS file of `DataModel` with the ASDF extension. """ path = tmpdir_factory.mktemp('skip_fits_update') path_just_fits = str(path / 'just_fits.fits') path_model = str(path / 'model.fits') primary_hdu = fits.PrimaryHDU() primary_hdu.header['exp_type'] = 'NRC_IMAGE' hduls = fits.HDUList([primary_hdu]) hduls.writeto(path_just_fits) model = DataModel(hduls) model.save(path_model) return {'just_fits': path_just_fits, 'model': path_model}
def test_metadata_from_fits(): from astropy.io import fits mask = np.array([[0, 1], [2, 3]]) fits.ImageHDU(data=mask, name='DQ').writeto(TMP_FITS, overwrite=True) with DataModel(init=TMP_FITS) as dm: dm.save(TMP_FITS2) with fits.open(TMP_FITS2, memmap=False) as hdulist: assert hdulist[2].name == 'ASDF'
def test_add_schema_entry(): with DataModel(strict_validation=True) as dm: dm.add_schema_entry('meta.foo.bar', {'enum': ['foo', 'bar', 'baz']}) dm.meta.foo.bar dm.meta.foo.bar = 'bar' try: dm.meta.foo.bar = 'what?' except jsonschema.ValidationError: pass else: assert False
def test_extra_fits(tmp_path): file_path = tmp_path / "test.fits" with FitsModel() as dm: dm.save(file_path) with fits.open(file_path) as hdul: hdul[0].header["FOO"] = "BAR" hdul.writeto(file_path, overwrite=True) with DataModel(file_path) as dm: assert any(h for h in dm.extra_fits.PRIMARY.header if h == ["FOO", "BAR", ""])
def test_stringify(tmpdir): im = DataModel() assert str(im) == '<DataModel>' im = ImageModel((10, 100)) assert str(im) == '<ImageModel(10, 100)>' path = str(tmpdir.join("nircam_mask.fits")) m = MaskModel((2048, 2048)) m.save(path) m.close() with MaskModel(path) as im: assert str(im) == '<MaskModel(2048, 2048) from nircam_mask.fits>'
def test_stringify(tmp_path): dm = DataModel() assert str(dm) == '<DataModel>' dm = BasicModel((10, 100)) assert str(dm) == '<BasicModel(10, 100)>' file_path = tmp_path / "test.asdf" dm.save(file_path) dm.close() with BasicModel(file_path) as dm: assert str(dm) == '<BasicModel(10, 100) from test.asdf>'
def test_replace_table(): from astropy.io import fits schema_narrow = { "allOf": [ mschema.load_schema(os.path.join(os.path.dirname(__file__), "../schemas/core.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "data": { "title": "relative sensitivity table", "fits_hdu": "RELSENS", "datatype": [{ "name": "TYPE", "datatype": ["ascii", 16] }, { "name": "T_OFFSET", "datatype": "float32" }, { "name": "DECAY_PEAK", "datatype": "float32" }, { "name": "DECAY_FREQ", "datatype": "float32" }, { "name": "TAU", "datatype": "float32" }] } } } ] } schema_wide = { "allOf": [ mschema.load_schema(os.path.join(os.path.dirname(__file__), "../schemas/core.schema.yaml"), resolve_references=True), { "type": "object", "properties": { "data": { "title": "relative sensitivity table", "fits_hdu": "RELSENS", "datatype": [{ "name": "TYPE", "datatype": ["ascii", 16] }, { "name": "T_OFFSET", "datatype": "float64" }, { "name": "DECAY_PEAK", "datatype": "float64" }, { "name": "DECAY_FREQ", "datatype": "float64" }, { "name": "TAU", "datatype": "float64" }] } } } ] } x = np.array([("string", 1., 2., 3., 4.)], dtype=[('TYPE', 'S16'), ('T_OFFSET', np.float32), ('DECAY_PEAK', np.float32), ('DECAY_FREQ', np.float32), ('TAU', np.float32)]) m = DataModel(schema=schema_narrow) m.data = x m.to_fits(TMP_FITS, overwrite=True) with fits.open(TMP_FITS, memmap=False) as hdulist: assert records_equal(x, np.asarray(hdulist[1].data)) assert hdulist[1].data.dtype[1].str == '>f4' assert hdulist[1].header['TFORM2'] == 'E' with DataModel(TMP_FITS, schema=schema_wide) as m: m.to_fits(TMP_FITS2, overwrite=True) with fits.open(TMP_FITS2, memmap=False) as hdulist: assert records_equal(x, np.asarray(hdulist[1].data)) assert hdulist[1].data.dtype[1].str == '>f8' assert hdulist[1].header['TFORM2'] == 'D'
def test_base_model_has_no_arrays(): with pytest.raises(AttributeError): with DataModel() as dm: dm.data
def test_hasattr(): model = DataModel() assert model.meta.hasattr('date') assert not model.meta.hasattr('filename')
def test_datamodel_raises_filenotfound(): with pytest.raises(FileNotFoundError): DataModel(init='file_does_not_exist.fits')
def test_subarray(): with DataModel(FITS_FILE) as dm: dm.meta.subarray.xstart
def test_replace_table(tmp_path): file_path = tmp_path / "test.fits" file_path2 = tmp_path / "test2.fits" schema_narrow = { "allOf": [ asdf.schema.load_schema("http://example.com/schemas/core_metadata", resolve_references=True), { "type": "object", "properties": { "data": { "title": "relative sensitivity table", "fits_hdu": "RELSENS", "datatype": [{ "name": "TYPE", "datatype": ["ascii", 16] }, { "name": "T_OFFSET", "datatype": "float32" }, { "name": "DECAY_PEAK", "datatype": "float32" }, { "name": "DECAY_FREQ", "datatype": "float32" }, { "name": "TAU", "datatype": "float32" }] } } } ] } schema_wide = { "allOf": [ asdf.schema.load_schema("http://example.com/schemas/core_metadata", resolve_references=True), { "type": "object", "properties": { "data": { "title": "relative sensitivity table", "fits_hdu": "RELSENS", "datatype": [{ "name": "TYPE", "datatype": ["ascii", 16] }, { "name": "T_OFFSET", "datatype": "float64" }, { "name": "DECAY_PEAK", "datatype": "float64" }, { "name": "DECAY_FREQ", "datatype": "float64" }, { "name": "TAU", "datatype": "float64" }] } } } ] } x = np.array([("string", 1., 2., 3., 4.)], dtype=[('TYPE', 'S16'), ('T_OFFSET', np.float32), ('DECAY_PEAK', np.float32), ('DECAY_FREQ', np.float32), ('TAU', np.float32)]) m = DataModel(schema=schema_narrow) m.data = x m.to_fits(file_path, overwrite=True) with fits.open(file_path, memmap=False) as hdulist: assert records_equal(x, np.asarray(hdulist[1].data)) assert hdulist[1].data.dtype[1].str == '>f4' assert hdulist[1].header['TFORM2'] == 'E' with DataModel(file_path, schema=schema_wide) as m: m.to_fits(file_path2, overwrite=True) with fits.open(file_path2, memmap=False) as hdulist: assert records_equal(x, np.asarray(hdulist[1].data)) assert hdulist[1].data.dtype[1].str == '>f8' assert hdulist[1].header['TFORM2'] == 'D'
def test_data_array(tmp_path): file_path = tmp_path / "test.fits" file_path2 = tmp_path / "test2.fits" data_array_schema = { "allOf": [ asdf.schema.load_schema("http://example.com/schemas/core_metadata", resolve_references=True), { "type": "object", "properties": { "arr": { 'title': 'An array of data', 'type': 'array', "fits_hdu": ["FOO", "DQ"], "items": { "title": "entry", "type": "object", "properties": { "data": { "fits_hdu": "FOO", "default": 0.0, "max_ndim": 2, "datatype": "float64" }, "dq": { "fits_hdu": "DQ", "default": 1, "datatype": "uint8" }, } } } } } ] } array1 = np.random.rand(5, 5) array2 = np.random.rand(5, 5) array3 = np.random.rand(5, 5) with DataModel(schema=data_array_schema) as x: x.arr.append(x.arr.item()) x.arr[0].data = array1 assert len(x.arr) == 1 x.arr.append(x.arr.item(data=array2)) assert len(x.arr) == 2 x.arr.append({}) assert len(x.arr) == 3 x.arr[2].data = array3 del x.arr[1] assert len(x.arr) == 2 x.to_fits(file_path) with DataModel(file_path, schema=data_array_schema) as x: assert len(x.arr) == 2 assert_array_almost_equal(x.arr[0].data, array1) assert_array_almost_equal(x.arr[1].data, array3) del x.arr[0] assert len(x.arr) == 1 x.arr = [] assert len(x.arr) == 0 x.arr.append({'data': np.empty((5, 5))}) assert len(x.arr) == 1 x.arr.extend([ x.arr.item(data=np.empty((5, 5))), x.arr.item(data=np.empty((5, 5)), dq=np.empty((5, 5), dtype=np.uint8)) ]) assert len(x.arr) == 3 del x.arr[1] assert len(x.arr) == 2 x.to_fits(file_path2, overwrite=True) with fits.open(file_path2) as hdulist: x = set() for hdu in hdulist: x.add((hdu.header.get('EXTNAME'), hdu.header.get('EXTVER'))) assert x == set([('FOO', 2), ('FOO', 1), ('ASDF', None), ('DQ', 2), (None, None)])
def test_from_fits_write(dm): dm.to_fits(TMP_FITS, overwrite=True) return DataModel.from_fits(TMP_FITS)
def test_delete(): with DataModel() as dm: dm.meta.instrument.name = 'NIRCAM' assert dm.meta.instrument.name == 'NIRCAM' del dm.meta.instrument.name assert dm.meta.instrument.name is None
def test_copy_model(): with DataModel() as dm: with DataModel(dm) as dm2: assert hasattr(dm2, 'meta')
def test_history_from_model_to_fits(tmpdir): tmpfits = str(tmpdir.join('tmp.fits')) m = DataModel() m.history = [ HistoryEntry({ 'description': 'First entry', 'time': Time(datetime.datetime.now()) }) ] m.history.append( HistoryEntry({ 'description': 'Second entry', 'time': Time(datetime.datetime.now()) })) m.save(tmpfits) with fits.open(tmpfits, memmap=False) as hdulist: assert list( hdulist[0].header['HISTORY']) == ["First entry", "Second entry"] with DataModel(tmpfits) as m2: m2 = DataModel() m2.update(m) m2.history = m.history assert m2.history == [{ 'description': "First entry" }, { 'description': "Second entry" }] m2.save(tmpfits) with fits.open(tmpfits, memmap=False) as hdulist: assert list( hdulist[0].header['HISTORY']) == ["First entry", "Second entry"]